<commit_before>package main

import (
	"encoding/json"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"html/template"
	"log"
	"net/http"
	"os"
	"strconv"
)

var (
	storeKey = []byte("dtf")
	store    = sessions.NewFilesystemStore("cookies.jar", storeKey)
)

// Server is an abstraction that runs via a web interface
type Server struct {
	DB     *Database
	router http.Handler
	logger http.Handler
}

// NewServer instantiates a server with an active database
func NewServer(db *Database) *Server {
	s := Server{DB: db}
	s.router = s.BuildRouter()

	http.Handle("/", s.router)
	s.logger = handlers.LoggingHandler(os.Stdout, s.router)

	return &s
}

// StartHandler is the entry point to the web GUI
//
// If not authenticated, it allows for registration.
// If authenticated and a tournament is running, show that tournament.
// If authenticated and no tournament is running, show a list of tournaments.
func (s *Server) StartHandler(w http.ResponseWriter, r *http.Request) {
	t := getTemplates("static/tournaments.html")
	data := struct {
		Tournaments []*Tournament
	}{
		s.DB.Tournaments,
	}

	render(t, w, r, data)
}

// NewHandler shows the page to create a new tournament
func (s *Server) NewHandler(w http.ResponseWriter, r *http.Request) {
	// If there is a post to this URL, it means we are making a new tournament
	if r.Method == "POST" {
		name := r.PostFormValue("name")
		id := r.PostFormValue("id")
		t, _ := NewTournament(name, id, s.DB)
		log.Printf("Created tournament %s!", t.Name)

		s.DB.Tournaments = append(s.DB.Tournaments, t)
		s.DB.tournamentRef[t.ID] = t

		http.Redirect(w, r, t.URL(), 302)
		return
	}

	// Elsewise, show the GUI
	t := getTemplates("static/new.html")
	render(t, w, r, struct{}{})
}

// TournamentHandler shows the tournament view and handles tournaments
func (s *Server) TournamentHandler(w http.ResponseWriter, r *http.Request) {
	canJoin := false
	vars := mux.Vars(r)

	tm := s.DB.tournamentRef[vars["id"]]
	session, _ := store.Get(r, tm.Name)
	if name, ok := session.Values["player"]; ok {
		canJoin = tm.CanJoin(name.(string))
	} else {
		canJoin = true
	}

	data := struct {
		Tournament *Tournament
		CanJoin    bool
	}{
		tm,
		canJoin,
	}

	t := getTemplates("static/tournament.html", "static/player.html", "static/match.html")
	render(t, w, r, data)
}

// JoinHandler shows the tournament view and handles tournaments
func (s *Server) JoinHandler(w http.ResponseWriter, r *http.Request) {
	t := getTemplates("static/join.html")
	tm := s.getTournament(r)
	data := struct {
		Tournament *Tournament
	}{
		tm,
	}

	if r.Method == "POST" {
		name := r.PostFormValue("name")
		if !tm.CanJoin(name) {
			http.Error(w, "too many players", 500)
			return
		}
		color := r.PostFormValue("color")
		if color == "" {
			http.Error(w, "need a color", 500)
			return
		}
		err := tm.AddPlayer(name, color)

		// TODO: This should not be here...
		tm.SetMatchPointers()

		if err != nil {
			// TODO: Flash error message
			http.Error(w, err.Error(), 500)
			return
		}

		log.Printf("%s has joined %s!", name, tm.Name)
		session, err := store.Get(r, tm.Name)
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}

		// TODO: Does not work. :/
		session.Values["player"] = name
		session.Save(r, w)

		http.Redirect(w, r, tm.URL(), 302)
		return
	}

	render(t, w, r, data)
}

// StartTournamentHandler starts tournaments
func (s *Server) StartTournamentHandler(w http.ResponseWriter, r *http.Request) {
	tm := s.getTournament(r)
	err := tm.StartTournament()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	http.Redirect(w, r, tm.URL(), 302)
}

// NextHandler starts tournaments
func (s *Server) NextHandler(w http.ResponseWriter, r *http.Request) {
	tm := s.getTournament(r)
	m, err := tm.NextMatch()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	http.Redirect(w, r, m.URL(), 302)
}

// MatchHandler shows the Match view and handles Matches
func (s *Server) MatchHandler(w http.ResponseWriter, r *http.Request) {
	data := struct {
		Match *Match
	}{
		s.getMatch(r),
	}

	t := getTemplates("static/matchcontrol.html", "static/player.html", "static/match.html")
	render(t, w, r, data)
}

// MatchToggleHandler starts and stops matches
func (s *Server) MatchToggleHandler(w http.ResponseWriter, r *http.Request) {
	m := s.getMatch(r)
	if !m.IsStarted() {
		log.Printf("%s started", m.String())
		m.Start()
	} else {
		log.Printf("%s ended", m.String())
		m.End()
	}

	http.Redirect(w, r, m.URL(), 302)
}

// ActionHandler handles judge requests for player action
func (s *Server) ActionHandler(w http.ResponseWriter, r *http.Request) {
	m := s.getMatch(r)
	if !m.IsOpen() {
		log.Print("Not allowing actions on non-started matches")
		http.Redirect(w, r, m.URL(), 302)
		return
	}

	vars := mux.Vars(r)
	index, _ := strconv.Atoi(vars["player"])
	m.Players[index].Action(vars["action"], vars["dir"])

	http.Redirect(w, r, m.URL(), 302)
}

// TournamentListHandler returns a list of all tournaments
func (s *Server) TournamentListHandler(w http.ResponseWriter, r *http.Request) {
	tournaments := s.DB.Tournaments
	// tournaments := sort.Reverse(s.DB.Tournaments)
	data, err := json.Marshal(tournaments)
	if err != nil {
		log.Fatal(err)
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(data)
}

// BuildRouter sets up the routes
func (s *Server) BuildRouter() http.Handler {
	n := mux.NewRouter()
	r := n.PathPrefix("/api/towerfall").Subrouter()

	m := "/{id}/{kind:(tryout|runnerup|semi|final)}/{index:[0-9]+}"
	r.HandleFunc("/tournament/", s.TournamentListHandler)
	r.HandleFunc("/new", s.NewHandler)
	r.HandleFunc("/{id}/", s.TournamentHandler)
	r.HandleFunc("/{id}/start", s.StartTournamentHandler)
	r.HandleFunc("/{id}/join", s.JoinHandler)
	r.HandleFunc("/{id}/next", s.NextHandler)
	r.HandleFunc(m, s.MatchHandler)
	r.HandleFunc(m+"/toggle", s.MatchToggleHandler)
	r.HandleFunc(m+"/{player:[0-3]}/{action}/{dir:(up|down)}", s.ActionHandler)

	return n
}

// Serve serves forever
func (s *Server) Serve() error {
	log.Print("Listening on :45000")
	return http.ListenAndServe(":45000", s.logger)
}

// getTemplates gets a template with the context set to `extra`, with index.html backing it.
func getTemplates(items ...string) *template.Template {
	items = append(items, "static/index.html")
	t, err := template.ParseFiles(items...)
	if err != nil {
		log.Fatal(err)
	}

	return t
}

func render(t *template.Template, w http.ResponseWriter, r *http.Request, data interface{}) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	err := t.ExecuteTemplate(w, "base", data)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	db, err := NewDatabase("production.db")
	if err != nil {
		log.Fatal(err)
	}

	err = NewServer(db).Serve()
	if err != nil {
		log.Fatal(err)
	}
}

func (s *Server) getMatch(r *http.Request) *Match {
	var m *Match
	vars := mux.Vars(r)

	tm := s.DB.tournamentRef[vars["id"]]
	kind := vars["kind"]
	index, _ := strconv.Atoi(vars["index"])

	if kind == "tryout" {
		m = tm.Tryouts[index]
	} else if kind == "semi" {
		m = tm.Semis[index]
	} else if kind == "final" {
		m = tm.Final
	}

	return m
}

func (s *Server) getTournament(r *http.Request) *Tournament {
	vars := mux.Vars(r)
	tm := s.DB.tournamentRef[vars["id"]]
	return tm
}
<commit_msg>Make matches be handled as a Subrouter()<commit_after>package main

import (
	"encoding/json"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"html/template"
	"log"
	"net/http"
	"os"
	"strconv"
)

var (
	storeKey = []byte("dtf")
	store    = sessions.NewFilesystemStore("cookies.jar", storeKey)
)

// Server is an abstraction that runs via a web interface
type Server struct {
	DB     *Database
	router http.Handler
	logger http.Handler
}

// NewServer instantiates a server with an active database
func NewServer(db *Database) *Server {
	s := Server{DB: db}
	s.router = s.BuildRouter()

	http.Handle("/", s.router)
	s.logger = handlers.LoggingHandler(os.Stdout, s.router)

	return &s
}

// StartHandler is the entry point to the web GUI
//
// If not authenticated, it allows for registration.
// If authenticated and a tournament is running, show that tournament.
// If authenticated and no tournament is running, show a list of tournaments.
func (s *Server) StartHandler(w http.ResponseWriter, r *http.Request) {
	t := getTemplates("static/tournaments.html")
	data := struct {
		Tournaments []*Tournament
	}{
		s.DB.Tournaments,
	}

	render(t, w, r, data)
}

// NewHandler shows the page to create a new tournament
func (s *Server) NewHandler(w http.ResponseWriter, r *http.Request) {
	// If there is a post to this URL, it means we are making a new tournament
	if r.Method == "POST" {
		name := r.PostFormValue("name")
		id := r.PostFormValue("id")
		t, _ := NewTournament(name, id, s.DB)
		log.Printf("Created tournament %s!", t.Name)

		s.DB.Tournaments = append(s.DB.Tournaments, t)
		s.DB.tournamentRef[t.ID] = t

		http.Redirect(w, r, t.URL(), 302)
		return
	}

	// Elsewise, show the GUI
	t := getTemplates("static/new.html")
	render(t, w, r, struct{}{})
}

// TournamentHandler shows the tournament view and handles tournaments
func (s *Server) TournamentHandler(w http.ResponseWriter, r *http.Request) {
	canJoin := false
	vars := mux.Vars(r)

	tm := s.DB.tournamentRef[vars["id"]]
	session, _ := store.Get(r, tm.Name)
	if name, ok := session.Values["player"]; ok {
		canJoin = tm.CanJoin(name.(string))
	} else {
		canJoin = true
	}

	data := struct {
		Tournament *Tournament
		CanJoin    bool
	}{
		tm,
		canJoin,
	}

	t := getTemplates("static/tournament.html", "static/player.html", "static/match.html")
	render(t, w, r, data)
}

// JoinHandler shows the tournament view and handles tournaments
func (s *Server) JoinHandler(w http.ResponseWriter, r *http.Request) {
	t := getTemplates("static/join.html")
	tm := s.getTournament(r)
	data := struct {
		Tournament *Tournament
	}{
		tm,
	}

	if r.Method == "POST" {
		name := r.PostFormValue("name")
		if !tm.CanJoin(name) {
			http.Error(w, "too many players", 500)
			return
		}
		color := r.PostFormValue("color")
		if color == "" {
			http.Error(w, "need a color", 500)
			return
		}
		err := tm.AddPlayer(name, color)

		// TODO: This should not be here...
		tm.SetMatchPointers()

		if err != nil {
			// TODO: Flash error message
			http.Error(w, err.Error(), 500)
			return
		}

		log.Printf("%s has joined %s!", name, tm.Name)
		session, err := store.Get(r, tm.Name)
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}

		// TODO: Does not work. :/
		session.Values["player"] = name
		session.Save(r, w)

		http.Redirect(w, r, tm.URL(), 302)
		return
	}

	render(t, w, r, data)
}

// StartTournamentHandler starts tournaments
func (s *Server) StartTournamentHandler(w http.ResponseWriter, r *http.Request) {
	tm := s.getTournament(r)
	err := tm.StartTournament()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	http.Redirect(w, r, tm.URL(), 302)
}

// NextHandler starts tournaments
func (s *Server) NextHandler(w http.ResponseWriter, r *http.Request) {
	tm := s.getTournament(r)
	m, err := tm.NextMatch()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	http.Redirect(w, r, m.URL(), 302)
}

// MatchHandler shows the Match view and handles Matches
func (s *Server) MatchHandler(w http.ResponseWriter, r *http.Request) {
	data := struct {
		Match *Match
	}{
		s.getMatch(r),
	}

	t := getTemplates("static/matchcontrol.html", "static/player.html", "static/match.html")
	render(t, w, r, data)
}

// MatchToggleHandler starts and stops matches
func (s *Server) MatchToggleHandler(w http.ResponseWriter, r *http.Request) {
	m := s.getMatch(r)
	if !m.IsStarted() {
		log.Printf("%s started", m.String())
		m.Start()
	} else {
		log.Printf("%s ended", m.String())
		m.End()
	}

	http.Redirect(w, r, m.URL(), 302)
}

// ActionHandler handles judge requests for player action
func (s *Server) ActionHandler(w http.ResponseWriter, r *http.Request) {
	m := s.getMatch(r)
	if !m.IsOpen() {
		log.Print("Not allowing actions on non-started matches")
		http.Redirect(w, r, m.URL(), 302)
		return
	}

	vars := mux.Vars(r)
	index, _ := strconv.Atoi(vars["player"])
	m.Players[index].Action(vars["action"], vars["dir"])

	http.Redirect(w, r, m.URL(), 302)
}

// TournamentListHandler returns a list of all tournaments
func (s *Server) TournamentListHandler(w http.ResponseWriter, r *http.Request) {
	tournaments := s.DB.Tournaments
	// tournaments := sort.Reverse(s.DB.Tournaments)
	data, err := json.Marshal(tournaments)
	if err != nil {
		log.Fatal(err)
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(data)
}

// BuildRouter sets up the routes
func (s *Server) BuildRouter() http.Handler {
	n := mux.NewRouter()
	r := n.PathPrefix("/api/towerfall").Subrouter()

	r.HandleFunc("/tournament/", s.TournamentListHandler)
	r.HandleFunc("/new", s.NewHandler)
	r.HandleFunc("/{id}/", s.TournamentHandler)
	r.HandleFunc("/{id}/start", s.StartTournamentHandler)
	r.HandleFunc("/{id}/join", s.JoinHandler)
	r.HandleFunc("/{id}/next", s.NextHandler)

	m := r.PathPrefix("/{id}/{kind:(tryout|runnerup|semi|final)}/{index:[0-9]+}").Subrouter()
	m.HandleFunc("/", s.MatchHandler)
	m.HandleFunc("/toggle", s.MatchToggleHandler)
	m.HandleFunc("/{player:[0-3]}/{action}/{dir:(up|down)}", s.ActionHandler)

	return n
}

// Serve serves forever
func (s *Server) Serve() error {
	log.Print("Listening on :45000")
	return http.ListenAndServe(":45000", s.logger)
}

// getTemplates gets a template with the context set to `extra`, with index.html backing it.
func getTemplates(items ...string) *template.Template {
	items = append(items, "static/index.html")
	t, err := template.ParseFiles(items...)
	if err != nil {
		log.Fatal(err)
	}

	return t
}

func render(t *template.Template, w http.ResponseWriter, r *http.Request, data interface{}) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	err := t.ExecuteTemplate(w, "base", data)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	db, err := NewDatabase("production.db")
	if err != nil {
		log.Fatal(err)
	}

	err = NewServer(db).Serve()
	if err != nil {
		log.Fatal(err)
	}
}

func (s *Server) getMatch(r *http.Request) *Match {
	var m *Match
	vars := mux.Vars(r)

	tm := s.DB.tournamentRef[vars["id"]]
	kind := vars["kind"]
	index, _ := strconv.Atoi(vars["index"])

	if kind == "tryout" {
		m = tm.Tryouts[index]
	} else if kind == "semi" {
		m = tm.Semis[index]
	} else if kind == "final" {
		m = tm.Final
	}

	return m
}

func (s *Server) getTournament(r *http.Request) *Tournament {
	vars := mux.Vars(r)
	tm := s.DB.tournamentRef[vars["id"]]
	return tm
}
<|endoftext|>


<commit_before><commit_msg>update version<commit_after><|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build go1.6

package http2

import (
	"crypto/tls"
	"fmt"
	"net/http"
)

func configureTransport(t1 *http.Transport) (*Transport, error) {
	connPool := new(clientConnPool)
	t2 := &Transport{
		ConnPool: noDialClientConnPool{connPool},
		t1:       t1,
	}
	connPool.t = t2
	if err := registerHTTPSProtocol(t1, noDialH2RoundTripper{t2}); err != nil {
		return nil, err
	}
	if t1.TLSClientConfig == nil {
		t1.TLSClientConfig = new(tls.Config)
	}
	if !strSliceContains(t1.TLSClientConfig.NextProtos, "h2") {
		t1.TLSClientConfig.NextProtos = append([]string{"h2"}, t1.TLSClientConfig.NextProtos...)
	}
	if !strSliceContains(t1.TLSClientConfig.NextProtos, "http/1.1") {
		t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1")
	}
	upgradeFn := func(authority string, c *tls.Conn) http.RoundTripper {
		addr := authorityAddr("https", authority)
		if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil {
			go c.Close()
			return erringRoundTripper{err}
		} else if !used {
			// Turns out we don't need this c.
			// For example, two goroutines made requests to the same host
			// at the same time, both kicking off TCP dials. (since protocol
			// was unknown)
			go c.Close()
		}
		return t2
	}
	if m := t1.TLSNextProto; len(m) == 0 {
		t1.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{
			"h2": upgradeFn,
		}
	} else {
		m["h2"] = upgradeFn
	}
	return t2, nil
}

// registerHTTPSProtocol calls Transport.RegisterProtocol but
// convering panics into errors.
func registerHTTPSProtocol(t *http.Transport, rt http.RoundTripper) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("%v", e)
		}
	}()
	t.RegisterProtocol("https", rt)
	return nil
}

// noDialH2RoundTripper is a RoundTripper which only tries to complete the request
// if there's already has a cached connection to the host.
type noDialH2RoundTripper struct{ t *Transport }

func (rt noDialH2RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	res, err := rt.t.RoundTrip(req)
	if err == ErrNoCachedConn {
		return nil, http.ErrSkipAltProtocol
	}
	return res, err
}
<commit_msg>http2: fix typo in comment<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build go1.6

package http2

import (
	"crypto/tls"
	"fmt"
	"net/http"
)

func configureTransport(t1 *http.Transport) (*Transport, error) {
	connPool := new(clientConnPool)
	t2 := &Transport{
		ConnPool: noDialClientConnPool{connPool},
		t1:       t1,
	}
	connPool.t = t2
	if err := registerHTTPSProtocol(t1, noDialH2RoundTripper{t2}); err != nil {
		return nil, err
	}
	if t1.TLSClientConfig == nil {
		t1.TLSClientConfig = new(tls.Config)
	}
	if !strSliceContains(t1.TLSClientConfig.NextProtos, "h2") {
		t1.TLSClientConfig.NextProtos = append([]string{"h2"}, t1.TLSClientConfig.NextProtos...)
	}
	if !strSliceContains(t1.TLSClientConfig.NextProtos, "http/1.1") {
		t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1")
	}
	upgradeFn := func(authority string, c *tls.Conn) http.RoundTripper {
		addr := authorityAddr("https", authority)
		if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil {
			go c.Close()
			return erringRoundTripper{err}
		} else if !used {
			// Turns out we don't need this c.
			// For example, two goroutines made requests to the same host
			// at the same time, both kicking off TCP dials. (since protocol
			// was unknown)
			go c.Close()
		}
		return t2
	}
	if m := t1.TLSNextProto; len(m) == 0 {
		t1.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{
			"h2": upgradeFn,
		}
	} else {
		m["h2"] = upgradeFn
	}
	return t2, nil
}

// registerHTTPSProtocol calls Transport.RegisterProtocol but
// converting panics into errors.
func registerHTTPSProtocol(t *http.Transport, rt http.RoundTripper) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("%v", e)
		}
	}()
	t.RegisterProtocol("https", rt)
	return nil
}

// noDialH2RoundTripper is a RoundTripper which only tries to complete the request
// if there's already has a cached connection to the host.
type noDialH2RoundTripper struct{ t *Transport }

func (rt noDialH2RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	res, err := rt.t.RoundTrip(req)
	if err == ErrNoCachedConn {
		return nil, http.ErrSkipAltProtocol
	}
	return res, err
}
<|endoftext|>


<commit_before>// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

package core

import "fmt"

// ProdInfo describes a product
type ProdInfo struct {
	Edition string
	Title   string
	Version string
	Major   string
	Minor   string
	Patch   string
}

// Product returns product edition details
func Product() (p ProdInfo) {
	p.Major = "0"
	p.Minor = "20"
	p.Patch = "0"
	p.Version = fmt.Sprintf("%s.%s.%s", p.Major, p.Minor, p.Patch)
	p.Edition = "Community"
	p.Title = fmt.Sprintf("%s Edition", p.Edition)

	return p
}
<commit_msg>Bump to version 0.22.0 (in line with Enterprise version)<commit_after>// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

package core

import "fmt"

// ProdInfo describes a product
type ProdInfo struct {
	Edition string
	Title   string
	Version string
	Major   string
	Minor   string
	Patch   string
}

// Product returns product edition details
func Product() (p ProdInfo) {
	p.Major = "0"
	p.Minor = "22"
	p.Patch = "0"
	p.Version = fmt.Sprintf("%s.%s.%s", p.Major, p.Minor, p.Patch)
	p.Edition = "Community"
	p.Title = fmt.Sprintf("%s Edition", p.Edition)

	return p
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/golang/protobuf/ptypes"
	"github.com/imdario/mergo"
	ipnspath "github.com/ipfs/go-ipfs/path"
	mh "gx/ipfs/QmbZ6Cee2uHjG7hf19qLHppgKDRtaG4CVtMzdmK9VCVqLu/go-multihash"
	"io/ioutil"
	"os"
	"path"
	"time"
)

var ErrorProfileNotFound error = errors.New("Profie not found")

func (n *OpenBazaarNode) GetProfile() (pb.Profile, error) {
	var profile pb.Profile
	f, err := os.Open(path.Join(n.RepoPath, "root", "profile"))
	if err != nil {
		return profile, ErrorProfileNotFound
	}
	defer f.Close()
	err = jsonpb.Unmarshal(f, &profile)
	if err != nil {
		return profile, err
	}
	return profile, nil
}

func (n *OpenBazaarNode) FetchProfile(peerId string) (pb.Profile, error) {
	profile, err := ipfs.ResolveThenCat(n.Context, ipnspath.FromString(path.Join(peerId, "profile")))
	if err != nil || len(profile) == 0 {
		return pb.Profile{}, err
	}
	var pro pb.Profile
	err = jsonpb.UnmarshalString(string(profile), &pro)
	if err != nil {
		return pb.Profile{}, err
	}
	if err := ValidateProfile(&pro); err != nil {
		return pb.Profile{}, err
	}
	return pro, nil
}

func (n *OpenBazaarNode) UpdateProfile(profile *pb.Profile) error {
	mPubkey, err := n.Wallet.MasterPublicKey().ECPubKey()
	if err != nil {
		return err
	}
	if err := ValidateProfile(profile); err != nil {
		return err
	}

	profile.BitcoinPubkey = hex.EncodeToString(mPubkey.SerializeCompressed())
	m := jsonpb.Marshaler{
		EnumsAsInts:  false,
		EmitDefaults: true,
		Indent:       "    ",
		OrigName:     false,
	}
	out, err := m.MarshalToString(profile)
	if err != nil {
		return err
	}
	profilePath := path.Join(n.RepoPath, "root", "profile")
	f, err := os.Create(profilePath)
	defer f.Close()
	if err != nil {
		return err
	}
	if _, err := f.WriteString(out); err != nil {
		return err
	}
	return nil
}

func (n *OpenBazaarNode) PatchProfile(patch map[string]interface{}) error {
	profilePath := path.Join(n.RepoPath, "root", "profile")

	// Read stored profile data
	file, err := os.Open(profilePath)
	if err != nil {
		return err
	}
	d := json.NewDecoder(file)
	d.UseNumber()

	var i interface{}
	err = d.Decode(&i)
	if err != nil {
		return err
	}
	profile := i.(map[string]interface{})

	patchMod, pok := patch["moderator"]
	storedMod, sok := profile["moderator"]
	if pok && sok {
		patchBool, ok := patchMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		storedBool, ok := storedMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		if patchBool && patchBool != storedBool {
			if err := n.SetSelfAsModerator(nil); err != nil {
				return err
			}
		} else if !patchBool && patchBool != storedBool {
			if err := n.RemoveSelfAsModerator(); err != nil {
				return err
			}
		}
	}

	// Assuming that `profile` map contains complete data, as it is read
	// from storage, and `patch` map is possibly incomplete, merge first
	// into second recursively, preserving new fields and adding missing
	// old ones
	if err := mergo.Map(&patch, &profile); err != nil {
		return err
	}

	// Execute UpdateProfile with new profile
	newProfile, err := json.Marshal(patch)
	p := new(pb.Profile)
	if err := jsonpb.Unmarshal(bytes.NewReader(newProfile), p); err != nil {
		return err
	}
	return n.UpdateProfile(p)
}

func (n *OpenBazaarNode) appendCountsToProfile(profile *pb.Profile) (*pb.Profile, error) {
	profile.PeerID = n.IpfsNode.Identity.Pretty()
	if profile.Stats == nil {
		profile.Stats = new(pb.Profile_Stats)
	}
	profile.Stats.ListingCount = uint32(n.GetListingCount())
	profile.Stats.FollowerCount = uint32(n.Datastore.Followers().Count())
	profile.Stats.FollowingCount = uint32(n.Datastore.Following().Count())

	ts, err := ptypes.TimestampProto(time.Now())
	if err != nil {
		return nil, err
	}
	profile.LastModified = ts
	return profile, nil
}

func (n *OpenBazaarNode) updateProfileCounts() error {
	profilePath := path.Join(n.RepoPath, "root", "profile")
	profile := new(pb.Profile)
	_, ferr := os.Stat(profilePath)
	if !os.IsNotExist(ferr) {
		// Read existing file
		file, err := ioutil.ReadFile(profilePath)
		if err != nil {
			return err
		}
		err = jsonpb.UnmarshalString(string(file), profile)
		if err != nil {
			return err
		}
	} else {
		return nil
	}
	profile, err := n.appendCountsToProfile(profile)
	if err != nil {
		return err
	}
	return n.UpdateProfile(profile)
}

func ValidateProfile(profile *pb.Profile) error {
	if len(profile.Handle) > WordMaxCharacters {
		return fmt.Errorf("Handle character length is greater than the max of %d", WordMaxCharacters)
	}
	if len(profile.Name) > WordMaxCharacters {
		return fmt.Errorf("Name character length is greater than the max of %d", WordMaxCharacters)
	}
	if len(profile.Location) > WordMaxCharacters {
		return fmt.Errorf("Location character length is greater than the max of %d", WordMaxCharacters)
	}
	if len(profile.About) > AboutMaxCharacteres {
		return fmt.Errorf("About character length is greater than the max of %d", AboutMaxCharacteres)
	}
	if len(profile.ShortDescription) > ShortDescriptionLength {
		return fmt.Errorf("Short description character length is greater than the max of %d", ShortDescriptionLength)
	}
	if profile.ContactInfo != nil {
		if len(profile.ContactInfo.Website) > URLMaxCharacters {
			return fmt.Errorf("Website character length is greater than the max of %d", URLMaxCharacters)
		}
		if len(profile.ContactInfo.Email) > SentenceMaxCharacters {
			return fmt.Errorf("Email character length is greater than the max of %d", SentenceMaxCharacters)
		}
		if len(profile.ContactInfo.PhoneNumber) > WordMaxCharacters {
			return fmt.Errorf("Phone number character length is greater than the max of %d", WordMaxCharacters)
		}
		if len(profile.ContactInfo.Social) > MaxListItems {
			return fmt.Errorf("Number of social accounts is greater than the max of %d", MaxListItems)
		}
		for _, s := range profile.ContactInfo.Social {
			if len(s.Username) > WordMaxCharacters {
				return fmt.Errorf("Social username character length is greater than the max of %d", WordMaxCharacters)
			}
			if len(s.Type) > WordMaxCharacters {
				return fmt.Errorf("Social account type character length is greater than the max of %d", WordMaxCharacters)
			}
			if len(s.Proof) > URLMaxCharacters {
				return fmt.Errorf("Social proof character length is greater than the max of %d", WordMaxCharacters)
			}
		}
	}
	if profile.ModeratorInfo != nil {
		if len(profile.ModeratorInfo.Description) > AboutMaxCharacteres {
			return fmt.Errorf("Moderator description character length is greater than the max of %d", AboutMaxCharacteres)
		}
		if len(profile.ModeratorInfo.TermsAndConditions) > PolicyMaxCharacters {
			return fmt.Errorf("Moderator terms and conditions character length is greater than the max of %d", PolicyMaxCharacters)
		}
		if len(profile.ModeratorInfo.Languages) > MaxListItems {
			return fmt.Errorf("Moderator number of languages greater than the max of %d", MaxListItems)
		}
		for _, l := range profile.ModeratorInfo.Languages {
			if len(l) > WordMaxCharacters {
				return fmt.Errorf("Moderator language character length is greater than the max of %d", WordMaxCharacters)
			}
		}
		if profile.ModeratorInfo.Fee != nil {
			if profile.ModeratorInfo.Fee.FixedFee != nil {
				if len(profile.ModeratorInfo.Fee.FixedFee.CurrencyCode) > WordMaxCharacters {
					return fmt.Errorf("Moderator fee currency code character length is greater than the max of %d", WordMaxCharacters)
				}
			}
		}
	}
	if profile.AvatarHashes != nil {
		_, err := mh.FromB58String(profile.AvatarHashes.Tiny)
		if err != nil {
			return errors.New("Tiny image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Small)
		if err != nil {
			return errors.New("Small image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Medium)
		if err != nil {
			return errors.New("Medium image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Large)
		if err != nil {
			return errors.New("Large image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Original)
		if err != nil {
			return errors.New("Original image hashes must be multihashes")
		}
	}
	if profile.HeaderHashes != nil {
		_, err := mh.FromB58String(profile.HeaderHashes.Tiny)
		if err != nil {
			return errors.New("Tiny image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Small)
		if err != nil {
			return errors.New("Small image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Medium)
		if err != nil {
			return errors.New("Medium image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Large)
		if err != nil {
			return errors.New("Large image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Original)
		if err != nil {
			return errors.New("Original image hashes must be multihashes")
		}
	}
	if len(profile.BitcoinPubkey) > 66 {
		return fmt.Errorf("Bitcoin public key character length is greater than the max of %d", 66)
	}
	if profile.Stats.AverageRating > 5 {
		return fmt.Errorf("Average rating cannot be greater than %d", 5)
	}
	return nil
}
<commit_msg>Check nil in stats validate profile<commit_after>package core

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/golang/protobuf/ptypes"
	"github.com/imdario/mergo"
	ipnspath "github.com/ipfs/go-ipfs/path"
	mh "gx/ipfs/QmbZ6Cee2uHjG7hf19qLHppgKDRtaG4CVtMzdmK9VCVqLu/go-multihash"
	"io/ioutil"
	"os"
	"path"
	"time"
)

var ErrorProfileNotFound error = errors.New("Profie not found")

func (n *OpenBazaarNode) GetProfile() (pb.Profile, error) {
	var profile pb.Profile
	f, err := os.Open(path.Join(n.RepoPath, "root", "profile"))
	if err != nil {
		return profile, ErrorProfileNotFound
	}
	defer f.Close()
	err = jsonpb.Unmarshal(f, &profile)
	if err != nil {
		return profile, err
	}
	return profile, nil
}

func (n *OpenBazaarNode) FetchProfile(peerId string) (pb.Profile, error) {
	profile, err := ipfs.ResolveThenCat(n.Context, ipnspath.FromString(path.Join(peerId, "profile")))
	if err != nil || len(profile) == 0 {
		return pb.Profile{}, err
	}
	var pro pb.Profile
	err = jsonpb.UnmarshalString(string(profile), &pro)
	if err != nil {
		return pb.Profile{}, err
	}
	if err := ValidateProfile(&pro); err != nil {
		return pb.Profile{}, err
	}
	return pro, nil
}

func (n *OpenBazaarNode) UpdateProfile(profile *pb.Profile) error {
	mPubkey, err := n.Wallet.MasterPublicKey().ECPubKey()
	if err != nil {
		return err
	}
	if err := ValidateProfile(profile); err != nil {
		return err
	}

	profile.BitcoinPubkey = hex.EncodeToString(mPubkey.SerializeCompressed())
	m := jsonpb.Marshaler{
		EnumsAsInts:  false,
		EmitDefaults: true,
		Indent:       "    ",
		OrigName:     false,
	}
	out, err := m.MarshalToString(profile)
	if err != nil {
		return err
	}
	profilePath := path.Join(n.RepoPath, "root", "profile")
	f, err := os.Create(profilePath)
	defer f.Close()
	if err != nil {
		return err
	}
	if _, err := f.WriteString(out); err != nil {
		return err
	}
	return nil
}

func (n *OpenBazaarNode) PatchProfile(patch map[string]interface{}) error {
	profilePath := path.Join(n.RepoPath, "root", "profile")

	// Read stored profile data
	file, err := os.Open(profilePath)
	if err != nil {
		return err
	}
	d := json.NewDecoder(file)
	d.UseNumber()

	var i interface{}
	err = d.Decode(&i)
	if err != nil {
		return err
	}
	profile := i.(map[string]interface{})

	patchMod, pok := patch["moderator"]
	storedMod, sok := profile["moderator"]
	if pok && sok {
		patchBool, ok := patchMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		storedBool, ok := storedMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		if patchBool && patchBool != storedBool {
			if err := n.SetSelfAsModerator(nil); err != nil {
				return err
			}
		} else if !patchBool && patchBool != storedBool {
			if err := n.RemoveSelfAsModerator(); err != nil {
				return err
			}
		}
	}

	// Assuming that `profile` map contains complete data, as it is read
	// from storage, and `patch` map is possibly incomplete, merge first
	// into second recursively, preserving new fields and adding missing
	// old ones
	if err := mergo.Map(&patch, &profile); err != nil {
		return err
	}

	// Execute UpdateProfile with new profile
	newProfile, err := json.Marshal(patch)
	p := new(pb.Profile)
	if err := jsonpb.Unmarshal(bytes.NewReader(newProfile), p); err != nil {
		return err
	}
	return n.UpdateProfile(p)
}

func (n *OpenBazaarNode) appendCountsToProfile(profile *pb.Profile) (*pb.Profile, error) {
	profile.PeerID = n.IpfsNode.Identity.Pretty()
	if profile.Stats == nil {
		profile.Stats = new(pb.Profile_Stats)
	}
	profile.Stats.ListingCount = uint32(n.GetListingCount())
	profile.Stats.FollowerCount = uint32(n.Datastore.Followers().Count())
	profile.Stats.FollowingCount = uint32(n.Datastore.Following().Count())

	ts, err := ptypes.TimestampProto(time.Now())
	if err != nil {
		return nil, err
	}
	profile.LastModified = ts
	return profile, nil
}

func (n *OpenBazaarNode) updateProfileCounts() error {
	profilePath := path.Join(n.RepoPath, "root", "profile")
	profile := new(pb.Profile)
	_, ferr := os.Stat(profilePath)
	if !os.IsNotExist(ferr) {
		// Read existing file
		file, err := ioutil.ReadFile(profilePath)
		if err != nil {
			return err
		}
		err = jsonpb.UnmarshalString(string(file), profile)
		if err != nil {
			return err
		}
	} else {
		return nil
	}
	profile, err := n.appendCountsToProfile(profile)
	if err != nil {
		return err
	}
	return n.UpdateProfile(profile)
}

func ValidateProfile(profile *pb.Profile) error {
	if len(profile.Handle) > WordMaxCharacters {
		return fmt.Errorf("Handle character length is greater than the max of %d", WordMaxCharacters)
	}
	if len(profile.Name) > WordMaxCharacters {
		return fmt.Errorf("Name character length is greater than the max of %d", WordMaxCharacters)
	}
	if len(profile.Location) > WordMaxCharacters {
		return fmt.Errorf("Location character length is greater than the max of %d", WordMaxCharacters)
	}
	if len(profile.About) > AboutMaxCharacteres {
		return fmt.Errorf("About character length is greater than the max of %d", AboutMaxCharacteres)
	}
	if len(profile.ShortDescription) > ShortDescriptionLength {
		return fmt.Errorf("Short description character length is greater than the max of %d", ShortDescriptionLength)
	}
	if profile.ContactInfo != nil {
		if len(profile.ContactInfo.Website) > URLMaxCharacters {
			return fmt.Errorf("Website character length is greater than the max of %d", URLMaxCharacters)
		}
		if len(profile.ContactInfo.Email) > SentenceMaxCharacters {
			return fmt.Errorf("Email character length is greater than the max of %d", SentenceMaxCharacters)
		}
		if len(profile.ContactInfo.PhoneNumber) > WordMaxCharacters {
			return fmt.Errorf("Phone number character length is greater than the max of %d", WordMaxCharacters)
		}
		if len(profile.ContactInfo.Social) > MaxListItems {
			return fmt.Errorf("Number of social accounts is greater than the max of %d", MaxListItems)
		}
		for _, s := range profile.ContactInfo.Social {
			if len(s.Username) > WordMaxCharacters {
				return fmt.Errorf("Social username character length is greater than the max of %d", WordMaxCharacters)
			}
			if len(s.Type) > WordMaxCharacters {
				return fmt.Errorf("Social account type character length is greater than the max of %d", WordMaxCharacters)
			}
			if len(s.Proof) > URLMaxCharacters {
				return fmt.Errorf("Social proof character length is greater than the max of %d", WordMaxCharacters)
			}
		}
	}
	if profile.ModeratorInfo != nil {
		if len(profile.ModeratorInfo.Description) > AboutMaxCharacteres {
			return fmt.Errorf("Moderator description character length is greater than the max of %d", AboutMaxCharacteres)
		}
		if len(profile.ModeratorInfo.TermsAndConditions) > PolicyMaxCharacters {
			return fmt.Errorf("Moderator terms and conditions character length is greater than the max of %d", PolicyMaxCharacters)
		}
		if len(profile.ModeratorInfo.Languages) > MaxListItems {
			return fmt.Errorf("Moderator number of languages greater than the max of %d", MaxListItems)
		}
		for _, l := range profile.ModeratorInfo.Languages {
			if len(l) > WordMaxCharacters {
				return fmt.Errorf("Moderator language character length is greater than the max of %d", WordMaxCharacters)
			}
		}
		if profile.ModeratorInfo.Fee != nil {
			if profile.ModeratorInfo.Fee.FixedFee != nil {
				if len(profile.ModeratorInfo.Fee.FixedFee.CurrencyCode) > WordMaxCharacters {
					return fmt.Errorf("Moderator fee currency code character length is greater than the max of %d", WordMaxCharacters)
				}
			}
		}
	}
	if profile.AvatarHashes != nil {
		_, err := mh.FromB58String(profile.AvatarHashes.Tiny)
		if err != nil {
			return errors.New("Tiny image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Small)
		if err != nil {
			return errors.New("Small image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Medium)
		if err != nil {
			return errors.New("Medium image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Large)
		if err != nil {
			return errors.New("Large image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.AvatarHashes.Original)
		if err != nil {
			return errors.New("Original image hashes must be multihashes")
		}
	}
	if profile.HeaderHashes != nil {
		_, err := mh.FromB58String(profile.HeaderHashes.Tiny)
		if err != nil {
			return errors.New("Tiny image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Small)
		if err != nil {
			return errors.New("Small image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Medium)
		if err != nil {
			return errors.New("Medium image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Large)
		if err != nil {
			return errors.New("Large image hashes must be multihashes")
		}
		_, err = mh.FromB58String(profile.HeaderHashes.Original)
		if err != nil {
			return errors.New("Original image hashes must be multihashes")
		}
	}
	if len(profile.BitcoinPubkey) > 66 {
		return fmt.Errorf("Bitcoin public key character length is greater than the max of %d", 66)
	}
	if profile.Stats != nil {
		if profile.Stats.AverageRating > 5 {
			return fmt.Errorf("Average rating cannot be greater than %d", 5)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package library

import (
	"container/heap"
	"errors"
	"github.com/nytlabs/gojee"                 // jee
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"   // util
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type Histogram struct {
	blocks.Block
	queryrule chan chan interface{}
	historule chan chan interface{}
	inrule    chan interface{}
	inpoll    chan interface{}
	in        chan interface{}
	out       chan interface{}
	quit      chan interface{}
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewHistogram() blocks.BlockInterface {
	return &Histogram{}
}

func buildHistogram(histogram map[string]*PriorityQueue) interface{} {
	var data interface{}
	var buckets []interface{}
	buckets = make([]interface{}, len(histogram))

	i := 0
	for k, pq := range histogram {
		var bucket interface{}
		bucket = map[string]interface{}{
			"Count": len(*pq),
			"Label": k,
		}
		buckets[i] = bucket
		i++
	}

	data = map[string]interface{}{
		"Histogram": buckets,
	}
	return data
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *Histogram) Setup() {
	b.Kind = "Histogram"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.historule = b.QueryRoute("histogram")
	b.inpoll = b.InRoute("poll")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Histogram) Run() {
	var tree *jee.TokenTree
	var path string
	waitTimer := time.NewTimer(100 * time.Millisecond)
	window := time.Duration(0)

	histogram := map[string]*PriorityQueue{}
	emptyByte := make([]byte, 0)
	for {
		select {
		case ruleI := <-b.inrule:
			// window
			windowString, err := util.ParseString(ruleI, "Window")
			if err != nil {
				b.Error(err)
			}
			window, err = time.ParseDuration(windowString)
			if err != nil {
				b.Error(err)
			}
			path, err = util.ParseString(ruleI, "Path")
			tree, err = util.BuildTokenTree(path)
			if err != nil {
				b.Error(err)
				break
			}

		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			if tree == nil {
				continue
			}
			v, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			valueString, ok := v.(string)
			if !ok {
				b.Error(errors.New("nil value against" + path + " - ignoring"))
				break
			}

			if pq, ok := histogram[valueString]; ok {
				queueMessage := &PQMessage{
					val: &emptyByte,
					t:   time.Now(),
				}
				heap.Push(pq, queueMessage)
			} else {
				pq := &PriorityQueue{}
				heap.Init(pq)
				histogram[valueString] = pq
				queueMessage := &PQMessage{
					val: &emptyByte,
					t:   time.Now(),
				}
				heap.Push(pq, queueMessage)
			}
		case <-waitTimer.C:

		case <-b.inpoll:
			// deal with a poll request
			data := buildHistogram(histogram)
			b.out <- data
		case respChan := <-b.queryrule:
			// deal with a query request
			out := map[string]interface{}{
				"Window": window.String(),
				"Path":   path,
			}
			respChan <- out
		case respChan := <-b.historule:
			data := buildHistogram(histogram)
			respChan <- data
		}
		for _, pq := range histogram {
			for {
				pqMsg, diff := pq.PeekAndShift(time.Now(), window)
				if pqMsg == nil {
					// either the queue is empty, or it's not time to emit
					if diff == 0 {
						// then the queue is empty. Pause for 5 seconds before checking again
						diff = time.Duration(500) * time.Millisecond
					}
					waitTimer.Reset(diff)
					break
				}
			}
		}
	}
}
<commit_msg>histogram counts are floats<commit_after>package library

import (
	"container/heap"
	"errors"
	"github.com/nytlabs/gojee"                 // jee
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"   // util
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type Histogram struct {
	blocks.Block
	queryrule chan chan interface{}
	historule chan chan interface{}
	inrule    chan interface{}
	inpoll    chan interface{}
	in        chan interface{}
	out       chan interface{}
	quit      chan interface{}
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewHistogram() blocks.BlockInterface {
	return &Histogram{}
}

func buildHistogram(histogram map[string]*PriorityQueue) interface{} {
	var data interface{}
	var buckets []interface{}
	buckets = make([]interface{}, len(histogram))

	i := 0
	for k, pq := range histogram {
		var bucket interface{}
		bucket = map[string]interface{}{
			"Count": float64(len(*pq)),
			"Label": k,
		}
		buckets[i] = bucket
		i++
	}

	data = map[string]interface{}{
		"Histogram": buckets,
	}
	return data
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *Histogram) Setup() {
	b.Kind = "Histogram"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.historule = b.QueryRoute("histogram")
	b.inpoll = b.InRoute("poll")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Histogram) Run() {
	var tree *jee.TokenTree
	var path string
	waitTimer := time.NewTimer(100 * time.Millisecond)
	window := time.Duration(0)

	histogram := map[string]*PriorityQueue{}
	emptyByte := make([]byte, 0)
	for {
		select {
		case ruleI := <-b.inrule:
			// window
			windowString, err := util.ParseString(ruleI, "Window")
			if err != nil {
				b.Error(err)
			}
			window, err = time.ParseDuration(windowString)
			if err != nil {
				b.Error(err)
			}
			path, err = util.ParseString(ruleI, "Path")
			tree, err = util.BuildTokenTree(path)
			if err != nil {
				b.Error(err)
				break
			}

		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			if tree == nil {
				continue
			}
			v, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			valueString, ok := v.(string)
			if !ok {
				b.Error(errors.New("nil value against" + path + " - ignoring"))
				break
			}

			if pq, ok := histogram[valueString]; ok {
				queueMessage := &PQMessage{
					val: &emptyByte,
					t:   time.Now(),
				}
				heap.Push(pq, queueMessage)
			} else {
				pq := &PriorityQueue{}
				heap.Init(pq)
				histogram[valueString] = pq
				queueMessage := &PQMessage{
					val: &emptyByte,
					t:   time.Now(),
				}
				heap.Push(pq, queueMessage)
			}
		case <-waitTimer.C:

		case <-b.inpoll:
			// deal with a poll request
			data := buildHistogram(histogram)
			b.out <- data
		case respChan := <-b.queryrule:
			// deal with a query request
			out := map[string]interface{}{
				"Window": window.String(),
				"Path":   path,
			}
			respChan <- out
		case respChan := <-b.historule:
			data := buildHistogram(histogram)
			respChan <- data
		}
		for _, pq := range histogram {
			for {
				pqMsg, diff := pq.PeekAndShift(time.Now(), window)
				if pqMsg == nil {
					// either the queue is empty, or it's not time to emit
					if diff == 0 {
						// then the queue is empty. Pause for 5 seconds before checking again
						diff = time.Duration(500) * time.Millisecond
					}
					waitTimer.Reset(diff)
					break
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package goose

import (
	"fmt"
	"io/ioutil"
	"strings"
	"testing"
)

func ReadRawHtml(a Article) string {
	path := fmt.Sprintf("sites/%s.html", a.Domain)
	file, err := ioutil.ReadFile(path)
	if err != nil {
		panic(fmt.Sprintf("cannot read %s", path))
	}

	return string(file)
}

func ValidateArticle(a Article) error {
	g := New()
	expected := g.ExtractFromRawHtml(a.FinalUrl, ReadRawHtml(a))

	if expected.Title != a.Title {
		return fmt.Errorf("article title does not match. Got %s", expected.Title)
	}

	if expected.MetaDescription != a.MetaDescription {
		return fmt.Errorf("article metaDescription does not match. Got %s", expected.MetaDescription)
	}

	if !strings.Contains(expected.CleanedText, a.CleanedText) {
		return fmt.Errorf("article cleanedText does not contains %s", a.CleanedText)
	}

	if expected.MetaKeywords != a.MetaKeywords {
		return fmt.Errorf("article keywords does not match. Got %s", expected.MetaKeywords)
	}

	if expected.FinalUrl != a.FinalUrl {
		return fmt.Errorf("article finalUrl does not match. Got %s", expected.FinalUrl)
	}

	if expected.TopImage != a.TopImage {
		return fmt.Errorf("article topImage does not match. Got %s", expected.TopImage)
	}

	return nil
}

func Test_GloboEsporteParse(t *testing.T) {
	article := Article{
		Domain:          "globoesporte.globo.com",
		Title:           "Rodrigo Caio treina até nas férias e tenta acelerar retorno aos gramados",
		MetaDescription: "Rodrigo Caio treina na esteira durante as férias em Dracena-SP (Foto: Divulgação)Rodrigo Caio quer ganhar tempo na recuperação da lesão que sofreu no joelho esquerdo. Apesar de ter sido liberado pelo departamento médico do São Paulo para as férias, o ...",
		CleanedText:     "Comissão técnica planeja volta dele para o fim de fevereiro ou início de março",
		MetaKeywords:    "notícias, notícia, presidente prudente região",
		FinalUrl:        "http://globoesporte.globo.com/sp/presidente-prudente-regiao/noticia/2014/12/rodrigo-caio-treina-ate-nas-ferias-e-tenta-acelerar-retorno-aos-gramados.html",
		TopImage:        "http://s.glbimg.com/es/ge/f/original/2014/12/26/10863872_894379987249341_2406060334390226774_o.jpg",
	}

	err := ValidateArticle(article)
	if err != nil {
		t.Error(err)
	}
}

func Test_EditionCnnParse(t *testing.T) {
	article := Article{
		Domain:          "edition.cnn.com",
		Title:           "What if you could make anything you wanted?",
		MetaDescription: "Massimo Banzi's pocket-sized open-source circuit board has become a key building block in the creation of a huge variety of innovative devices.",
		CleanedText:     "In the 20th century, getting your child a toy car meant a trip to a shopping mall.",
		MetaKeywords:    "",
		FinalUrl:        "http://edition.cnn.com/2012/07/08/opinion/banzi-ted-open-source/index.html",
		TopImage:        "http://i2.cdn.turner.com/cnn/dam/assets/120706022111-ted-cnn-ideas-massimo-banzi-00003302-story-top.jpg",
	}

	err := ValidateArticle(article)
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>fix/enhance test: quoted string, compare canonical link, not final link<commit_after>package goose

import (
	"fmt"
	"io/ioutil"
	"strings"
	"testing"
)

func ReadRawHtml(a Article) string {
	path := fmt.Sprintf("sites/%s.html", a.Domain)
	file, err := ioutil.ReadFile(path)
	if err != nil {
		panic(fmt.Sprintf("cannot read %q", path))
	}

	return string(file)
}

func ValidateArticle(a Article) error {
	g := New()
	expected := g.ExtractFromRawHtml(a.FinalUrl, ReadRawHtml(a))

	if expected.Title != a.Title {
		return fmt.Errorf("article title does not match. Got %q", expected.Title)
	}

	if expected.MetaDescription != a.MetaDescription {
		return fmt.Errorf("article metaDescription does not match. Got %q", expected.MetaDescription)
	}

	if !strings.Contains(expected.CleanedText, a.CleanedText) {
		return fmt.Errorf("article cleanedText does not contains %q", a.CleanedText)
	}

	if expected.MetaKeywords != a.MetaKeywords {
		return fmt.Errorf("article keywords does not match. Got %q", expected.MetaKeywords)
	}
	if expected.CanonicalLink != a.CanonicalLink {
		return fmt.Errorf("article CanonicalLink does not match. Got %q", expected.CanonicalLink)
	}

	if expected.TopImage != a.TopImage {
		return fmt.Errorf("article topImage does not match. Got %q", expected.TopImage)
	}

	return nil
}

func Test_GloboEsporteParse(t *testing.T) {
	article := Article{
		Domain:          "globoesporte.globo.com",
		Title:           "Rodrigo Caio treina até nas férias e tenta acelerar retorno aos gramados",
		MetaDescription: "Rodrigo Caio treina na esteira durante as férias em Dracena-SP (Foto: Divulgação)Rodrigo Caio quer ganhar tempo na recuperação da lesão que sofreu no joelho esquerdo. Apesar de ter sido liberado pelo departamento médico do São Paulo para as férias, o ...",
		CleanedText:     "Comissão técnica planeja volta dele para o fim de fevereiro ou início de março",
		MetaKeywords:    "notícias, notícia, presidente prudente região",
		CanonicalLink:   "http://globoesporte.globo.com/sp/presidente-prudente-regiao/noticia/2014/12/rodrigo-caio-treina-ate-nas-ferias-e-tenta-acelerar-retorno-aos-gramados.html",
		TopImage:        "http://s.glbimg.com/es/ge/f/original/2014/12/26/10863872_894379987249341_2406060334390226774_o.jpg",
	}

	err := ValidateArticle(article)
	if err != nil {
		t.Error(err)
	}
}

func Test_EditionCnnParse(t *testing.T) {
	article := Article{
		Domain:          "edition.cnn.com",
		Title:           "What if you could make anything you wanted?",
		MetaDescription: "Massimo Banzi's pocket-sized open-source circuit board has become a key building block in the creation of a huge variety of innovative devices.",
		CleanedText:     "In the 20th century, getting your child a toy car meant a trip to a shopping mall.",
		MetaKeywords:    "",
		CanonicalLink:   "http://www.cnn.com/2012/07/08/opinion/banzi-ted-open-source/index.html",
		TopImage:        "http://i2.cdn.turner.com/cnn/dam/assets/120706022111-ted-cnn-ideas-massimo-banzi-00003302-story-top.jpg",
	}

	err := ValidateArticle(article)
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package rainforest

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
)

// RFMLReader reads form RFML formatted file.
// It exports some settings that can be set before parsing.
type RFMLReader struct {
	r *bufio.Reader
	// Version sets the RFML spec version, it's set by NewRFMLReader to the newest one.
	Version int
	// Sets the default value of redirect, that's used when it's not specified in RFML
	RedirectDefault bool
}

// parseError is a custom error implementing error interface for reporting RFML parsing errors.
type parseError struct {
	location string
	reason   string
}

func (e *parseError) Error() string {
	if _, err := strconv.Atoi(e.location); err == nil {
		// location is a line number
		return fmt.Sprintf("RFML parsing error in line %v: %v", e.location, e.reason)
	} else {
		// location is a field
		return fmt.Sprintf("RFML parsing error for test field \"%v\": %v", e.location, e.reason)
	}
}

// NewRFMLReader returns RFML parser based on passed io.Reader - typically a RFML file.
func NewRFMLReader(r io.Reader) *RFMLReader {
	return &RFMLReader{
		r:               bufio.NewReader(r),
		Version:         1,
		RedirectDefault: true,
	}
}

// ReadAll parses whole RFML file using RFML version specified by Version parameter of reader
// and returns resulting RFTest
func (r *RFMLReader) ReadAll() (*RFTest, error) {
	parsedRFTest := &RFTest{}
	// Set up a new scanner to read in data line by line
	scanner := bufio.NewScanner(r.r)
	lineNum := 0
	// Temp variables where we put stuff while parsing
	currStep := make([]string, 0, 2)
	currStepRedirect := r.RedirectDefault
	for scanner.Scan() {
		lineNum++
		lineNumStr := strconv.Itoa(lineNum)

		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "#!") {
			if parsedRFTest.RFMLID != "" {
				return parsedRFTest, &parseError{lineNumStr, "Only one RFML ID may be specified"}
			}
			// Trim shebang and then take only first part of id before any spaces
			rfmlIDLine := strings.TrimSpace(line[2:])
			parsedRFTest.RFMLID = strings.Split(rfmlIDLine, " ")[0]
		} else if strings.HasPrefix(line, "#") {
			// Handle hashed lines
			content := line[1:]
			if strings.Contains(content, ":") {
				// Handle the key value pair
				split := strings.SplitN(content, ":", 2)
				key := strings.TrimSpace(split[0])
				value := strings.TrimSpace(split[1])
				switch key {
				case "title":
					parsedRFTest.Title = value
				case "start_uri":
					parsedRFTest.StartURI = value
				case "site_id":
					siteID, err := strconv.Atoi(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNumStr, "Site ID must be a valid integer"}
					}
					parsedRFTest.SiteID = siteID
				case "tags":
					splitTags := strings.Split(value, ",")
					strippedTags := make([]string, len(splitTags))
					for i, tag := range splitTags {
						strippedTags[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Tags = strippedTags
				case "browsers":
					splitBrowsers := strings.Split(value, ",")
					strippedBrowsers := make([]string, len(splitBrowsers))
					for i, tag := range splitBrowsers {
						strippedBrowsers[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Browsers = strippedBrowsers
				case "redirect":
					redirect, err := strconv.ParseBool(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNumStr, "Redirect value must be a valid boolean"}
					}
					currStepRedirect = redirect
				default:
					// If it doesn't match known key add it to description
					parsedRFTest.Description += strings.TrimSpace(content) + "\n"
				}
			} else {
				// If it'a a hashed line without key-value pair add it as a comment
				parsedRFTest.Description += strings.TrimSpace(content) + "\n"
			}
		} else {
			// Handle non prefixed lines
			// Here what we do depends on the fact if we have some step data collected already
			switch len(currStep) {
			case 0:
				if strings.HasPrefix(line, "-") {
					embeddedID := strings.TrimSpace(line[strings.Index(line, "-")+1:])
					embeddedStep := RFEmbeddedTest{embeddedID, currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, embeddedStep)
					// Reset currStepRedirect
					currStepRedirect = r.RedirectDefault
				} else if line != "" {
					currStep = append(currStep, line)
				}
			case 1:
				if strings.Contains(line, "?") {
					currStep = append(currStep, line)
				} else {
					return parsedRFTest, &parseError{lineNumStr, "Each step must contain a question, with a `?`"}
				}
			case 2:
				if line == "" {
					parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
					// Reset temp vars to defaults
					currStep = make([]string, 0, 2)
					currStepRedirect = r.RedirectDefault
				} else {
					return parsedRFTest, &parseError{lineNumStr, "Steps must be separated with empty lines"}
				}
			}
		}
	}

	// Check if parsing stopped before adding a step
	if len(currStep) == 1 {
		return parsedRFTest, &parseError{strconv.Itoa(lineNum), "Must have a corresponding question with your action."}
	}

	if len(currStep) == 2 {
		parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
		parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
	}

	if parsedRFTest.RFMLID == "" {
		return parsedRFTest, &parseError{"#!", "RFML ID is required for .rfml files. Specify it using #! followed by a unique RFML ID"}
	}

	if parsedRFTest.Title == "" {
		return parsedRFTest, &parseError{"# title", "Title is required for .rfml files. Specify it using \"# title\" followed by your test's title."}
	}

	return parsedRFTest, nil
}

// RFMLWriter writes a RFML formatted test to a given file.
type RFMLWriter struct {
	w *bufio.Writer
	// Version sets the RFML spec version
	Version int
}

// NewRFMLWriter returns RFML writer based on passed io.Writer - typically a RFML file.
func NewRFMLWriter(w io.Writer) *RFMLWriter {
	return &RFMLWriter{
		w:       bufio.NewWriter(w),
		Version: 1,
	}
}

// WriteRFMLTest writes a given RFTest to its writer in the given RFML version.
func (r *RFMLWriter) WriteRFMLTest(test *RFTest) error {
	writer := r.w
	headerTemplate := `#! %v
# title: %v
# start_uri: %v
`

	header := fmt.Sprintf(headerTemplate, test.RFMLID, test.Title, test.StartURI)
	_, err := writer.WriteString(header)

	if err != nil {
		return err
	}

	if test.SiteID != 0 {
		_, err = writer.WriteString("# site_id: " + strconv.Itoa(test.SiteID) + "\n")

		if err != nil {
			return err
		}
	}

	if len(test.Tags) > 0 {
		tags := strings.Join(test.Tags, ", ")
		tagsHeader := fmt.Sprintf("# tags: %v\n", tags)

		_, err = writer.WriteString(tagsHeader)

		if err != nil {
			return err
		}
	}

	if len(test.Browsers) > 0 {
		browsers := strings.Join(test.Browsers, ", ")
		browsersHeader := fmt.Sprintf("# browsers: %v\n", browsers)

		_, err = writer.WriteString(browsersHeader)

		if err != nil {
			return err
		}
	}

	if test.Description != "" {
		_, err = writer.WriteString("# " + strings.Replace(test.Description, "\n", "\n# ", -1) + "\n")

		if err != nil {
			return err
		}
	}

	firstStepProcessed := false
	processStep := func(idx int, step RFTestStep) string {
		stepText := ""
		if idx > 0 && firstStepProcessed == false {
			stepText = stepText + fmt.Sprintf("# redirect: %v\n", step.Redirect)
		}
		action := strings.Replace(step.Action, "\n", " ", -1)
		response := strings.Replace(step.Response, "\n", " ", -1)
		firstStepProcessed = true

		return stepText + action + "\n" + response
	}

	for idx, step := range test.Steps {
		var stepText string
		switch step.(type) {
		case RFTestStep:
			stepText = processStep(idx, step.(RFTestStep))
		case RFEmbeddedTest:
			embeddedTest := step.(RFEmbeddedTest)
			if idx > 0 {
				stepText = "# redirect: " + strconv.FormatBool(embeddedTest.Redirect) + "\n"
			}
			stepText = stepText + "- " + embeddedTest.RFMLID
		}

		_, err = writer.WriteString("\n" + stepText + "\n")

		if err != nil {
			return err
		}
	}

	// Writes buffered data to the underlying io.Writer
	err = writer.Flush()

	if err != nil {
		return err
	}

	return nil
}

// ParseEmbeddedFiles replaces file step variable paths with values expected
// by Rainforest. eg: {{ file.screenshot(my_screenshot.gif) }} would be translated
// to the format {{ file.screenshot(FILE_ID, FILE_SIGNATURE) }}.
func (c *Client) ParseEmbeddedFiles(test *RFTest) error {
	if test.TestID == 0 {
		return fmt.Errorf("Cannot parse embedded files without a test ID.")
	}

	uploadedFiles, err := c.getUploadedFiles(test.TestID)
	if err != nil {
		return err
	}

	digestToFileMap := map[string]uploadedFile{}
	for _, f := range uploadedFiles {
		digestToFileMap[f.Digest] = f
	}

	replaceEmbeddedFilePaths := func(text string, embeddedFiles []embeddedFile) (string, error) {
		out := text
		for _, embed := range embeddedFiles {
			filePath := embed.path
			if strings.HasPrefix(filePath, "~/") {
				var usr *user.User
				usr, err = user.Current()
				if err != nil {
					return "", err
				}
				filePath = filepath.Join(usr.HomeDir, filePath[2:])
			} else if test.RFMLPath == "" {
				return "", fmt.Errorf("Cannot parse relative file path %v for RFML test %v. RFMLPath field cannot be blank.", filePath, test.RFMLID)
			} else {
				rfmlDirectory := filepath.Dir(test.RFMLPath)
				filePath = filepath.Join(rfmlDirectory, filePath)
			}

			filePath, err = filepath.Abs(filePath)
			if err != nil {
				return "", err
			}

			var file *os.File
			file, err = os.Open(filePath)
			if err != nil {
				return "", err
			}

			var data []byte
			data, err = ioutil.ReadAll(file)
			defer file.Close()
			if err != nil {
				return "", err
			}

			checksum := md5.Sum(data)
			fileDigest := hex.EncodeToString(checksum[:])
			uploadedFileInfo, ok := digestToFileMap[fileDigest]
			// TODO: Check mime type as well
			if !ok {
				// File has not been uploaded before
				// Upload to RF
				var awsInfo *awsFileInfo
				awsInfo, err = c.createTestFile(test.TestID, file, data)
				if err != nil {
					return "", err
				}
				// Upload to AWS
				err = c.uploadEmbeddedFile(filepath.Base(filePath), data, awsInfo)
				if err != nil {
					return "", err
				}
				uploadedFileInfo = uploadedFile{
					ID:        awsInfo.FileID,
					Signature: awsInfo.FileSignature,
					Digest:    fileDigest,
				}
				// Add to the mappings for future reference
				digestToFileMap[fileDigest] = uploadedFileInfo
			}

			sig := uploadedFileInfo.Signature[0:6]
			var replacement string
			if embed.stepVar == "screenshot" {
				replacement = fmt.Sprintf("{{ file.screenshot(%v, %v) }}", uploadedFileInfo.ID, sig)
			} else if embed.stepVar == "download" {
				replacement = fmt.Sprintf("{{ file.download(%v, %v, %v) }}", uploadedFileInfo.ID, sig, filepath.Base(filePath))
			}

			out = strings.Replace(out, embed.text, replacement, 1)
		}

		return out, nil
	}

	for idx, step := range test.Steps {
		s, ok := step.(RFTestStep)
		if ok && s.hasUploadableFiles() {
			if embeddedFiles := s.embeddedFilesInAction(); len(embeddedFiles) > 0 {
				s.Action, err = replaceEmbeddedFilePaths(s.Action, embeddedFiles)
				if err != nil {
					return err
				}
			}

			if embeddedFiles := s.embeddedFilesInResponse(); len(embeddedFiles) > 0 {
				s.Response, err = replaceEmbeddedFilePaths(s.Response, embeddedFiles)
				if err != nil {
					return err
				}
			}
			test.Steps[idx] = s
		}
	}

	return nil
}
<commit_msg>fix typo<commit_after>package rainforest

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
)

// RFMLReader reads form RFML formatted file.
// It exports some settings that can be set before parsing.
type RFMLReader struct {
	r *bufio.Reader
	// Version sets the RFML spec version, it's set by NewRFMLReader to the newest one.
	Version int
	// Sets the default value of redirect, that's used when it's not specified in RFML
	RedirectDefault bool
}

// parseError is a custom error implementing error interface for reporting RFML parsing errors.
type parseError struct {
	location string
	reason   string
}

func (e *parseError) Error() string {
	if _, err := strconv.Atoi(e.location); err == nil {
		// location is a line number
		return fmt.Sprintf("RFML parsing error in line %v: %v", e.location, e.reason)
	} else {
		// location is a field
		return fmt.Sprintf("RFML parsing error for test field \"%v\": %v", e.location, e.reason)
	}
}

// NewRFMLReader returns RFML parser based on passed io.Reader - typically a RFML file.
func NewRFMLReader(r io.Reader) *RFMLReader {
	return &RFMLReader{
		r:               bufio.NewReader(r),
		Version:         1,
		RedirectDefault: true,
	}
}

// ReadAll parses whole RFML file using RFML version specified by Version parameter of reader
// and returns resulting RFTest
func (r *RFMLReader) ReadAll() (*RFTest, error) {
	parsedRFTest := &RFTest{}
	// Set up a new scanner to read in data line by line
	scanner := bufio.NewScanner(r.r)
	lineNum := 0
	// Temp variables where we put stuff while parsing
	currStep := make([]string, 0, 2)
	currStepRedirect := r.RedirectDefault
	for scanner.Scan() {
		lineNum++
		lineNumStr := strconv.Itoa(lineNum)

		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "#!") {
			if parsedRFTest.RFMLID != "" {
				return parsedRFTest, &parseError{lineNumStr, "Only one RFML ID may be specified"}
			}
			// Trim shebang and then take only first part of id before any spaces
			rfmlIDLine := strings.TrimSpace(line[2:])
			parsedRFTest.RFMLID = strings.Split(rfmlIDLine, " ")[0]
		} else if strings.HasPrefix(line, "#") {
			// Handle hashed lines
			content := line[1:]
			if strings.Contains(content, ":") {
				// Handle the key value pair
				split := strings.SplitN(content, ":", 2)
				key := strings.TrimSpace(split[0])
				value := strings.TrimSpace(split[1])
				switch key {
				case "title":
					parsedRFTest.Title = value
				case "start_uri":
					parsedRFTest.StartURI = value
				case "site_id":
					siteID, err := strconv.Atoi(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNumStr, "Site ID must be a valid integer"}
					}
					parsedRFTest.SiteID = siteID
				case "tags":
					splitTags := strings.Split(value, ",")
					strippedTags := make([]string, len(splitTags))
					for i, tag := range splitTags {
						strippedTags[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Tags = strippedTags
				case "browsers":
					splitBrowsers := strings.Split(value, ",")
					strippedBrowsers := make([]string, len(splitBrowsers))
					for i, tag := range splitBrowsers {
						strippedBrowsers[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Browsers = strippedBrowsers
				case "redirect":
					redirect, err := strconv.ParseBool(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNumStr, "Redirect value must be a valid boolean"}
					}
					currStepRedirect = redirect
				default:
					// If it doesn't match known key add it to description
					parsedRFTest.Description += strings.TrimSpace(content) + "\n"
				}
			} else {
				// If it'a a hashed line without key-value pair add it as a comment
				parsedRFTest.Description += strings.TrimSpace(content) + "\n"
			}
		} else {
			// Handle non prefixed lines
			// Here what we do depends on the fact if we have some step data collected already
			switch len(currStep) {
			case 0:
				if strings.HasPrefix(line, "-") {
					embeddedID := strings.TrimSpace(line[strings.Index(line, "-")+1:])
					embeddedStep := RFEmbeddedTest{embeddedID, currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, embeddedStep)
					// Reset currStepRedirect
					currStepRedirect = r.RedirectDefault
				} else if line != "" {
					currStep = append(currStep, line)
				}
			case 1:
				if strings.Contains(line, "?") {
					currStep = append(currStep, line)
				} else {
					return parsedRFTest, &parseError{lineNumStr, "Each step must contain a question, with a `?`"}
				}
			case 2:
				if line == "" {
					parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
					// Reset temp vars to defaults
					currStep = make([]string, 0, 2)
					currStepRedirect = r.RedirectDefault
				} else {
					return parsedRFTest, &parseError{lineNumStr, "Steps must be separated with empty lines"}
				}
			}
		}
	}

	// Check if parsing stopped before adding a step
	if len(currStep) == 1 {
		return parsedRFTest, &parseError{strconv.Itoa(lineNum), "Must have a corresponding question with your action."}
	}

	if len(currStep) == 2 {
		parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
		parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
	}

	if parsedRFTest.RFMLID == "" {
		return parsedRFTest, &parseError{"#!", "RFML ID is required for .rfml files. Specify it using #! followed by a unique RFML ID"}
	}

	if parsedRFTest.Title == "" {
		return parsedRFTest, &parseError{"# title", "Title is required for .rfml files. Specify it using \"# title: \" followed by your test's title."}
	}

	return parsedRFTest, nil
}

// RFMLWriter writes a RFML formatted test to a given file.
type RFMLWriter struct {
	w *bufio.Writer
	// Version sets the RFML spec version
	Version int
}

// NewRFMLWriter returns RFML writer based on passed io.Writer - typically a RFML file.
func NewRFMLWriter(w io.Writer) *RFMLWriter {
	return &RFMLWriter{
		w:       bufio.NewWriter(w),
		Version: 1,
	}
}

// WriteRFMLTest writes a given RFTest to its writer in the given RFML version.
func (r *RFMLWriter) WriteRFMLTest(test *RFTest) error {
	writer := r.w
	headerTemplate := `#! %v
# title: %v
# start_uri: %v
`

	header := fmt.Sprintf(headerTemplate, test.RFMLID, test.Title, test.StartURI)
	_, err := writer.WriteString(header)

	if err != nil {
		return err
	}

	if test.SiteID != 0 {
		_, err = writer.WriteString("# site_id: " + strconv.Itoa(test.SiteID) + "\n")

		if err != nil {
			return err
		}
	}

	if len(test.Tags) > 0 {
		tags := strings.Join(test.Tags, ", ")
		tagsHeader := fmt.Sprintf("# tags: %v\n", tags)

		_, err = writer.WriteString(tagsHeader)

		if err != nil {
			return err
		}
	}

	if len(test.Browsers) > 0 {
		browsers := strings.Join(test.Browsers, ", ")
		browsersHeader := fmt.Sprintf("# browsers: %v\n", browsers)

		_, err = writer.WriteString(browsersHeader)

		if err != nil {
			return err
		}
	}

	if test.Description != "" {
		_, err = writer.WriteString("# " + strings.Replace(test.Description, "\n", "\n# ", -1) + "\n")

		if err != nil {
			return err
		}
	}

	firstStepProcessed := false
	processStep := func(idx int, step RFTestStep) string {
		stepText := ""
		if idx > 0 && firstStepProcessed == false {
			stepText = stepText + fmt.Sprintf("# redirect: %v\n", step.Redirect)
		}
		action := strings.Replace(step.Action, "\n", " ", -1)
		response := strings.Replace(step.Response, "\n", " ", -1)
		firstStepProcessed = true

		return stepText + action + "\n" + response
	}

	for idx, step := range test.Steps {
		var stepText string
		switch step.(type) {
		case RFTestStep:
			stepText = processStep(idx, step.(RFTestStep))
		case RFEmbeddedTest:
			embeddedTest := step.(RFEmbeddedTest)
			if idx > 0 {
				stepText = "# redirect: " + strconv.FormatBool(embeddedTest.Redirect) + "\n"
			}
			stepText = stepText + "- " + embeddedTest.RFMLID
		}

		_, err = writer.WriteString("\n" + stepText + "\n")

		if err != nil {
			return err
		}
	}

	// Writes buffered data to the underlying io.Writer
	err = writer.Flush()

	if err != nil {
		return err
	}

	return nil
}

// ParseEmbeddedFiles replaces file step variable paths with values expected
// by Rainforest. eg: {{ file.screenshot(my_screenshot.gif) }} would be translated
// to the format {{ file.screenshot(FILE_ID, FILE_SIGNATURE) }}.
func (c *Client) ParseEmbeddedFiles(test *RFTest) error {
	if test.TestID == 0 {
		return fmt.Errorf("Cannot parse embedded files without a test ID.")
	}

	uploadedFiles, err := c.getUploadedFiles(test.TestID)
	if err != nil {
		return err
	}

	digestToFileMap := map[string]uploadedFile{}
	for _, f := range uploadedFiles {
		digestToFileMap[f.Digest] = f
	}

	replaceEmbeddedFilePaths := func(text string, embeddedFiles []embeddedFile) (string, error) {
		out := text
		for _, embed := range embeddedFiles {
			filePath := embed.path
			if strings.HasPrefix(filePath, "~/") {
				var usr *user.User
				usr, err = user.Current()
				if err != nil {
					return "", err
				}
				filePath = filepath.Join(usr.HomeDir, filePath[2:])
			} else if test.RFMLPath == "" {
				return "", fmt.Errorf("Cannot parse relative file path %v for RFML test %v. RFMLPath field cannot be blank.", filePath, test.RFMLID)
			} else {
				rfmlDirectory := filepath.Dir(test.RFMLPath)
				filePath = filepath.Join(rfmlDirectory, filePath)
			}

			filePath, err = filepath.Abs(filePath)
			if err != nil {
				return "", err
			}

			var file *os.File
			file, err = os.Open(filePath)
			if err != nil {
				return "", err
			}

			var data []byte
			data, err = ioutil.ReadAll(file)
			defer file.Close()
			if err != nil {
				return "", err
			}

			checksum := md5.Sum(data)
			fileDigest := hex.EncodeToString(checksum[:])
			uploadedFileInfo, ok := digestToFileMap[fileDigest]
			// TODO: Check mime type as well
			if !ok {
				// File has not been uploaded before
				// Upload to RF
				var awsInfo *awsFileInfo
				awsInfo, err = c.createTestFile(test.TestID, file, data)
				if err != nil {
					return "", err
				}
				// Upload to AWS
				err = c.uploadEmbeddedFile(filepath.Base(filePath), data, awsInfo)
				if err != nil {
					return "", err
				}
				uploadedFileInfo = uploadedFile{
					ID:        awsInfo.FileID,
					Signature: awsInfo.FileSignature,
					Digest:    fileDigest,
				}
				// Add to the mappings for future reference
				digestToFileMap[fileDigest] = uploadedFileInfo
			}

			sig := uploadedFileInfo.Signature[0:6]
			var replacement string
			if embed.stepVar == "screenshot" {
				replacement = fmt.Sprintf("{{ file.screenshot(%v, %v) }}", uploadedFileInfo.ID, sig)
			} else if embed.stepVar == "download" {
				replacement = fmt.Sprintf("{{ file.download(%v, %v, %v) }}", uploadedFileInfo.ID, sig, filepath.Base(filePath))
			}

			out = strings.Replace(out, embed.text, replacement, 1)
		}

		return out, nil
	}

	for idx, step := range test.Steps {
		s, ok := step.(RFTestStep)
		if ok && s.hasUploadableFiles() {
			if embeddedFiles := s.embeddedFilesInAction(); len(embeddedFiles) > 0 {
				s.Action, err = replaceEmbeddedFilePaths(s.Action, embeddedFiles)
				if err != nil {
					return err
				}
			}

			if embeddedFiles := s.embeddedFilesInResponse(); len(embeddedFiles) > 0 {
				s.Response, err = replaceEmbeddedFilePaths(s.Response, embeddedFiles)
				if err != nil {
					return err
				}
			}
			test.Steps[idx] = s
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package rainforest

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
)

// RFMLReader reads form RFML formatted file.
// It exports some settings that can be set before parsing.
type RFMLReader struct {
	r *bufio.Reader
	// Version sets the RFML spec version, it's set by NewRFMLReader to the newest one.
	Version int
	// Sets the default value of redirect, that's used when it's not specified in RFML
	RedirectDefault bool
}

// parseError is a custom error implementing error interface for reporting RFML parsing errors.
type parseError struct {
	line   int
	reason string
}

func (e *parseError) Error() string {
	return fmt.Sprintf("RFML parsing error in line %v: %v", e.line, e.reason)
}

// NewRFMLReader returns RFML parser based on passed io.Reader - typically a RFML file.
func NewRFMLReader(r io.Reader) *RFMLReader {
	return &RFMLReader{
		r:               bufio.NewReader(r),
		Version:         1,
		RedirectDefault: true,
	}
}

// ReadAll parses whole RFML file using RFML version specified by Version parameter of reader
// and returns resulting RFTest
func (r *RFMLReader) ReadAll() (*RFTest, error) {
	parsedRFTest := &RFTest{}
	// Set up a new scanner to read in data line by line
	scanner := bufio.NewScanner(r.r)
	lineNum := 0
	// Temp variables where we put stuff while parsing
	currStep := make([]string, 0, 2)
	currStepRedirect := r.RedirectDefault
	for scanner.Scan() {
		lineNum++
		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "#!") {
			if parsedRFTest.RFMLID != "" {
				return parsedRFTest, &parseError{lineNum, "Only one RFML ID may be specified"}
			}
			// Trim shebang and then take only first part of id before any spaces
			rfmlIDLine := strings.TrimSpace(line[2:])
			parsedRFTest.RFMLID = strings.Split(rfmlIDLine, " ")[0]
		} else if strings.HasPrefix(line, "#") {
			// Handle hashed lines
			content := line[1:]
			if strings.Contains(content, ":") {
				// Handle the key value pair
				split := strings.SplitN(content, ":", 2)
				key := strings.TrimSpace(split[0])
				value := strings.TrimSpace(split[1])
				switch key {
				case "title":
					parsedRFTest.Title = value
				case "start_uri":
					parsedRFTest.StartURI = value
				case "site_id":
					siteID, err := strconv.Atoi(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNum, "Site ID must be a valid integer"}
					}
					parsedRFTest.SiteID = siteID
				case "tags":
					splitTags := strings.Split(value, ",")
					strippedTags := make([]string, len(splitTags))
					for i, tag := range splitTags {
						strippedTags[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Tags = strippedTags
				case "browsers":
					splitBrowsers := strings.Split(value, ",")
					strippedBrowsers := make([]string, len(splitBrowsers))
					for i, tag := range splitBrowsers {
						strippedBrowsers[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Browsers = strippedBrowsers
				case "redirect":
					redirect, err := strconv.ParseBool(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNum, "Redirect value must be a valid boolean"}
					}
					currStepRedirect = redirect
				default:
					// If it doesn't match known key add it to description
					parsedRFTest.Description += strings.TrimSpace(content) + "\n"
				}
			} else {
				// If it'a a hashed line without key-value pair add it as a comment
				parsedRFTest.Description += strings.TrimSpace(content) + "\n"
			}
		} else {
			// Handle non prefixed lines
			// Here what we do depends on the fact if we have some step data collected already
			switch len(currStep) {
			case 0:
				if strings.HasPrefix(line, "-") {
					embeddedID := strings.TrimSpace(line[strings.Index(line, "-")+1:])
					embeddedStep := RFEmbeddedTest{embeddedID, currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, embeddedStep)
					// Reset currStepRedirect
					currStepRedirect = r.RedirectDefault
				} else if line != "" {
					currStep = append(currStep, line)
				}
			case 1:
				if strings.Contains(line, "?") {
					currStep = append(currStep, line)
				} else {
					return parsedRFTest, &parseError{lineNum, "Each step must contain a question, with a `?`"}
				}
			case 2:
				if line == "" {
					parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
					// Reset temp vars to defaults
					currStep = make([]string, 0, 2)
					currStepRedirect = r.RedirectDefault
				} else {
					return parsedRFTest, &parseError{lineNum, "Steps must be separated with empty lines"}
				}
			}
		}
	}

	// Check if parsing stopped before adding a step
	if len(currStep) == 1 {
		return parsedRFTest, &parseError{lineNum, "Must have a corresponding question with your action."}
	}

	if len(currStep) == 2 {
		parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
		parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
	}

	if parsedRFTest.RFMLID == "" {
		return parsedRFTest, &parseError{1, "RFML ID is required for .rfml files, specify it using #!"}
	}
	return parsedRFTest, nil
}

// RFMLWriter writes a RFML formatted test to a given file.
type RFMLWriter struct {
	w *bufio.Writer
	// Version sets the RFML spec version
	Version int
}

// NewRFMLWriter returns RFML writer based on passed io.Writer - typically a RFML file.
func NewRFMLWriter(w io.Writer) *RFMLWriter {
	return &RFMLWriter{
		w:       bufio.NewWriter(w),
		Version: 1,
	}
}

// WriteRFMLTest writes a given RFTest to its writer in the given RFML version.
func (r *RFMLWriter) WriteRFMLTest(test *RFTest) error {
	writer := r.w
	headerTemplate := `#! %v
# title: %v
# start_uri: %v
`

	header := fmt.Sprintf(headerTemplate, test.RFMLID, test.Title, test.StartURI)
	_, err := writer.WriteString(header)

	if err != nil {
		return err
	}

	if test.SiteID != 0 {
		_, err = writer.WriteString("# site_id: " + strconv.Itoa(test.SiteID) + "\n")

		if err != nil {
			return err
		}
	}

	if len(test.Tags) > 0 {
		tags := strings.Join(test.Tags, ", ")
		tagsHeader := fmt.Sprintf("# tags: %v\n", tags)

		_, err = writer.WriteString(tagsHeader)

		if err != nil {
			return err
		}
	}

	if len(test.Browsers) > 0 {
		browsers := strings.Join(test.Browsers, ", ")
		browsersHeader := fmt.Sprintf("# browsers: %v\n", browsers)

		_, err = writer.WriteString(browsersHeader)

		if err != nil {
			return err
		}
	}

	if test.Description != "" {
		_, err = writer.WriteString("# " + strings.Replace(test.Description, "\n", "\n# ", -1) + "\n")

		if err != nil {
			return err
		}
	}

	firstStepProcessed := false
	processStep := func(idx int, step RFTestStep) string {
		stepText := ""
		if idx > 0 && firstStepProcessed == false {
			stepText = stepText + fmt.Sprintf("# redirect: %v\n", step.Redirect)
		}
		action := strings.Replace(step.Action, "\n", " ", -1)
		response := strings.Replace(step.Response, "\n", " ", -1)
		firstStepProcessed = true

		return stepText + action + "\n" + response
	}

	for idx, step := range test.Steps {
		var stepText string
		switch step.(type) {
		case RFTestStep:
			stepText = processStep(idx, step.(RFTestStep))
		case RFEmbeddedTest:
			embeddedTest := step.(RFEmbeddedTest)
			if idx > 0 {
				stepText = "# redirect: " + strconv.FormatBool(embeddedTest.Redirect) + "\n"
			}
			stepText = stepText + "- " + embeddedTest.RFMLID
		}

		_, err = writer.WriteString("\n" + stepText + "\n")

		if err != nil {
			return err
		}
	}

	// Writes buffered data to the underlying io.Writer
	err = writer.Flush()

	if err != nil {
		return err
	}

	return nil
}

// ParseEmbeddedFiles replaces file step variable paths with values expected
// by Rainforest. eg: {{ file.screenshot(my_screenshot.gif) }} would be translated
// to the format {{ file.screenshot(FILE_ID, FILE_SIGNATURE) }}.
func (c *Client) ParseEmbeddedFiles(test *RFTest) error {
	if test.TestID == 0 {
		return fmt.Errorf("Cannot parse embedded files without a test ID.")
	}

	uploadedFiles, err := c.getUploadedFiles(test.TestID)
	if err != nil {
		return err
	}

	digestToFileMap := map[string]uploadedFile{}
	for _, f := range uploadedFiles {
		digestToFileMap[f.Digest] = f
	}

	replaceEmbeddedFilePaths := func(text string, embeddedFiles []embeddedFile) (string, error) {
		out := text
		for _, embed := range embeddedFiles {
			filePath := embed.path
			if strings.HasPrefix(filePath, "~/") {
				var usr *user.User
				usr, err = user.Current()
				if err != nil {
					return "", err
				}
				filePath = filepath.Join(usr.HomeDir, filePath[2:])
			} else if test.RFMLPath == "" {
				return "", fmt.Errorf("Cannot parse relative file path %v for RFML test %v. RFMLPath field cannot be blank.", filePath, test.RFMLID)
			} else {
				rfmlDirectory := filepath.Dir(test.RFMLPath)
				filePath = filepath.Join(rfmlDirectory, filePath)
			}

			filePath, err = filepath.Abs(filePath)
			if err != nil {
				return "", err
			}

			var file *os.File
			file, err = os.Open(filePath)
			if err != nil {
				return "", err
			}

			var data []byte
			data, err = ioutil.ReadAll(file)
			defer file.Close()
			if err != nil {
				return "", err
			}

			checksum := md5.Sum(data)
			fileDigest := hex.EncodeToString(checksum[:])
			uploadedFileInfo, ok := digestToFileMap[fileDigest]
			// TODO: Check mime type as well
			if !ok {
				// File has not been uploaded before
				// Upload to RF
				var awsInfo *awsFileInfo
				awsInfo, err = c.createTestFile(test.TestID, file, data)
				if err != nil {
					return "", err
				}
				// Upload to AWS
				err = c.uploadEmbeddedFile(filepath.Base(filePath), data, awsInfo)
				if err != nil {
					return "", err
				}
				uploadedFileInfo = uploadedFile{
					ID:        awsInfo.FileID,
					Signature: awsInfo.FileSignature,
					Digest:    fileDigest,
				}
				// Add to the mappings for future reference
				digestToFileMap[fileDigest] = uploadedFileInfo
			}

			sig := uploadedFileInfo.Signature[0:6]
			var replacement string
			if embed.stepVar == "screenshot" {
				replacement = fmt.Sprintf("{{ file.screenshot(%v, %v) }}", uploadedFileInfo.ID, sig)
			} else if embed.stepVar == "download" {
				replacement = fmt.Sprintf("{{ file.download(%v, %v, %v) }}", uploadedFileInfo.ID, sig, filepath.Base(filePath))
			}

			out = strings.Replace(out, embed.text, replacement, 1)
		}

		return out, nil
	}

	for idx, step := range test.Steps {
		s, ok := step.(RFTestStep)
		if ok && s.hasUploadableFiles() {
			if embeddedFiles := s.embeddedFilesInAction(); len(embeddedFiles) > 0 {
				s.Action, err = replaceEmbeddedFilePaths(s.Action, embeddedFiles)
				if err != nil {
					return err
				}
			}

			if embeddedFiles := s.embeddedFilesInResponse(); len(embeddedFiles) > 0 {
				s.Response, err = replaceEmbeddedFilePaths(s.Response, embeddedFiles)
				if err != nil {
					return err
				}
			}
		}
		test.Steps[idx] = s
	}

	return nil
}
<commit_msg>replace step only if it was tinkered<commit_after>package rainforest

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
)

// RFMLReader reads form RFML formatted file.
// It exports some settings that can be set before parsing.
type RFMLReader struct {
	r *bufio.Reader
	// Version sets the RFML spec version, it's set by NewRFMLReader to the newest one.
	Version int
	// Sets the default value of redirect, that's used when it's not specified in RFML
	RedirectDefault bool
}

// parseError is a custom error implementing error interface for reporting RFML parsing errors.
type parseError struct {
	line   int
	reason string
}

func (e *parseError) Error() string {
	return fmt.Sprintf("RFML parsing error in line %v: %v", e.line, e.reason)
}

// NewRFMLReader returns RFML parser based on passed io.Reader - typically a RFML file.
func NewRFMLReader(r io.Reader) *RFMLReader {
	return &RFMLReader{
		r:               bufio.NewReader(r),
		Version:         1,
		RedirectDefault: true,
	}
}

// ReadAll parses whole RFML file using RFML version specified by Version parameter of reader
// and returns resulting RFTest
func (r *RFMLReader) ReadAll() (*RFTest, error) {
	parsedRFTest := &RFTest{}
	// Set up a new scanner to read in data line by line
	scanner := bufio.NewScanner(r.r)
	lineNum := 0
	// Temp variables where we put stuff while parsing
	currStep := make([]string, 0, 2)
	currStepRedirect := r.RedirectDefault
	for scanner.Scan() {
		lineNum++
		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "#!") {
			if parsedRFTest.RFMLID != "" {
				return parsedRFTest, &parseError{lineNum, "Only one RFML ID may be specified"}
			}
			// Trim shebang and then take only first part of id before any spaces
			rfmlIDLine := strings.TrimSpace(line[2:])
			parsedRFTest.RFMLID = strings.Split(rfmlIDLine, " ")[0]
		} else if strings.HasPrefix(line, "#") {
			// Handle hashed lines
			content := line[1:]
			if strings.Contains(content, ":") {
				// Handle the key value pair
				split := strings.SplitN(content, ":", 2)
				key := strings.TrimSpace(split[0])
				value := strings.TrimSpace(split[1])
				switch key {
				case "title":
					parsedRFTest.Title = value
				case "start_uri":
					parsedRFTest.StartURI = value
				case "site_id":
					siteID, err := strconv.Atoi(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNum, "Site ID must be a valid integer"}
					}
					parsedRFTest.SiteID = siteID
				case "tags":
					splitTags := strings.Split(value, ",")
					strippedTags := make([]string, len(splitTags))
					for i, tag := range splitTags {
						strippedTags[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Tags = strippedTags
				case "browsers":
					splitBrowsers := strings.Split(value, ",")
					strippedBrowsers := make([]string, len(splitBrowsers))
					for i, tag := range splitBrowsers {
						strippedBrowsers[i] = strings.TrimSpace(tag)
					}
					parsedRFTest.Browsers = strippedBrowsers
				case "redirect":
					redirect, err := strconv.ParseBool(value)
					if err != nil {
						return parsedRFTest, &parseError{lineNum, "Redirect value must be a valid boolean"}
					}
					currStepRedirect = redirect
				default:
					// If it doesn't match known key add it to description
					parsedRFTest.Description += strings.TrimSpace(content) + "\n"
				}
			} else {
				// If it'a a hashed line without key-value pair add it as a comment
				parsedRFTest.Description += strings.TrimSpace(content) + "\n"
			}
		} else {
			// Handle non prefixed lines
			// Here what we do depends on the fact if we have some step data collected already
			switch len(currStep) {
			case 0:
				if strings.HasPrefix(line, "-") {
					embeddedID := strings.TrimSpace(line[strings.Index(line, "-")+1:])
					embeddedStep := RFEmbeddedTest{embeddedID, currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, embeddedStep)
					// Reset currStepRedirect
					currStepRedirect = r.RedirectDefault
				} else if line != "" {
					currStep = append(currStep, line)
				}
			case 1:
				if strings.Contains(line, "?") {
					currStep = append(currStep, line)
				} else {
					return parsedRFTest, &parseError{lineNum, "Each step must contain a question, with a `?`"}
				}
			case 2:
				if line == "" {
					parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
					parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
					// Reset temp vars to defaults
					currStep = make([]string, 0, 2)
					currStepRedirect = r.RedirectDefault
				} else {
					return parsedRFTest, &parseError{lineNum, "Steps must be separated with empty lines"}
				}
			}
		}
	}

	// Check if parsing stopped before adding a step
	if len(currStep) == 1 {
		return parsedRFTest, &parseError{lineNum, "Must have a corresponding question with your action."}
	}

	if len(currStep) == 2 {
		parsedStep := RFTestStep{currStep[0], currStep[1], currStepRedirect}
		parsedRFTest.Steps = append(parsedRFTest.Steps, parsedStep)
	}

	if parsedRFTest.RFMLID == "" {
		return parsedRFTest, &parseError{1, "RFML ID is required for .rfml files, specify it using #!"}
	}
	return parsedRFTest, nil
}

// RFMLWriter writes a RFML formatted test to a given file.
type RFMLWriter struct {
	w *bufio.Writer
	// Version sets the RFML spec version
	Version int
}

// NewRFMLWriter returns RFML writer based on passed io.Writer - typically a RFML file.
func NewRFMLWriter(w io.Writer) *RFMLWriter {
	return &RFMLWriter{
		w:       bufio.NewWriter(w),
		Version: 1,
	}
}

// WriteRFMLTest writes a given RFTest to its writer in the given RFML version.
func (r *RFMLWriter) WriteRFMLTest(test *RFTest) error {
	writer := r.w
	headerTemplate := `#! %v
# title: %v
# start_uri: %v
`

	header := fmt.Sprintf(headerTemplate, test.RFMLID, test.Title, test.StartURI)
	_, err := writer.WriteString(header)

	if err != nil {
		return err
	}

	if test.SiteID != 0 {
		_, err = writer.WriteString("# site_id: " + strconv.Itoa(test.SiteID) + "\n")

		if err != nil {
			return err
		}
	}

	if len(test.Tags) > 0 {
		tags := strings.Join(test.Tags, ", ")
		tagsHeader := fmt.Sprintf("# tags: %v\n", tags)

		_, err = writer.WriteString(tagsHeader)

		if err != nil {
			return err
		}
	}

	if len(test.Browsers) > 0 {
		browsers := strings.Join(test.Browsers, ", ")
		browsersHeader := fmt.Sprintf("# browsers: %v\n", browsers)

		_, err = writer.WriteString(browsersHeader)

		if err != nil {
			return err
		}
	}

	if test.Description != "" {
		_, err = writer.WriteString("# " + strings.Replace(test.Description, "\n", "\n# ", -1) + "\n")

		if err != nil {
			return err
		}
	}

	firstStepProcessed := false
	processStep := func(idx int, step RFTestStep) string {
		stepText := ""
		if idx > 0 && firstStepProcessed == false {
			stepText = stepText + fmt.Sprintf("# redirect: %v\n", step.Redirect)
		}
		action := strings.Replace(step.Action, "\n", " ", -1)
		response := strings.Replace(step.Response, "\n", " ", -1)
		firstStepProcessed = true

		return stepText + action + "\n" + response
	}

	for idx, step := range test.Steps {
		var stepText string
		switch step.(type) {
		case RFTestStep:
			stepText = processStep(idx, step.(RFTestStep))
		case RFEmbeddedTest:
			embeddedTest := step.(RFEmbeddedTest)
			if idx > 0 {
				stepText = "# redirect: " + strconv.FormatBool(embeddedTest.Redirect) + "\n"
			}
			stepText = stepText + "- " + embeddedTest.RFMLID
		}

		_, err = writer.WriteString("\n" + stepText + "\n")

		if err != nil {
			return err
		}
	}

	// Writes buffered data to the underlying io.Writer
	err = writer.Flush()

	if err != nil {
		return err
	}

	return nil
}

// ParseEmbeddedFiles replaces file step variable paths with values expected
// by Rainforest. eg: {{ file.screenshot(my_screenshot.gif) }} would be translated
// to the format {{ file.screenshot(FILE_ID, FILE_SIGNATURE) }}.
func (c *Client) ParseEmbeddedFiles(test *RFTest) error {
	if test.TestID == 0 {
		return fmt.Errorf("Cannot parse embedded files without a test ID.")
	}

	uploadedFiles, err := c.getUploadedFiles(test.TestID)
	if err != nil {
		return err
	}

	digestToFileMap := map[string]uploadedFile{}
	for _, f := range uploadedFiles {
		digestToFileMap[f.Digest] = f
	}

	replaceEmbeddedFilePaths := func(text string, embeddedFiles []embeddedFile) (string, error) {
		out := text
		for _, embed := range embeddedFiles {
			filePath := embed.path
			if strings.HasPrefix(filePath, "~/") {
				var usr *user.User
				usr, err = user.Current()
				if err != nil {
					return "", err
				}
				filePath = filepath.Join(usr.HomeDir, filePath[2:])
			} else if test.RFMLPath == "" {
				return "", fmt.Errorf("Cannot parse relative file path %v for RFML test %v. RFMLPath field cannot be blank.", filePath, test.RFMLID)
			} else {
				rfmlDirectory := filepath.Dir(test.RFMLPath)
				filePath = filepath.Join(rfmlDirectory, filePath)
			}

			filePath, err = filepath.Abs(filePath)
			if err != nil {
				return "", err
			}

			var file *os.File
			file, err = os.Open(filePath)
			if err != nil {
				return "", err
			}

			var data []byte
			data, err = ioutil.ReadAll(file)
			defer file.Close()
			if err != nil {
				return "", err
			}

			checksum := md5.Sum(data)
			fileDigest := hex.EncodeToString(checksum[:])
			uploadedFileInfo, ok := digestToFileMap[fileDigest]
			// TODO: Check mime type as well
			if !ok {
				// File has not been uploaded before
				// Upload to RF
				var awsInfo *awsFileInfo
				awsInfo, err = c.createTestFile(test.TestID, file, data)
				if err != nil {
					return "", err
				}
				// Upload to AWS
				err = c.uploadEmbeddedFile(filepath.Base(filePath), data, awsInfo)
				if err != nil {
					return "", err
				}
				uploadedFileInfo = uploadedFile{
					ID:        awsInfo.FileID,
					Signature: awsInfo.FileSignature,
					Digest:    fileDigest,
				}
				// Add to the mappings for future reference
				digestToFileMap[fileDigest] = uploadedFileInfo
			}

			sig := uploadedFileInfo.Signature[0:6]
			var replacement string
			if embed.stepVar == "screenshot" {
				replacement = fmt.Sprintf("{{ file.screenshot(%v, %v) }}", uploadedFileInfo.ID, sig)
			} else if embed.stepVar == "download" {
				replacement = fmt.Sprintf("{{ file.download(%v, %v, %v) }}", uploadedFileInfo.ID, sig, filepath.Base(filePath))
			}

			out = strings.Replace(out, embed.text, replacement, 1)
		}

		return out, nil
	}

	for idx, step := range test.Steps {
		s, ok := step.(RFTestStep)
		if ok && s.hasUploadableFiles() {
			if embeddedFiles := s.embeddedFilesInAction(); len(embeddedFiles) > 0 {
				s.Action, err = replaceEmbeddedFilePaths(s.Action, embeddedFiles)
				if err != nil {
					return err
				}
			}

			if embeddedFiles := s.embeddedFilesInResponse(); len(embeddedFiles) > 0 {
				s.Response, err = replaceEmbeddedFilePaths(s.Response, embeddedFiles)
				if err != nil {
					return err
				}
			}
			test.Steps[idx] = s
		}
	}

	return nil
}
<|endoftext|>