<commit_before>package main
<commit_msg>Create date format constants.<commit_after>package main

const (
	FormatLongMonth             = "January"
	FormatMonth                 = "Jan"
	FormatNumMonth              = "1"
	FormatZeroMonth             = "01"
	FormatLongWeekDay           = "Monday"
	FormatWeekDay               = "Mon"
	FormatDay                   = "2"
	FormatUnderDay              = "_2"
	FormatZeroDay               = "02"
	FormatHour                  = "15"
	FormatHour12                = "3"
	FormatZeroHour12            = "03"
	FormatMinute                = "4"
	FormatZeroMinute            = "04"
	FormatSecond                = "5"
	FormatZeroSecond            = "05"
	FormatLongYear              = "2006"
	FormatYear                  = "06"
	FormatPM                    = "PM"
	Formatpm                    = "pm"
	FormatTZ                    = "MST"
	FormatISO8601TZ             = "Z0700" // prints Z for UTC
	FormatISO8601SecondsTZ      = "Z070000"
	FormatISO8601ColonTZ        = "Z07:00" // prints Z for UTC
	FormatISO8601ColonSecondsTZ = "Z07:00:00"
	FormatNumTZ                 = "-0700" // always numeric
	FormatNumSecondsTz          = "-070000"
	FormatNumShortTZ            = "-07"    // always numeric
	FormatNumColonTZ            = "-07:00" // always numeric
	FormatNumColonSecondsTZ     = "-07:00:00"
)
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/Gujarats/API-Golang/config"
	"github.com/Gujarats/API-Golang/database"

	driverModel "github.com/Gujarats/API-Golang/model/driver"
	driverInterface "github.com/Gujarats/API-Golang/model/driver/interface"

	cityModel "github.com/Gujarats/API-Golang/model/city"
	"github.com/Gujarats/API-Golang/model/city/interface"

	driverController "github.com/Gujarats/API-Golang/controller/driver"

	"github.com/Gujarats/API-Golang/util/logger"

	mgo "gopkg.in/mgo.v2"
	redis "gopkg.in/redis.v5"
)

var cfg config.Config

func init() {
	logger.InitLogger("App :: ", "./logs/", "App.txt")
}

func main() {
	//getting list of all the connection.
	listConnection := database.SystemConnection()

	//getting redis connection convert it from interface to *redisClient.
	redisConn := listConnection["redis"].(*redis.Client)

	// get postgre connection.
	mongoConn := listConnection["mongodb"].(*mgo.Session)

	//pass connection to model to model
	//reviewData := &reviewModel.ReviewData{}
	//reviewData.GetConn(postgreConn)

	// set driver interface
	var driver driverInterface.DriverInterfacce
	driverData := &driverModel.DriverData{}
	driver = driverData

	// set city interface
	var city cityInterface.CityInterfacce
	cityData := &cityModel.City{}
	city = cityData

	// pass database connections to model
	driver.GetConn(mongoConn, redisConn)
	city.GetConn(mongoConn, redisConn)

	// driver router
	http.Handle("/driver/find", driverController.FindDriver(driver, city))
	http.Handle("/driver/update", driverController.UpdateDriver(driver))

	port := ":8080"
	fmt.Println("App Started on port = ", port)
	err := http.ListenAndServe(port, nil)
	if err != nil {
		log.Panic("App Started Failed = ", err.Error())
	}

}
<commit_msg>change UpdateDriver handler call<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/Gujarats/API-Golang/config"
	"github.com/Gujarats/API-Golang/database"

	driverModel "github.com/Gujarats/API-Golang/model/driver"
	driverInterface "github.com/Gujarats/API-Golang/model/driver/interface"

	cityModel "github.com/Gujarats/API-Golang/model/city"
	"github.com/Gujarats/API-Golang/model/city/interface"

	driverController "github.com/Gujarats/API-Golang/controller/driver"

	"github.com/Gujarats/API-Golang/util/logger"

	mgo "gopkg.in/mgo.v2"
	redis "gopkg.in/redis.v5"
)

var cfg config.Config

func init() {
	logger.InitLogger("App :: ", "./logs/", "App.txt")
}

func main() {
	//getting list of all the connection.
	listConnection := database.SystemConnection()

	//getting redis connection convert it from interface to *redisClient.
	redisConn := listConnection["redis"].(*redis.Client)

	// get postgre connection.
	mongoConn := listConnection["mongodb"].(*mgo.Session)

	//pass connection to model to model
	//reviewData := &reviewModel.ReviewData{}
	//reviewData.GetConn(postgreConn)

	// set driver interface
	var driver driverInterface.DriverInterfacce
	driverData := &driverModel.DriverData{}
	driver = driverData

	// set city interface
	var city cityInterface.CityInterfacce
	cityData := &cityModel.City{}
	city = cityData

	// pass database connections to model
	driver.GetConn(mongoConn, redisConn)
	city.GetConn(mongoConn, redisConn)

	// driver router
	http.Handle("/driver/find", driverController.FindDriver(driver, city))
	http.Handle("/driver/update", driverController.UpdateDriver(driver, city))

	port := ":8080"
	fmt.Println("App Started on port = ", port)
	err := http.ListenAndServe(port, nil)
	if err != nil {
		log.Panic("App Started Failed = ", err.Error())
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

const CAFFEINE_APP_NAME = "tsuru-caffeine-proxy"

type App struct {
	Name  string
	Cname []string
}

func startApp(host string) {
	app, err := appName(host)
	if err != nil {
		log.Println(err)
		return
	}
	log.Printf("app name: %s\n", app)

	startAppURL := fmt.Sprintf("%s/apps/%s/start", os.Getenv("TSURU_HOST"), app)

	client := &http.Client{}
	req, _ := http.NewRequest("POST", startAppURL, nil)
	req.Header.Add("Authorization", authToken())
	resp, _ := client.Do(req)
	if resp.StatusCode != 200 {
		log.Printf("Error trying to start app %s", app)
		return
	}

	log.Printf("app %s started", app)
}

func appName(host string) (string, error) {
	app := strings.Split(host, ".")[0]
	if app == CAFFEINE_APP_NAME {
		return "", errors.New("invalid app name")
	}

	appName, err := getAppName(app)
	if err == nil {
		return appName, nil
	}

	appName, err = getAppNameByCname(app)
	if err == nil {
		return appName, nil
	}

	return "", err
}

func getAppName(appName string) (string, error) {
	listAppsURL := fmt.Sprintf("%s/apps/?name=%s", os.Getenv("TSURU_HOST"), appName)

	client := &http.Client{}
	req, _ := http.NewRequest("GET", listAppsURL, nil)
	req.Header.Add("Authorization", authToken())
	resp, _ := client.Do(req)
	if resp.StatusCode != 200 {
		log.Printf("Error trying to get app %s", appName)
		return "", errors.New("Error trying to get app info")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Error trying to get app %s", appName)
		return "", errors.New("Error trying to get app info")
	}

	var apps []App
	json.Unmarshal(body, &apps)
	if len(apps) == 0 {
		log.Printf("App %s not found", appName)
		return "", errors.New("App not found")
	}

	return apps[0].Name, nil
}

func getAppNameByCname(appName string) (string, error) {
	listAppsURL := fmt.Sprintf("%s/apps/", os.Getenv("TSURU_HOST"))

	client := &http.Client{}
	req, _ := http.NewRequest("GET", listAppsURL, nil)
	req.Header.Add("Authorization", authToken())
	resp, _ := client.Do(req)
	if resp.StatusCode != 200 {
		log.Printf("Error trying to get app %s", appName)
		return "", errors.New("Error trying to get app info")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Error trying to get app %s", appName)
		return "", errors.New("Error trying to get app info")
	}

	var apps []App
	err = json.Unmarshal(body, &apps)
	if err != nil {
		log.Printf("Error trying to get app %s", appName)
		return "", errors.New("Error trying to get app info")
	}
	for _, app := range apps {
		for _, cname := range app.Cname {
			if cname == appName {
				return app.Name, nil
			}
		}
	}

	log.Printf("App %s not found", appName)
	return "", errors.New("App not found")
}

func authToken() string {
	return fmt.Sprintf("bearer %s", os.Getenv("TOKEN"))
}
<commit_msg>Refactoring app<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

const CAFFEINE_APP_NAME = "tsuru-caffeine-proxy"

type App struct {
	Name  string
	Cname []string
}

func startApp(host string) {
	app, err := appName(host)
	if err != nil {
		log.Println(err)
		return
	}
	log.Printf("app name: %s\n", app)

	startAppURL := fmt.Sprintf("%s/apps/%s/start", os.Getenv("TSURU_HOST"), app)

	client := &http.Client{}
	req, _ := http.NewRequest("POST", startAppURL, nil)
	req.Header.Add("Authorization", authToken())
	resp, _ := client.Do(req)
	if resp.StatusCode != 200 {
		log.Printf("Error trying to start app %s", app)
		return
	}

	log.Printf("app %s started", app)
}

func appName(host string) (string, error) {
	appName := strings.Split(host, ".")[0]
	if appName == CAFFEINE_APP_NAME {
		return "", errors.New("invalid app name")
	}

	app, err := getAppByName(appName)
	if err == nil {
		return app.Name, nil
	}

	app, err = getAppByCname(appName)
	if err == nil {
		return app.Name, nil
	}

	return "", err
}

func getAppByName(appName string) (*App, error) {
	listAppsURL := fmt.Sprintf("%s/apps/?name=%s", os.Getenv("TSURU_HOST"), appName)

	client := &http.Client{}
	req, _ := http.NewRequest("GET", listAppsURL, nil)
	req.Header.Add("Authorization", authToken())
	resp, _ := client.Do(req)
	if resp.StatusCode != 200 {
		log.Printf("Error trying to get app %s", appName)
		return nil, errors.New("Error trying to get app info")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Error trying to get app %s", appName)
		return nil, errors.New("Error trying to get app info")
	}

	var apps []App
	json.Unmarshal(body, &apps)
	if len(apps) == 0 {
		log.Printf("App %s not found", appName)
		return nil, errors.New("App not found")
	}

	return &apps[0], nil
}

func getAppByCname(appName string) (*App, error) {
	listAppsURL := fmt.Sprintf("%s/apps/", os.Getenv("TSURU_HOST"))

	client := &http.Client{}
	req, _ := http.NewRequest("GET", listAppsURL, nil)
	req.Header.Add("Authorization", authToken())
	resp, _ := client.Do(req)
	if resp.StatusCode != 200 {
		log.Printf("Error trying to get app %s", appName)
		return nil, errors.New("Error trying to get app info")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Error trying to get app %s", appName)
		return nil, errors.New("Error trying to get app info")
	}

	var apps []App
	err = json.Unmarshal(body, &apps)
	if err != nil {
		log.Printf("Error trying to get app %s", appName)
		return nil, errors.New("Error trying to get app info")
	}
	for _, app := range apps {
		for _, cname := range app.Cname {
			if cname == appName {
				return &app, nil
			}
		}
	}

	log.Printf("App %s not found", appName)
	return nil, errors.New("App not found")
}

func authToken() string {
	return fmt.Sprintf("bearer %s", os.Getenv("TOKEN"))
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"math/rand"
	"net"
	"net/rpc"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	log "github.com/avabot/ava/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/jbrukh/bayesian"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/jmoiron/sqlx"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/labstack/echo"
	_ "github.com/avabot/ava/Godeps/_workspace/src/github.com/lib/pq"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/stripe/stripe-go"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/subosito/twilio"
	"github.com/avabot/ava/shared/datatypes"
	"github.com/avabot/ava/shared/language"
	"github.com/avabot/ava/shared/sms"
)

// TODO variable routes. e.g. "Help me get drunk" could route to purchase
// (alcohol) or bars nearby. Ava should ask the user which route to send them
// to on packages with overlapping routes.

var db *sqlx.DB
var tc *twilio.Client
var mc *dt.MailClient
var bayes *bayesian.Classifier
var phoneRegex *regexp.Regexp
var ErrInvalidCommand = errors.New("invalid command")
var ErrMissingPackage = errors.New("missing package")
var ErrInvalidUserPass = errors.New("Invalid username/password combination")

type Ctx struct {
	Msg           *dt.Msg
	NeedsTraining bool
}

func main() {
	rand.Seed(time.Now().UnixNano())
	log.SetLevel(log.DebugLevel)
	app := cli.NewApp()
	app.Name = "ava"
	app.Usage = "general purpose ai platform"
	app.Version = "0.0.1"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "server, s",
			Usage: "run server",
		},
		cli.StringFlag{
			Name:  "port, p",
			Usage: "set port for server",
		},
		cli.BoolFlag{
			Name:  "install, i",
			Usage: "install packages in package.json",
		},
	}
	app.Action = func(c *cli.Context) {
		showHelp := true
		if c.Bool("install") {
			log.Info("TODO: install packages")
			showHelp = false
		}
		if c.Bool("server") {
			db = connectDB()
			startServer(os.Getenv("PORT"))
			showHelp = false
		}
		if showHelp {
			cli.ShowAppHelp(c)
		}
	}
	app.Run(os.Args)
}

func startServer(port string) {
	var err error
	phoneRegex = regexp.MustCompile(`^\+?[0-9\-\s()]+$`)
	if err = checkRequiredEnvVars(); err != nil {
		log.Errorln("checking env vars", err)
	}
	bayes, err = loadClassifier(bayes)
	if err != nil {
		log.Errorln("loading classifier", err)
	}
	bootRPCServer(port)
	tc = sms.NewClient()
	mc = dt.NewMailClient()
	appVocab = dt.NewAtomicMap()
	bootDependencies()
	stripe.Key = os.Getenv("STRIPE_ACCESS_TOKEN")
	e := echo.New()
	initRoutes(e)
	log.Infoln("booted ava")
	e.Run(":" + port)
}

func bootRPCServer(port string) {
	ava := new(Ava)
	if err := rpc.Register(ava); err != nil {
		log.Errorln("register ava in rpc", err)
	}
	p, err := strconv.Atoi(port)
	if err != nil {
		log.Errorln("convert port to int", err)
	}
	pt := strconv.Itoa(p + 1)
	l, err := net.Listen("tcp", ":"+pt)
	log.WithFields(log.Fields{
		"port": pt,
	}).Debugln("booting rpc server")
	if err != nil {
		log.Errorln("rpc listen", err)
	}
	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Errorln("rpc accept", err)
			}
			go rpc.ServeConn(conn)
		}
	}()
}

func connectDB() *sqlx.DB {
	log.Debugln("connecting to db")
	var d *sqlx.DB
	var err error
	if os.Getenv("AVA_ENV") == "production" {
		d, err = sqlx.Connect("postgres", os.Getenv("DATABASE_URL"))
	} else {
		d, err = sqlx.Connect("postgres",
			"user=egtann dbname=ava sslmode=disable")
	}
	if err != nil {
		log.Errorln("connecting to db", err)
	}
	log.Infoln("connected to db")
	return d
}

func preprocess(c *echo.Context) (*Ctx, error) {
	cmd := c.Get("cmd").(string)
	if len(cmd) == 0 {
		return nil, ErrInvalidCommand
	}
	if len(cmd) >= 5 && strings.ToLower(cmd)[0:5] == "train" {
		if err := train(bayes, cmd[6:]); err != nil {
			return nil, err
		}
		return nil, nil
	}
	uid, fid, fidT := validateParams(c)
	in, u, needsTraining, err := buildInput(cmd, fid, uid, fidT)
	if err != nil {
		return nil, err
	}
	ctx := &Ctx{
		Msg:           dt.NewMessage(db, u, in),
		NeedsTraining: needsTraining,
	}
	return ctx, nil
}

func buildInput(cmd, fid string, uid uint64, fidT int) (*dt.Input, *dt.User,
	bool, error) {
	si, annotated, needsTraining, err := classify(bayes, cmd)
	if err != nil {
		log.Errorln("classifying sentence", err)
	}
	in := &dt.Input{
		Sentence:          cmd,
		SentenceFields:    dt.SentenceFields(cmd),
		StructuredInput:   si,
		FlexID:            fid,
		FlexIDType:        fidT,
		UserID:            uid,
		SentenceAnnotated: annotated,
	}
	u, err := getUser(in)
	if err == dt.ErrMissingUser {
		log.Infoln("missing user", err)
	} else if err != nil {
		log.WithField("fn", "getUser").Errorln(err)
		return nil, u, needsTraining, err
	}
	in.UserID = u.ID
	return in, u, needsTraining, nil
}

func processKnowledge(ctx *Ctx, ret *dt.RespMsg, followup bool) (*dt.RespMsg,
	error) {
	var edges []*edge
	var err error
	if len(ret.Sentence) == 0 {
		edges, err = searchEdgesForTerm(ctx.Msg.Input.Sentence)
		if err != nil {
			return nil, err
		}
		for _, e := range edges {
			ctx, ret, err = processAgain(ctx, e, followup)
			if err != nil {
				return nil, err
			}
			if len(ret.Sentence) > 0 {
				e.IncrementConfidence(db)
				break
			}
			e.DecrementConfidence(db)
		}
	}
	var nodes []*node
	if len(ret.Sentence) == 0 {
		nodes, err = searchNodes(ctx.Msg.Input.Sentence,
			int64(len(edges)))
		if err != nil {
			return nil, err
		}
		for _, n := range nodes {
			if len(n.Rel()) == 0 {
				break
			}
			ctx, ret, err = processAgain(ctx, n, followup)
			if err != nil {
				return nil, err
			}
			if len(ret.Sentence) > 0 {
				n.IncrementConfidence(db)
				break
			}
			n.DecrementConfidence(db)
		}
	}
	log.Debugln("nodes found", nodes)
	log.Debugln("ret.Sentence", ret.Sentence)
	if len(ret.Sentence) == 0 && len(nodes) == 0 {
		nodes, err := newNodes(db, appVocab, ctx.Msg)
		if err != nil {
			return nil, err
		}
		if len(nodes) > 0 {
			log.Debugln("created nodes, still need to save")
			ret.Sentence = nodes[0].Text()
			return ret, nil
		}
	}
	if len(nodes) > 0 {
		ctx, ret, err = processAgain(ctx, nodes[0], followup)
		if err != nil {
			return nil, err
		}
	}
	return ret, nil
}

func processAgain(ctx *Ctx, g graphObj, followup bool) (*Ctx, *dt.RespMsg,
	error) {
	var err error
	ctx.Msg.Input.Sentence, err = replaceSentence(db, ctx.Msg, g)
	if err != nil {
		return ctx, nil, err
	}
	si, _, _, err := classify(bayes, ctx.Msg.Input.Sentence)
	if err != nil {
		log.Errorln("classifying sentence", err)
	}
	pkg, route, _, err := getPkg(ctx.Msg)
	if err != nil && err != ErrMissingPackage {
		log.WithField("fn", "getPkg").Error(err)
		return ctx, nil, err
	}
	ctx.Msg = dt.NewMessage(db, ctx.Msg.User, ctx.Msg.Input)
	ctx.Msg.Input.StructuredInput = si
	ctx.Msg.Route = route
	ret, err := callPkg(pkg, ctx.Msg, followup)
	if err != nil {
		return ctx, nil, err
	}
	return ctx, ret, nil
}

func processText(c *echo.Context) (string, error) {
	ctx, err := preprocess(c)
	if err != nil || ctx == nil /* trained */ {
		log.WithField("fn", "preprocessForMessage").Error(err)
		return "", err
	}
	pkg, route, followup, err := getPkg(ctx.Msg)
	if err != nil && err != ErrMissingPackage {
		log.WithField("fn", "getPkg").Error(err)
		return "", err
	}
	ctx.Msg.Route = route
	log.Debugln("followup?", followup)
	n, err := getActiveNode(db, ctx.Msg.User)
	if err != nil {
		return "", err
	}
	var ret *dt.RespMsg
	if n != nil {
		err = n.updateRelation(db, ctx.Msg.Input.StructuredInput)
		if err == ErrRelEqTerm {
			s := "I didn't understand that. What's " + n.Term +
				" again?"
			return s, nil
		}
		if err != nil {
			return "", err
		}
	}
	if !followup {
		log.Debugln("conversation change. deleting unused knowledgequeries")
		if err := deleteNodes(db, ctx.Msg.User); err != nil {
			return "", err
		}
	}
	ret, err = callPkg(pkg, ctx.Msg, followup)
	if err != nil {
		return "", err
	}
	if len(ret.Sentence) == 0 {
		ret, err = processKnowledge(ctx, ret, followup)
		if err != nil {
			log.WithField("fn", "callPkg").Errorln(err)
			return "", err
		}
	}
	if len(ret.Sentence) == 0 {
		ret.Sentence = language.Confused()
		log.Debugln("confused. node?", n)
		if n != nil {
			log.Debugln("confused with node. deleting unused and last knowledgequeries")
			err := deleteRecentNodes(db, ctx.Msg.User)
			if err != nil {
				return "", err
			}
		}
	}
	var pkgName string
	if pkg != nil {
		pkgName = pkg.P.Config.Name
	} else {
		pkgName = ""
	}
	id, err := saveStructuredInput(ctx.Msg, ret.ResponseID, pkgName, route)
	if err != nil {
		return "", err
	}
	ctx.Msg.Input.ID = id
	if ctx.NeedsTraining {
		log.WithField("inputID", id).Infoln("needed training")
		if err = supervisedTrain(ctx.Msg.Input); err != nil {
			return ret.Sentence, err
		}
	}
	return ret.Sentence, nil
}

func validateParams(c *echo.Context) (uint64, string, int) {
	var uid uint64
	var fidT int
	var fid string
	var err error
	tmp, ok := c.Get("uid").(string)
	if !ok {
		tmp = ""
	}
	if len(tmp) > 0 {
		uid, err = strconv.ParseUint(tmp, 10, 64)
		if err.Error() == `strconv.ParseInt: parsing "": invalid syntax` {
			uid = 0
		} else if err != nil {
			log.WithField("fn", "validateParams").Fatalln(err)
		}
	}
	tmp, ok = c.Get("flexid").(string)
	if !ok {
		tmp = ""
	}
	if len(tmp) > 0 {
		fid = tmp
		if len(fid) == 0 {
			log.WithField("fn", "validateParams").
				Fatalln("flexid is blank")
		}
	}
	tmp, ok = c.Get("flexidtype").(string)
	if !ok {
		tmp = ""
	}
	if len(tmp) > 0 {
		fidT, err = strconv.Atoi(tmp)
		if err != nil && err.Error() ==
			`strconv.ParseInt: parsing "": invalid syntax` {
			// default to 2 (SMS)
			fidT = 2
		} else if err != nil {
			log.WithField("fn", "validateParams").Fatalln(err)
		}
	}
	return uid, fid, fidT
}

func checkRequiredEnvVars() error {
	port := os.Getenv("PORT")
	_, err := strconv.Atoi(port)
	if err != nil {
		return errors.New("PORT is not set to an integer")
	}
	base := os.Getenv("BASE_URL")
	l := len(base)
	if l == 0 {
		return errors.New("BASE_URL not set")
	}
	if l < 4 || base[0:4] != "http" {
		return errors.New("BASE_URL invalid. Must include http/https")
	}
	// TODO Check for DATABASE_URL if AVA_ENV==production
	return nil
}
<commit_msg>Delete broken knowledgenodes<commit_after>package main

import (
	"errors"
	"math/rand"
	"net"
	"net/rpc"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	log "github.com/avabot/ava/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/jbrukh/bayesian"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/jmoiron/sqlx"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/labstack/echo"
	_ "github.com/avabot/ava/Godeps/_workspace/src/github.com/lib/pq"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/stripe/stripe-go"
	"github.com/avabot/ava/Godeps/_workspace/src/github.com/subosito/twilio"
	"github.com/avabot/ava/shared/datatypes"
	"github.com/avabot/ava/shared/language"
	"github.com/avabot/ava/shared/sms"
)

// TODO variable routes. e.g. "Help me get drunk" could route to purchase
// (alcohol) or bars nearby. Ava should ask the user which route to send them
// to on packages with overlapping routes.

var db *sqlx.DB
var tc *twilio.Client
var mc *dt.MailClient
var bayes *bayesian.Classifier
var phoneRegex *regexp.Regexp
var ErrInvalidCommand = errors.New("invalid command")
var ErrMissingPackage = errors.New("missing package")
var ErrInvalidUserPass = errors.New("Invalid username/password combination")

type Ctx struct {
	Msg           *dt.Msg
	NeedsTraining bool
}

func main() {
	rand.Seed(time.Now().UnixNano())
	log.SetLevel(log.DebugLevel)
	app := cli.NewApp()
	app.Name = "ava"
	app.Usage = "general purpose ai platform"
	app.Version = "0.0.1"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "server, s",
			Usage: "run server",
		},
		cli.StringFlag{
			Name:  "port, p",
			Usage: "set port for server",
		},
		cli.BoolFlag{
			Name:  "install, i",
			Usage: "install packages in package.json",
		},
	}
	app.Action = func(c *cli.Context) {
		showHelp := true
		if c.Bool("install") {
			log.Info("TODO: install packages")
			showHelp = false
		}
		if c.Bool("server") {
			db = connectDB()
			startServer(os.Getenv("PORT"))
			showHelp = false
		}
		if showHelp {
			cli.ShowAppHelp(c)
		}
	}
	app.Run(os.Args)
}

func startServer(port string) {
	var err error
	phoneRegex = regexp.MustCompile(`^\+?[0-9\-\s()]+$`)
	if err = checkRequiredEnvVars(); err != nil {
		log.Errorln("checking env vars", err)
	}
	bayes, err = loadClassifier(bayes)
	if err != nil {
		log.Errorln("loading classifier", err)
	}
	bootRPCServer(port)
	tc = sms.NewClient()
	mc = dt.NewMailClient()
	appVocab = dt.NewAtomicMap()
	bootDependencies()
	stripe.Key = os.Getenv("STRIPE_ACCESS_TOKEN")
	e := echo.New()
	initRoutes(e)
	log.Infoln("booted ava")
	e.Run(":" + port)
}

func bootRPCServer(port string) {
	ava := new(Ava)
	if err := rpc.Register(ava); err != nil {
		log.Errorln("register ava in rpc", err)
	}
	p, err := strconv.Atoi(port)
	if err != nil {
		log.Errorln("convert port to int", err)
	}
	pt := strconv.Itoa(p + 1)
	l, err := net.Listen("tcp", ":"+pt)
	log.WithFields(log.Fields{
		"port": pt,
	}).Debugln("booting rpc server")
	if err != nil {
		log.Errorln("rpc listen", err)
	}
	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				log.Errorln("rpc accept", err)
			}
			go rpc.ServeConn(conn)
		}
	}()
}

func connectDB() *sqlx.DB {
	log.Debugln("connecting to db")
	var d *sqlx.DB
	var err error
	if os.Getenv("AVA_ENV") == "production" {
		d, err = sqlx.Connect("postgres", os.Getenv("DATABASE_URL"))
	} else {
		d, err = sqlx.Connect("postgres",
			"user=egtann dbname=ava sslmode=disable")
	}
	if err != nil {
		log.Errorln("connecting to db", err)
	}
	log.Infoln("connected to db")
	return d
}

func preprocess(c *echo.Context) (*Ctx, error) {
	cmd := c.Get("cmd").(string)
	if len(cmd) == 0 {
		return nil, ErrInvalidCommand
	}
	if len(cmd) >= 5 && strings.ToLower(cmd)[0:5] == "train" {
		if err := train(bayes, cmd[6:]); err != nil {
			return nil, err
		}
		return nil, nil
	}
	uid, fid, fidT := validateParams(c)
	in, u, needsTraining, err := buildInput(cmd, fid, uid, fidT)
	if err != nil {
		return nil, err
	}
	ctx := &Ctx{
		Msg:           dt.NewMessage(db, u, in),
		NeedsTraining: needsTraining,
	}
	return ctx, nil
}

func buildInput(cmd, fid string, uid uint64, fidT int) (*dt.Input, *dt.User,
	bool, error) {
	si, annotated, needsTraining, err := classify(bayes, cmd)
	if err != nil {
		log.Errorln("classifying sentence", err)
	}
	in := &dt.Input{
		Sentence:          cmd,
		SentenceFields:    dt.SentenceFields(cmd),
		StructuredInput:   si,
		FlexID:            fid,
		FlexIDType:        fidT,
		UserID:            uid,
		SentenceAnnotated: annotated,
	}
	u, err := getUser(in)
	if err == dt.ErrMissingUser {
		log.Infoln("missing user", err)
	} else if err != nil {
		log.WithField("fn", "getUser").Errorln(err)
		return nil, u, needsTraining, err
	}
	in.UserID = u.ID
	return in, u, needsTraining, nil
}

func processKnowledge(ctx *Ctx, ret *dt.RespMsg, followup bool) (*dt.RespMsg,
	bool, error) {
	var edges []*edge
	var err error
	if len(ret.Sentence) == 0 {
		edges, err = searchEdgesForTerm(ctx.Msg.Input.Sentence)
		if err != nil {
			return nil, false, err
		}
		for _, e := range edges {
			ctx, ret, err = processAgain(ctx, e, followup)
			if err != nil {
				return nil, false, err
			}
			if len(ret.Sentence) > 0 {
				e.IncrementConfidence(db)
				break
			}
			e.DecrementConfidence(db)
		}
	}
	var nodes []*node
	if len(ret.Sentence) == 0 {
		nodes, err = searchNodes(ctx.Msg.Input.Sentence,
			int64(len(edges)))
		if err != nil {
			return nil, false, err
		}
		for _, n := range nodes {
			if len(n.Rel()) == 0 {
				break
			}
			ctx, ret, err = processAgain(ctx, n, followup)
			if err != nil {
				return nil, false, err
			}
			if len(ret.Sentence) > 0 {
				n.IncrementConfidence(db)
				break
			}
			n.DecrementConfidence(db)
		}
	}
	log.Debugln("nodes found", nodes)
	log.Debugln("ret.Sentence", ret.Sentence)
	if len(ret.Sentence) == 0 && len(nodes) == 0 {
		nodes, err := newNodes(db, appVocab, ctx.Msg)
		if err != nil {
			return nil, false, err
		}
		if len(nodes) > 0 {
			log.Debugln("created nodes, still need to save")
			ret.Sentence = nodes[0].Text()
			return ret, false, nil
		}
	}
	var changed bool
	if len(nodes) > 0 {
		ctx, ret, err = processAgain(ctx, nodes[0], followup)
		if err != nil {
			return nil, false, err
		}
		changed = true
	}
	return ret, changed, nil
}

func processAgain(ctx *Ctx, g graphObj, followup bool) (*Ctx, *dt.RespMsg,
	error) {
	var err error
	ctx.Msg.Input.Sentence, err = replaceSentence(db, ctx.Msg, g)
	if err != nil {
		return ctx, nil, err
	}
	si, _, _, err := classify(bayes, ctx.Msg.Input.Sentence)
	if err != nil {
		log.Errorln("classifying sentence", err)
	}
	pkg, route, _, err := getPkg(ctx.Msg)
	if err != nil && err != ErrMissingPackage {
		log.WithField("fn", "getPkg").Error(err)
		return ctx, nil, err
	}
	ctx.Msg = dt.NewMessage(db, ctx.Msg.User, ctx.Msg.Input)
	ctx.Msg.Input.StructuredInput = si
	ctx.Msg.Route = route
	ret, err := callPkg(pkg, ctx.Msg, followup)
	if err != nil {
		return ctx, nil, err
	}
	return ctx, ret, nil
}

func processText(c *echo.Context) (string, error) {
	ctx, err := preprocess(c)
	if err != nil || ctx == nil /* trained */ {
		log.WithField("fn", "preprocessForMessage").Error(err)
		return "", err
	}
	pkg, route, followup, err := getPkg(ctx.Msg)
	if err != nil && err != ErrMissingPackage {
		log.WithField("fn", "getPkg").Error(err)
		return "", err
	}
	ctx.Msg.Route = route
	log.Debugln("followup?", followup)
	n, err := getActiveNode(db, ctx.Msg.User)
	if err != nil {
		return "", err
	}
	var ret *dt.RespMsg
	if n != nil {
		err = n.updateRelation(db, ctx.Msg.Input.StructuredInput)
		if err == ErrRelEqTerm {
			s := "I didn't understand that. What's " + n.Term +
				" again?"
			return s, nil
		}
		if err != nil {
			return "", err
		}
	}
	if !followup {
		log.Debugln("conversation change. deleting unused knowledgequeries")
		if err := deleteNodes(db, ctx.Msg.User); err != nil {
			return "", err
		}
	}
	ret, err = callPkg(pkg, ctx.Msg, followup)
	if err != nil {
		return "", err
	}
	var changed bool
	if len(ret.Sentence) == 0 {
		ret, changed, err = processKnowledge(ctx, ret, followup)
		if err != nil {
			log.WithField("fn", "callPkg").Errorln(err)
			return "", err
		}
	}
	if len(ret.Sentence) == 0 {
		ret.Sentence = language.Confused()
		log.Debugln("confused. node?", n)
		if changed {
			log.Debugln("confused with node. deleting unused and last knowledgequeries")
			err := deleteRecentNodes(db, ctx.Msg.User)
			if err != nil {
				return "", err
			}
		}
	}
	var pkgName string
	if pkg != nil {
		pkgName = pkg.P.Config.Name
	} else {
		pkgName = ""
	}
	id, err := saveStructuredInput(ctx.Msg, ret.ResponseID, pkgName, route)
	if err != nil {
		return "", err
	}
	ctx.Msg.Input.ID = id
	if ctx.NeedsTraining {
		log.WithField("inputID", id).Infoln("needed training")
		if err = supervisedTrain(ctx.Msg.Input); err != nil {
			return ret.Sentence, err
		}
	}
	return ret.Sentence, nil
}

func validateParams(c *echo.Context) (uint64, string, int) {
	var uid uint64
	var fidT int
	var fid string
	var err error
	tmp, ok := c.Get("uid").(string)
	if !ok {
		tmp = ""
	}
	if len(tmp) > 0 {
		uid, err = strconv.ParseUint(tmp, 10, 64)
		if err.Error() == `strconv.ParseInt: parsing "": invalid syntax` {
			uid = 0
		} else if err != nil {
			log.WithField("fn", "validateParams").Fatalln(err)
		}
	}
	tmp, ok = c.Get("flexid").(string)
	if !ok {
		tmp = ""
	}
	if len(tmp) > 0 {
		fid = tmp
		if len(fid) == 0 {
			log.WithField("fn", "validateParams").
				Fatalln("flexid is blank")
		}
	}
	tmp, ok = c.Get("flexidtype").(string)
	if !ok {
		tmp = ""
	}
	if len(tmp) > 0 {
		fidT, err = strconv.Atoi(tmp)
		if err != nil && err.Error() ==
			`strconv.ParseInt: parsing "": invalid syntax` {
			// default to 2 (SMS)
			fidT = 2
		} else if err != nil {
			log.WithField("fn", "validateParams").Fatalln(err)
		}
	}
	return uid, fid, fidT
}

func checkRequiredEnvVars() error {
	port := os.Getenv("PORT")
	_, err := strconv.Atoi(port)
	if err != nil {
		return errors.New("PORT is not set to an integer")
	}
	base := os.Getenv("BASE_URL")
	l := len(base)
	if l == 0 {
		return errors.New("BASE_URL not set")
	}
	if l < 4 || base[0:4] != "http" {
		return errors.New("BASE_URL invalid. Must include http/https")
	}
	// TODO Check for DATABASE_URL if AVA_ENV==production
	return nil
}
<|endoftext|>


<commit_before>package avs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
)

const (
	EndpointURL   = "https://avs-alexa-na.amazon.com/v20160207"
	DirectivesURL = EndpointURL + "/directives"
	EventsURL     = EndpointURL + "/events"
	PingURL       = EndpointURL + "/ping"
)

// Request to AVS.
type Request struct {
	// Access token for the user that this request should be made for.
	AccessToken string         `json:"-"`
	Audio       io.Reader      `json:"-"`
	Context     []TypedMessage `json:"context"`
	Event       TypedMessage   `json:"event"`
}

func NewRequest(accessToken string) *Request {
	return &Request{
		AccessToken: accessToken,
		Context:     []TypedMessage{},
	}
}

func (r *Request) AddContext(m TypedMessage) {
	r.Context = append(r.Context, m)
}

// Representation of an AVS response.
type Response struct {
	// The Amazon request id (for debugging purposes).
	RequestId string
	// All the directives in the response.
	Directives []TypedMessage
	// Attachments (usually audio). Key is the Content-ID header value.
	Content map[string][]byte
}

// Multipart object returned by AVS.
type responsePart struct {
	Directive *Message
}

// An AVS client.
type Client struct {
}

var DefaultClient = &Client{}

// Posts a request to the AVS service.
func (c *Client) Do(request *Request) (*Response, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	err := writeJSON(writer, "metadata", request)
	if err != nil {
		return nil, err
	}
	if request.Audio != nil {
		p, err := writer.CreateFormFile("audio", "audio.wav")
		if err != nil {
			return nil, err
		}
		// TODO: Write the audio data directly to the HTTP/2 socket for faster delivery.
		_, err = io.Copy(p, request.Audio)
		if err != nil {
			return nil, err
		}
		err = writer.Close()
		if err != nil {
			return nil, err
		}
	}
	// Send the request to AVS.
	req, err := http.NewRequest("POST", EventsURL, body)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", request.AccessToken))
	req.Header.Add("Content-Type", writer.FormDataContentType())
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	switch resp.StatusCode {
	case 200:
		// Keep going.
	case 204:
		// No content.
		return nil, nil
	default:
		// Attempt to parse the response as a System.Exception message.
		data, _ := ioutil.ReadAll(resp.Body)
		var exception Exception
		json.Unmarshal(data, &exception)
		if exception.Payload.Code != "" {
			return nil, &exception
		}
		// Fallback error.
		return nil, fmt.Errorf("request failed with %s", resp.Status)
	}
	// Parse the multipart response.
	mr, err := newMultipartReaderFromResponse(resp)
	if err != nil {
		return nil, err
	}
	response := &Response{
		RequestId:  resp.Header.Get("x-amzn-requestid"),
		Directives: []TypedMessage{},
		Content:    map[string][]byte{},
	}
	for {
		p, err := mr.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		mediatype, _, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
		if err != nil {
			return nil, err
		}
		data, err := ioutil.ReadAll(p)
		if err != nil {
			return nil, err
		}
		if contentId := p.Header.Get("Content-ID"); contentId != "" {
			// This part is a referencable piece of content.
			// XXX: Content-ID generally always has angle brackets, but there may be corner cases?
			response.Content[contentId[1:len(contentId)-1]] = data
		} else if mediatype == "application/json" {
			// This is a directive.
			var resp responsePart
			err = json.Unmarshal(data, &resp)
			if err != nil {
				return nil, err
			}
			if resp.Directive == nil {
				return nil, fmt.Errorf("missing directive %s", string(data))
			}
			response.Directives = append(response.Directives, resp.Directive)
		} else {
			return nil, fmt.Errorf("unhandled part %s", p)
		}
	}
	return response, nil
}

func (c *Client) Ping(accessToken string) error {
	// TODO: Once Go supports sending PING frames, that would be a better alternative.
	req, err := http.NewRequest("GET", PingURL, nil)
	if err != nil {
		return err
	}
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", accessToken))
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	resp.Body.Close()
	return nil
}
<commit_msg>Add doc to Ping<commit_after>package avs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
)

const (
	EndpointURL   = "https://avs-alexa-na.amazon.com/v20160207"
	DirectivesURL = EndpointURL + "/directives"
	EventsURL     = EndpointURL + "/events"
	PingURL       = EndpointURL + "/ping"
)

// Request to AVS.
type Request struct {
	// Access token for the user that this request should be made for.
	AccessToken string         `json:"-"`
	Audio       io.Reader      `json:"-"`
	Context     []TypedMessage `json:"context"`
	Event       TypedMessage   `json:"event"`
}

func NewRequest(accessToken string) *Request {
	return &Request{
		AccessToken: accessToken,
		Context:     []TypedMessage{},
	}
}

func (r *Request) AddContext(m TypedMessage) {
	r.Context = append(r.Context, m)
}

// Representation of an AVS response.
type Response struct {
	// The Amazon request id (for debugging purposes).
	RequestId string
	// All the directives in the response.
	Directives []TypedMessage
	// Attachments (usually audio). Key is the Content-ID header value.
	Content map[string][]byte
}

// Multipart object returned by AVS.
type responsePart struct {
	Directive *Message
}

// An AVS client.
type Client struct {
}

var DefaultClient = &Client{}

// Posts a request to the AVS service.
func (c *Client) Do(request *Request) (*Response, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	err := writeJSON(writer, "metadata", request)
	if err != nil {
		return nil, err
	}
	if request.Audio != nil {
		p, err := writer.CreateFormFile("audio", "audio.wav")
		if err != nil {
			return nil, err
		}
		// TODO: Write the audio data directly to the HTTP/2 socket for faster delivery.
		_, err = io.Copy(p, request.Audio)
		if err != nil {
			return nil, err
		}
		err = writer.Close()
		if err != nil {
			return nil, err
		}
	}
	// Send the request to AVS.
	req, err := http.NewRequest("POST", EventsURL, body)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", request.AccessToken))
	req.Header.Add("Content-Type", writer.FormDataContentType())
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	switch resp.StatusCode {
	case 200:
		// Keep going.
	case 204:
		// No content.
		return nil, nil
	default:
		// Attempt to parse the response as a System.Exception message.
		data, _ := ioutil.ReadAll(resp.Body)
		var exception Exception
		json.Unmarshal(data, &exception)
		if exception.Payload.Code != "" {
			return nil, &exception
		}
		// Fallback error.
		return nil, fmt.Errorf("request failed with %s", resp.Status)
	}
	// Parse the multipart response.
	mr, err := newMultipartReaderFromResponse(resp)
	if err != nil {
		return nil, err
	}
	response := &Response{
		RequestId:  resp.Header.Get("x-amzn-requestid"),
		Directives: []TypedMessage{},
		Content:    map[string][]byte{},
	}
	for {
		p, err := mr.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		mediatype, _, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
		if err != nil {
			return nil, err
		}
		data, err := ioutil.ReadAll(p)
		if err != nil {
			return nil, err
		}
		if contentId := p.Header.Get("Content-ID"); contentId != "" {
			// This part is a referencable piece of content.
			// XXX: Content-ID generally always has angle brackets, but there may be corner cases?
			response.Content[contentId[1:len(contentId)-1]] = data
		} else if mediatype == "application/json" {
			// This is a directive.
			var resp responsePart
			err = json.Unmarshal(data, &resp)
			if err != nil {
				return nil, err
			}
			if resp.Directive == nil {
				return nil, fmt.Errorf("missing directive %s", string(data))
			}
			response.Directives = append(response.Directives, resp.Directive)
		} else {
			return nil, fmt.Errorf("unhandled part %s", p)
		}
	}
	return response, nil
}

// Pings AVS on behalf of a user to indicate that the connection is still alive.
func (c *Client) Ping(accessToken string) error {
	// TODO: Once Go supports sending PING frames, that would be a better alternative.
	req, err := http.NewRequest("GET", PingURL, nil)
	if err != nil {
		return err
	}
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", accessToken))
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	resp.Body.Close()
	return nil
}
<|endoftext|>


<commit_before>package ber

import (
	"bytes"
	"fmt"
	"io"
	"reflect"
)

type Packet struct {
	ClassType   uint8
	TagType     uint8
	Tag         uint8
	Value       interface{}
	Data        *bytes.Buffer
	Children    []*Packet
	Description string
}

const (
	TagEOC              = 0x00
	TagBoolean          = 0x01
	TagInteger          = 0x02
	TagBitString        = 0x03
	TagOctetString      = 0x04
	TagNULL             = 0x05
	TagObjectIdentifier = 0x06
	TagObjectDescriptor = 0x07
	TagExternal         = 0x08
	TagRealFloat        = 0x09
	TagEnumerated       = 0x0a
	TagEmbeddedPDV      = 0x0b
	TagUTF8String       = 0x0c
	TagRelativeOID      = 0x0d
	TagSequence         = 0x10
	TagSet              = 0x11
	TagNumericString    = 0x12
	TagPrintableString  = 0x13
	TagT61String        = 0x14
	TagVideotexString   = 0x15
	TagIA5String        = 0x16
	TagUTCTime          = 0x17
	TagGeneralizedTime  = 0x18
	TagGraphicString    = 0x19
	TagVisibleString    = 0x1a
	TagGeneralString    = 0x1b
	TagUniversalString  = 0x1c
	TagCharacterString  = 0x1d
	TagBMPString        = 0x1e
	TagBitmask          = 0x1f // xxx11111b
)

var TagMap = map[uint8]string{
	TagEOC:              "EOC (End-of-Content)",
	TagBoolean:          "Boolean",
	TagInteger:          "Integer",
	TagBitString:        "Bit String",
	TagOctetString:      "Octet String",
	TagNULL:             "NULL",
	TagObjectIdentifier: "Object Identifier",
	TagObjectDescriptor: "Object Descriptor",
	TagExternal:         "External",
	TagRealFloat:        "Real (float)",
	TagEnumerated:       "Enumerated",
	TagEmbeddedPDV:      "Embedded PDV",
	TagUTF8String:       "UTF8 String",
	TagRelativeOID:      "Relative-OID",
	TagSequence:         "Sequence and Sequence of",
	TagSet:              "Set and Set OF",
	TagNumericString:    "Numeric String",
	TagPrintableString:  "Printable String",
	TagT61String:        "T61 String",
	TagVideotexString:   "Videotex String",
	TagIA5String:        "IA5 String",
	TagUTCTime:          "UTC Time",
	TagGeneralizedTime:  "Generalized Time",
	TagGraphicString:    "Graphic String",
	TagVisibleString:    "Visible String",
	TagGeneralString:    "General String",
	TagUniversalString:  "Universal String",
	TagCharacterString:  "Character String",
	TagBMPString:        "BMP String",
}

const (
	ClassUniversal   = 0   // 00xxxxxxb
	ClassApplication = 64  // 01xxxxxxb
	ClassContext     = 128 // 10xxxxxxb
	ClassPrivate     = 192 // 11xxxxxxb
	ClassBitmask     = 192 // 11xxxxxxb
)

var ClassMap = map[uint8]string{
	ClassUniversal:   "Universal",
	ClassApplication: "Application",
	ClassContext:     "Context",
	ClassPrivate:     "Private",
}

const (
	TypePrimative   = 0  // xx0xxxxxb
	TypeConstructed = 32 // xx1xxxxxb
	TypeBitmask     = 32 // xx1xxxxxb
)

var TypeMap = map[uint8]string{
	TypePrimative:   "Primative",
	TypeConstructed: "Constructed",
}

var Debug bool = false

func PrintBytes(buf []byte, indent string) {
	data_lines := make([]string, (len(buf)/30)+1)
	num_lines := make([]string, (len(buf)/30)+1)

	for i, b := range buf {
		data_lines[i/30] += fmt.Sprintf("%02x ", b)
		num_lines[i/30] += fmt.Sprintf("%02d ", (i+1)%100)
	}

	for i := 0; i < len(data_lines); i++ {
		fmt.Print(indent + data_lines[i] + "\n")
		fmt.Print(indent + num_lines[i] + "\n\n")
	}
}

func PrintPacket(p *Packet) {
	printPacket(p, 0, false)
}

func printPacket(p *Packet, indent int, printBytes bool) {
	indent_str := ""
	for len(indent_str) != indent {
		indent_str += " "
	}

	class_str := ClassMap[p.ClassType]
	tagtype_str := TypeMap[p.TagType]
	tag_str := fmt.Sprintf("0x%02X", p.Tag)

	if p.ClassType == ClassUniversal {
		tag_str = TagMap[p.Tag]
	}

	value := fmt.Sprint(p.Value)
	description := ""
	if p.Description != "" {
		description = p.Description + ": "
	}

	fmt.Printf("%s%s(%s, %s, %s) Len=%d %q\n", indent_str, description, class_str, tagtype_str, tag_str, p.Data.Len(), value)

	if printBytes {
		PrintBytes(p.Bytes(), indent_str)
	}

	for _, child := range p.Children {
		printPacket(child, indent+1, printBytes)
	}
}

func resizeBuffer(in []byte, new_size uint64) (out []byte) {
	out = make([]byte, new_size)
	copy(out, in)
	return
}

func readBytes(reader io.Reader, buf []byte) error {
	idx := 0
	buflen := len(buf)
	for idx < buflen {
		n, err := reader.Read(buf[idx:])
		if err != nil {
			return err
		}
		idx += n
	}
	return nil
}

func ReadPacket(reader io.Reader) (*Packet, error) {
	buf := make([]byte, 2)
	err := readBytes(reader, buf)
	if err != nil {
		return nil, err
	}
	idx := uint64(2)
	datalen := uint64(buf[1])
	if Debug {
		fmt.Printf("Read: datalen = %d len(buf) = %d ", datalen, len(buf))
		for _, b := range buf {
			fmt.Printf("%02X ", b)
		}
		fmt.Printf("\n")
	}
	if datalen&128 != 0 {
		a := datalen - 128
		idx += a
		buf = resizeBuffer(buf, 2+a)
		err := readBytes(reader, buf[2:])
		if err != nil {
			return nil, err
		}
		datalen = DecodeInteger(buf[2 : 2+a])
		if Debug {
			fmt.Printf("Read: a = %d  idx = %d  datalen = %d  len(buf) = %d", a, idx, datalen, len(buf))
			for _, b := range buf {
				fmt.Printf("%02X ", b)
			}
			fmt.Printf("\n")
		}
	}

	buf = resizeBuffer(buf, idx+datalen)
	err = readBytes(reader, buf[idx:])
	if err != nil {
		return nil, err
	}

	if Debug {
		fmt.Printf("Read: len( buf ) = %d  idx=%d datalen=%d idx+datalen=%d\n", len(buf), idx, datalen, idx+datalen)
		for _, b := range buf {
			fmt.Printf("%02X ", b)
		}
	}

	p := DecodePacket(buf)
	return p, nil
}

func DecodeString(data []byte) (ret string) {
	for _, c := range data {
		ret += fmt.Sprintf("%c", c)
	}
	return
}

func DecodeInteger(data []byte) (ret uint64) {
	for _, i := range data {
		ret = ret * 256
		ret = ret + uint64(i)
	}
	return
}

func EncodeInteger(val uint64) []byte {
	var out bytes.Buffer
	found := false
	shift := uint(56)
	mask := uint64(0xFF00000000000000)
	for mask > 0 {
		if !found && (val&mask != 0) {
			found = true
		}
		if found || (shift == 0) {
			out.Write([]byte{byte((val & mask) >> shift)})
		}
		shift -= 8
		mask = mask >> 8
	}
	return out.Bytes()
}

func DecodePacket(data []byte) *Packet {
	p, _ := decodePacket(data)
	return p
}

func decodePacket(data []byte) (*Packet, []byte) {
	if Debug {
		fmt.Printf("decodePacket: enter %d\n", len(data))
	}
	p := new(Packet)
	p.ClassType = data[0] & ClassBitmask
	p.TagType = data[0] & TypeBitmask
	p.Tag = data[0] & TagBitmask

	datalen := DecodeInteger(data[1:2])
	datapos := uint64(2)
	if datalen&128 != 0 {
		datalen -= 128
		datapos += datalen
		datalen = DecodeInteger(data[2 : 2+datalen])
	}

	p.Data = new(bytes.Buffer)
	p.Children = make([]*Packet, 0, 2)
	p.Value = nil

	value_data := data[datapos : datapos+datalen]

	if p.TagType == TypeConstructed {
		for len(value_data) != 0 {
			var child *Packet
			child, value_data = decodePacket(value_data)
			p.AppendChild(child)
		}
	} else if p.ClassType == ClassUniversal {
		p.Data.Write(data[datapos : datapos+datalen])
		switch p.Tag {
		case TagEOC:
		case TagBoolean:
			val := DecodeInteger(value_data)
			p.Value = val != 0
		case TagInteger:
			p.Value = DecodeInteger(value_data)
		case TagBitString:
		case TagOctetString:
			p.Value = DecodeString(value_data)
		case TagNULL:
		case TagObjectIdentifier:
		case TagObjectDescriptor:
		case TagExternal:
		case TagRealFloat:
		case TagEnumerated:
			p.Value = DecodeInteger(value_data)
		case TagEmbeddedPDV:
		case TagUTF8String:
		case TagRelativeOID:
		case TagSequence:
		case TagSet:
		case TagNumericString:
		case TagPrintableString:
			p.Value = DecodeString(value_data)
		case TagT61String:
		case TagVideotexString:
		case TagIA5String:
		case TagUTCTime:
		case TagGeneralizedTime:
		case TagGraphicString:
		case TagVisibleString:
		case TagGeneralString:
		case TagUniversalString:
		case TagCharacterString:
		case TagBMPString:
		}
	} else {
		p.Data.Write(data[datapos : datapos+datalen])
	}

	return p, data[datapos+datalen:]
}

func (p *Packet) DataLength() uint64 {
	return uint64(p.Data.Len())
}

func (p *Packet) Bytes() []byte {
	var out bytes.Buffer
	out.Write([]byte{p.ClassType | p.TagType | p.Tag})
	packet_length := EncodeInteger(p.DataLength())
	if p.DataLength() > 127 || len(packet_length) > 1 {
		out.Write([]byte{byte(len(packet_length) | 128)})
		out.Write(packet_length)
	} else {
		out.Write(packet_length)
	}
	out.Write(p.Data.Bytes())
	return out.Bytes()
}

func (p *Packet) AppendChild(child *Packet) {
	p.Data.Write(child.Bytes())
	if len(p.Children) == cap(p.Children) {
		newChildren := make([]*Packet, cap(p.Children)*2)
		copy(newChildren, p.Children)
		p.Children = newChildren[0:len(p.Children)]
	}
	p.Children = p.Children[0 : len(p.Children)+1]
	p.Children[len(p.Children)-1] = child
}

func Encode(ClassType, TagType, Tag uint8, Value interface{}, Description string) *Packet {
	p := new(Packet)
	p.ClassType = ClassType
	p.TagType = TagType
	p.Tag = Tag
	p.Data = new(bytes.Buffer)
	p.Children = make([]*Packet, 0, 2)
	p.Value = Value
	p.Description = Description

	if Value != nil {
		v := reflect.ValueOf(Value)

		if ClassType == ClassUniversal {
			switch Tag {
			case TagOctetString:
				sv, ok := v.Interface().(string)
				if ok {
					p.Data.Write([]byte(sv))
				}
			}
		}
	}

	return p
}

func NewSequence(Description string) *Packet {
	return Encode(ClassUniversal, TypePrimative, TagSequence, nil, Description)
}

func NewBoolean(ClassType, TagType, Tag uint8, Value bool, Description string) *Packet {
	intValue := 0
	if Value {
		intValue = 1
	}

	p := Encode(ClassType, TagType, Tag, nil, Description)
	p.Value = Value
	p.Data.Write(EncodeInteger(uint64(intValue)))
	return p
}

func NewInteger(ClassType, TagType, Tag uint8, Value uint64, Description string) *Packet {
	p := Encode(ClassType, TagType, Tag, nil, Description)
	p.Value = Value
	p.Data.Write(EncodeInteger(Value))
	return p
}

func NewString(ClassType, TagType, Tag uint8, Value, Description string) *Packet {
	p := Encode(ClassType, TagType, Tag, nil, Description)
	p.Value = Value
	p.Data.Write([]byte(Value))
	return p
}
<commit_msg>OctetString should not be interpreted as Unicode code point<commit_after>package ber

import (
	"bytes"
	"fmt"
	"io"
	"reflect"
)

type Packet struct {
	ClassType   uint8
	TagType     uint8
	Tag         uint8
	Value       interface{}
	Data        *bytes.Buffer
	Children    []*Packet
	Description string
}

const (
	TagEOC              = 0x00
	TagBoolean          = 0x01
	TagInteger          = 0x02
	TagBitString        = 0x03
	TagOctetString      = 0x04
	TagNULL             = 0x05
	TagObjectIdentifier = 0x06
	TagObjectDescriptor = 0x07
	TagExternal         = 0x08
	TagRealFloat        = 0x09
	TagEnumerated       = 0x0a
	TagEmbeddedPDV      = 0x0b
	TagUTF8String       = 0x0c
	TagRelativeOID      = 0x0d
	TagSequence         = 0x10
	TagSet              = 0x11
	TagNumericString    = 0x12
	TagPrintableString  = 0x13
	TagT61String        = 0x14
	TagVideotexString   = 0x15
	TagIA5String        = 0x16
	TagUTCTime          = 0x17
	TagGeneralizedTime  = 0x18
	TagGraphicString    = 0x19
	TagVisibleString    = 0x1a
	TagGeneralString    = 0x1b
	TagUniversalString  = 0x1c
	TagCharacterString  = 0x1d
	TagBMPString        = 0x1e
	TagBitmask          = 0x1f // xxx11111b
)

var TagMap = map[uint8]string{
	TagEOC:              "EOC (End-of-Content)",
	TagBoolean:          "Boolean",
	TagInteger:          "Integer",
	TagBitString:        "Bit String",
	TagOctetString:      "Octet String",
	TagNULL:             "NULL",
	TagObjectIdentifier: "Object Identifier",
	TagObjectDescriptor: "Object Descriptor",
	TagExternal:         "External",
	TagRealFloat:        "Real (float)",
	TagEnumerated:       "Enumerated",
	TagEmbeddedPDV:      "Embedded PDV",
	TagUTF8String:       "UTF8 String",
	TagRelativeOID:      "Relative-OID",
	TagSequence:         "Sequence and Sequence of",
	TagSet:              "Set and Set OF",
	TagNumericString:    "Numeric String",
	TagPrintableString:  "Printable String",
	TagT61String:        "T61 String",
	TagVideotexString:   "Videotex String",
	TagIA5String:        "IA5 String",
	TagUTCTime:          "UTC Time",
	TagGeneralizedTime:  "Generalized Time",
	TagGraphicString:    "Graphic String",
	TagVisibleString:    "Visible String",
	TagGeneralString:    "General String",
	TagUniversalString:  "Universal String",
	TagCharacterString:  "Character String",
	TagBMPString:        "BMP String",
}

const (
	ClassUniversal   = 0   // 00xxxxxxb
	ClassApplication = 64  // 01xxxxxxb
	ClassContext     = 128 // 10xxxxxxb
	ClassPrivate     = 192 // 11xxxxxxb
	ClassBitmask     = 192 // 11xxxxxxb
)

var ClassMap = map[uint8]string{
	ClassUniversal:   "Universal",
	ClassApplication: "Application",
	ClassContext:     "Context",
	ClassPrivate:     "Private",
}

const (
	TypePrimative   = 0  // xx0xxxxxb
	TypeConstructed = 32 // xx1xxxxxb
	TypeBitmask     = 32 // xx1xxxxxb
)

var TypeMap = map[uint8]string{
	TypePrimative:   "Primative",
	TypeConstructed: "Constructed",
}

var Debug bool = false

func PrintBytes(buf []byte, indent string) {
	data_lines := make([]string, (len(buf)/30)+1)
	num_lines := make([]string, (len(buf)/30)+1)

	for i, b := range buf {
		data_lines[i/30] += fmt.Sprintf("%02x ", b)
		num_lines[i/30] += fmt.Sprintf("%02d ", (i+1)%100)
	}

	for i := 0; i < len(data_lines); i++ {
		fmt.Print(indent + data_lines[i] + "\n")
		fmt.Print(indent + num_lines[i] + "\n\n")
	}
}

func PrintPacket(p *Packet) {
	printPacket(p, 0, false)
}

func printPacket(p *Packet, indent int, printBytes bool) {
	indent_str := ""
	for len(indent_str) != indent {
		indent_str += " "
	}

	class_str := ClassMap[p.ClassType]
	tagtype_str := TypeMap[p.TagType]
	tag_str := fmt.Sprintf("0x%02X", p.Tag)

	if p.ClassType == ClassUniversal {
		tag_str = TagMap[p.Tag]
	}

	value := fmt.Sprint(p.Value)
	description := ""
	if p.Description != "" {
		description = p.Description + ": "
	}

	fmt.Printf("%s%s(%s, %s, %s) Len=%d %q\n", indent_str, description, class_str, tagtype_str, tag_str, p.Data.Len(), value)

	if printBytes {
		PrintBytes(p.Bytes(), indent_str)
	}

	for _, child := range p.Children {
		printPacket(child, indent+1, printBytes)
	}
}

func resizeBuffer(in []byte, new_size uint64) (out []byte) {
	out = make([]byte, new_size)
	copy(out, in)
	return
}

func readBytes(reader io.Reader, buf []byte) error {
	idx := 0
	buflen := len(buf)
	for idx < buflen {
		n, err := reader.Read(buf[idx:])
		if err != nil {
			return err
		}
		idx += n
	}
	return nil
}

func ReadPacket(reader io.Reader) (*Packet, error) {
	buf := make([]byte, 2)
	err := readBytes(reader, buf)
	if err != nil {
		return nil, err
	}
	idx := uint64(2)
	datalen := uint64(buf[1])
	if Debug {
		fmt.Printf("Read: datalen = %d len(buf) = %d ", datalen, len(buf))
		for _, b := range buf {
			fmt.Printf("%02X ", b)
		}
		fmt.Printf("\n")
	}
	if datalen&128 != 0 {
		a := datalen - 128
		idx += a
		buf = resizeBuffer(buf, 2+a)
		err := readBytes(reader, buf[2:])
		if err != nil {
			return nil, err
		}
		datalen = DecodeInteger(buf[2 : 2+a])
		if Debug {
			fmt.Printf("Read: a = %d  idx = %d  datalen = %d  len(buf) = %d", a, idx, datalen, len(buf))
			for _, b := range buf {
				fmt.Printf("%02X ", b)
			}
			fmt.Printf("\n")
		}
	}

	buf = resizeBuffer(buf, idx+datalen)
	err = readBytes(reader, buf[idx:])
	if err != nil {
		return nil, err
	}

	if Debug {
		fmt.Printf("Read: len( buf ) = %d  idx=%d datalen=%d idx+datalen=%d\n", len(buf), idx, datalen, idx+datalen)
		for _, b := range buf {
			fmt.Printf("%02X ", b)
		}
	}

	p := DecodePacket(buf)
	return p, nil
}

func DecodeString(data []byte) (ret string) {
	for _, c := range data {
		ret += fmt.Sprintf("%c", c)
	}
	return
}

func DecodeInteger(data []byte) (ret uint64) {
	for _, i := range data {
		ret = ret * 256
		ret = ret + uint64(i)
	}
	return
}

func EncodeInteger(val uint64) []byte {
	var out bytes.Buffer
	found := false
	shift := uint(56)
	mask := uint64(0xFF00000000000000)
	for mask > 0 {
		if !found && (val&mask != 0) {
			found = true
		}
		if found || (shift == 0) {
			out.Write([]byte{byte((val & mask) >> shift)})
		}
		shift -= 8
		mask = mask >> 8
	}
	return out.Bytes()
}

func DecodePacket(data []byte) *Packet {
	p, _ := decodePacket(data)
	return p
}

func decodePacket(data []byte) (*Packet, []byte) {
	if Debug {
		fmt.Printf("decodePacket: enter %d\n", len(data))
	}
	p := new(Packet)
	p.ClassType = data[0] & ClassBitmask
	p.TagType = data[0] & TypeBitmask
	p.Tag = data[0] & TagBitmask

	datalen := DecodeInteger(data[1:2])
	datapos := uint64(2)
	if datalen&128 != 0 {
		datalen -= 128
		datapos += datalen
		datalen = DecodeInteger(data[2 : 2+datalen])
	}

	p.Data = new(bytes.Buffer)
	p.Children = make([]*Packet, 0, 2)
	p.Value = nil

	value_data := data[datapos : datapos+datalen]

	if p.TagType == TypeConstructed {
		for len(value_data) != 0 {
			var child *Packet
			child, value_data = decodePacket(value_data)
			p.AppendChild(child)
		}
	} else if p.ClassType == ClassUniversal {
		p.Data.Write(data[datapos : datapos+datalen])
		switch p.Tag {
		case TagEOC:
		case TagBoolean:
			val := DecodeInteger(value_data)
			p.Value = val != 0
		case TagInteger:
			p.Value = DecodeInteger(value_data)
		case TagBitString:
		case TagOctetString:
			// should not be interpreted as Unicode code point
			// p.Value = DecodeString(value_data)
			p.Value = string(value_data)
		case TagNULL:
		case TagObjectIdentifier:
		case TagObjectDescriptor:
		case TagExternal:
		case TagRealFloat:
		case TagEnumerated:
			p.Value = DecodeInteger(value_data)
		case TagEmbeddedPDV:
		case TagUTF8String:
		case TagRelativeOID:
		case TagSequence:
		case TagSet:
		case TagNumericString:
		case TagPrintableString:
			p.Value = DecodeString(value_data)
		case TagT61String:
		case TagVideotexString:
		case TagIA5String:
		case TagUTCTime:
		case TagGeneralizedTime:
		case TagGraphicString:
		case TagVisibleString:
		case TagGeneralString:
		case TagUniversalString:
		case TagCharacterString:
		case TagBMPString:
		}
	} else {
		p.Data.Write(data[datapos : datapos+datalen])
	}

	return p, data[datapos+datalen:]
}

func (p *Packet) DataLength() uint64 {
	return uint64(p.Data.Len())
}

func (p *Packet) Bytes() []byte {
	var out bytes.Buffer
	out.Write([]byte{p.ClassType | p.TagType | p.Tag})
	packet_length := EncodeInteger(p.DataLength())
	if p.DataLength() > 127 || len(packet_length) > 1 {
		out.Write([]byte{byte(len(packet_length) | 128)})
		out.Write(packet_length)
	} else {
		out.Write(packet_length)
	}
	out.Write(p.Data.Bytes())
	return out.Bytes()
}

func (p *Packet) AppendChild(child *Packet) {
	p.Data.Write(child.Bytes())
	if len(p.Children) == cap(p.Children) {
		newChildren := make([]*Packet, cap(p.Children)*2)
		copy(newChildren, p.Children)
		p.Children = newChildren[0:len(p.Children)]
	}
	p.Children = p.Children[0 : len(p.Children)+1]
	p.Children[len(p.Children)-1] = child
}

func Encode(ClassType, TagType, Tag uint8, Value interface{}, Description string) *Packet {
	p := new(Packet)
	p.ClassType = ClassType
	p.TagType = TagType
	p.Tag = Tag
	p.Data = new(bytes.Buffer)
	p.Children = make([]*Packet, 0, 2)
	p.Value = Value
	p.Description = Description

	if Value != nil {
		v := reflect.ValueOf(Value)

		if ClassType == ClassUniversal {
			switch Tag {
			case TagOctetString:
				sv, ok := v.Interface().(string)
				if ok {
					p.Data.Write([]byte(sv))
				}
			}
		}
	}

	return p
}

func NewSequence(Description string) *Packet {
	return Encode(ClassUniversal, TypePrimative, TagSequence, nil, Description)
}

func NewBoolean(ClassType, TagType, Tag uint8, Value bool, Description string) *Packet {
	intValue := 0
	if Value {
		intValue = 1
	}

	p := Encode(ClassType, TagType, Tag, nil, Description)
	p.Value = Value
	p.Data.Write(EncodeInteger(uint64(intValue)))
	return p
}

func NewInteger(ClassType, TagType, Tag uint8, Value uint64, Description string) *Packet {
	p := Encode(ClassType, TagType, Tag, nil, Description)
	p.Value = Value
	p.Data.Write(EncodeInteger(Value))
	return p
}

func NewString(ClassType, TagType, Tag uint8, Value, Description string) *Packet {
	p := Encode(ClassType, TagType, Tag, nil, Description)
	p.Value = Value
	p.Data.Write([]byte(Value))
	return p
}
<|endoftext|>


<commit_before>package pkcs7

import (
	"bytes"
	"errors"
	"fmt"
)

type asn1Object interface {
	EncodeTo(writer *bytes.Buffer) error
}

type asn1Structured struct {
	tagBytes []byte
	content  []asn1Object
}

func (s asn1Structured) EncodeTo(out *bytes.Buffer) error {
	inner := new(bytes.Buffer)
	for _, obj := range s.content {
		err := obj.EncodeTo(inner)
		if err != nil {
			return err
		}
	}
	out.Write(s.tagBytes)
	encodeLength(out, inner.Len())
	out.Write(inner.Bytes())
	return nil
}

type asn1Primitive struct {
	tagBytes []byte
	length   int
	content  []byte
}

func (p asn1Primitive) EncodeTo(out *bytes.Buffer) error {
	_, err := out.Write(p.tagBytes)
	if err != nil {
		return err
	}
	if err = encodeLength(out, p.length); err != nil {
		return err
	}

	return nil
}

func ber2der(ber []byte) ([]byte, error) {
	fmt.Printf("--> ber2der: Transcoding %d bytes\n", len(ber))
	out := new(bytes.Buffer)

	obj, offset, err := readObject(ber, 0)
	if err != nil {
		return nil, err
	}
	obj.EncodeTo(out)

	if offset < len(ber) {
		return nil, errors.New("ber2der: Content longer than expected")
	}

	return out.Bytes(), nil
}

func marshalLongLength(out *bytes.Buffer, i int) (err error) {
	n := lengthLength(i)

	for ; n > 0; n-- {
		err = out.WriteByte(byte(i >> uint((n-1)*8)))
		if err != nil {
			return
		}
	}

	return nil
}

func lengthLength(i int) (numBytes int) {
	numBytes = 1
	for i > 255 {
		numBytes++
		i >>= 8
	}
	return
}

func encodeLength(out *bytes.Buffer, length int) (err error) {
	if length >= 128 {
		l := lengthLength(length)
		err = out.WriteByte(0x80 | byte(l))
		if err != nil {
			return
		}
		err = marshalLongLength(out, length)
		if err != nil {
			return
		}
	} else {
		err = out.WriteByte(byte(length))
		if err != nil {
			return
		}
	}
	return
}

func readObject(ber []byte, offset int) (asn1Object, int, error) {
	fmt.Printf("\n====> Starting readObject at offset: %d\n\n", offset)
	tagStart := offset
	b := ber[offset]
	offset++
	tag := b & 0x1F // last 5 bits
	if tag == 0x1F {
		tag = 0
		for ber[offset] >= 0x80 {
			tag = tag*128 + ber[offset] - 0x80
			offset++
		}
		tag = tag*128 + ber[offset] - 0x80
		offset++
	}
	tagEnd := offset

	// read length
	var length int
	l := ber[offset]
	offset++
	if l > 0x80 {
		numberOfBytes := (int)(l & 0x7F)
		fmt.Printf("--> length indicator: %x\n", l)
		fmt.Printf("--> length bytes: %x\n", ber[offset:offset+numberOfBytes])
		for i := 0; i < numberOfBytes; i++ {
			length = length*0xFF + (int)(ber[offset])
			offset++
		}
	} else if l == 0x80 {
		// find length by searching content
		markerIndex := bytes.LastIndex(ber[offset:], []byte{0x0, 0x0})
		if markerIndex == -1 {
			return nil, 0, errors.New("ber2der: Invalid BER format")
		}
		length = markerIndex - offset
		fmt.Printf("--> Undefined Length Found: %d\n", length)
	} else {
		length = (int)(l)
	}
	contentEnd := offset + length
	fmt.Printf("--> offset: %d\n", offset)
	fmt.Printf("--> contentEnd: %d\n", contentEnd)
	fmt.Printf("--> content: %x\n", ber[offset:contentEnd])
	kind := b & 0x20
	var obj asn1Object
	if kind == 0 {
		obj = asn1Primitive{
			tagBytes: ber[tagStart:tagEnd],
			length:   length,
			content:  ber[offset:contentEnd],
		}
	} else {
		var subObjects []asn1Object
		for offset < contentEnd {
			var subObj asn1Object
			var err error
			subObj, offset, err = readObject(ber, offset)
			if err != nil {
				return nil, 0, err
			}
			subObjects = append(subObjects, subObj)
		}
		obj = asn1Structured{
			tagBytes: ber[tagStart:tagEnd],
			content:  subObjects,
		}
	}

	return obj, contentEnd, nil
}
<commit_msg>Fixes to BER decoding<commit_after>package pkcs7

import (
	"bytes"
	"errors"
	"fmt"
)

type asn1Object interface {
	EncodeTo(writer *bytes.Buffer) error
}

type asn1Structured struct {
	tagBytes []byte
	content  []asn1Object
}

func (s asn1Structured) EncodeTo(out *bytes.Buffer) error {
	inner := new(bytes.Buffer)
	for _, obj := range s.content {
		err := obj.EncodeTo(inner)
		if err != nil {
			return err
		}
	}
	out.Write(s.tagBytes)
	encodeLength(out, inner.Len())
	out.Write(inner.Bytes())
	return nil
}

type asn1Primitive struct {
	tagBytes []byte
	length   int
	content  []byte
}

func (p asn1Primitive) EncodeTo(out *bytes.Buffer) error {
	_, err := out.Write(p.tagBytes)
	if err != nil {
		return err
	}
	if err = encodeLength(out, p.length); err != nil {
		return err
	}

	return nil
}

func ber2der(ber []byte) ([]byte, error) {
	fmt.Printf("--> ber2der: Transcoding %d bytes\n", len(ber))
	out := new(bytes.Buffer)

	obj, offset, err := readObject(ber, 0)
	if err != nil {
		return nil, err
	}
	obj.EncodeTo(out)

	if offset < len(ber) {
		return nil, errors.New("ber2der: Content longer than expected")
	}

	return out.Bytes(), nil
}

func marshalLongLength(out *bytes.Buffer, i int) (err error) {
	n := lengthLength(i)

	for ; n > 0; n-- {
		err = out.WriteByte(byte(i >> uint((n-1)*8)))
		if err != nil {
			return
		}
	}

	return nil
}

func lengthLength(i int) (numBytes int) {
	numBytes = 1
	for i > 255 {
		numBytes++
		i >>= 8
	}
	return
}

func encodeLength(out *bytes.Buffer, length int) (err error) {
	if length >= 128 {
		l := lengthLength(length)
		err = out.WriteByte(0x80 | byte(l))
		if err != nil {
			return
		}
		err = marshalLongLength(out, length)
		if err != nil {
			return
		}
	} else {
		err = out.WriteByte(byte(length))
		if err != nil {
			return
		}
	}
	return
}

func readObject(ber []byte, offset int) (asn1Object, int, error) {
	fmt.Printf("\n====> Starting readObject at offset: %d\n\n", offset)
	tagStart := offset
	b := ber[offset]
	offset++
	tag := b & 0x1F // last 5 bits
	if tag == 0x1F {
		tag = 0
		for ber[offset] >= 0x80 {
			tag = tag*128 + ber[offset] - 0x80
			offset++
		}
		tag = tag*128 + ber[offset] - 0x80
		offset++
	}
	tagEnd := offset

	// read length
	var length int
	l := ber[offset]
	offset++
	if l > 0x80 {
		numberOfBytes := (int)(l & 0x7F)
		fmt.Printf("--> length indicator: %x\n", l)
		fmt.Printf("--> length bytes: %x\n", ber[offset:offset+numberOfBytes])
		for i := 0; i < numberOfBytes; i++ {
			length = length*0xFF + (int)(ber[offset])
			length = length*256 + (int)(ber[offset])
			offset++
		}
	} else if l == 0x80 {
		// find length by searching content
		markerIndex := bytes.LastIndex(ber[offset:], []byte{0x0, 0x0})
		if markerIndex == -1 {
			return nil, 0, errors.New("ber2der: Invalid BER format")
		}
		fmt.Printf("--> Undefined Length Found: %d\n", length)
	} else {
		length = (int)(l)
	}
	contentEnd := offset + length
	fmt.Printf("--> offset: %d\n", offset)
	fmt.Printf("--> contentEnd: %d\n", contentEnd)
	fmt.Printf("--> content: %x\n", ber[offset:contentEnd])
	kind := b & 0x20
	var obj asn1Object
	if kind == 0 {
		obj = asn1Primitive{
			tagBytes: ber[tagStart:tagEnd],
			length:   length,
			content:  ber[offset:contentEnd],
		}
	} else {
		var subObjects []asn1Object
		for offset < contentEnd {
			var subObj asn1Object
			var err error
			subObj, offset, err = readObject(ber[:contentEnd], offset)
			if err != nil {
				return nil, 0, err
			}
			subObjects = append(subObjects, subObj)
		}
		obj = asn1Structured{
			tagBytes: ber[tagStart:tagEnd],
			content:  subObjects,
		}
	}

	return obj, contentEnd, nil
}
<|endoftext|>


<commit_before>// Tabix queries for go
package bix

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"unsafe"

	"github.com/biogo/hts/bgzf"
	"github.com/biogo/hts/bgzf/index"
	"github.com/biogo/hts/tabix"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/irelate/parsers"
	"github.com/brentp/vcfgo"
)

const (
	VCF = iota
	BED
	GENERIC
)

type location struct {
	chrom string
	start int
	end   int
}

func (s location) RefName() string {
	return s.chrom
}
func (s location) Start() int {
	return s.start
}
func (s location) End() int {
	return s.end
}

// Bix provides read access to tabix files.
type Bix struct {
	*tabix.Index
	bgzf *bgzf.Reader
	path string

	VReader *vcfgo.Reader

	file *os.File
	buf  *bufio.Reader
}

// create a new bix that does as little as possible from the old bix
func newShort(old *Bix) (*Bix, error) {
	tbx := &Bix{
		Index:   old.Index,
		path:    old.path,
		VReader: old.VReader,
	}
	var err error
	tbx.file, err = os.Open(tbx.path)
	if err != nil {
		return nil, err
	}
	tbx.bgzf, err = bgzf.NewReader(tbx.file, 1)
	if err != nil {
		return nil, err
	}
	return tbx, nil
}

// New returns a &Bix
func New(path string, workers ...int) (*Bix, error) {
	f, err := os.Open(path + ".tbi")
	if err != nil {
		return nil, err
	}
	defer f.Close()

	gz, err := gzip.NewReader(f)
	if err != nil {
		return nil, err
	}
	defer gz.Close()

	idx, err := tabix.ReadFrom(gz)
	if err != nil {
		return nil, err
	}
	n := 1
	if len(workers) > 0 {
		n = workers[0]
	}

	b, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	bgz, err := bgzf.NewReader(b, n)
	if err != nil {
		return nil, err
	}

	/*
		c := cache.NewFIFO(4)
		bgz.SetCache(c)
	*/

	buf := bufio.NewReaderSize(bgz, 16384)
	h := make([]string, 0)
	tbx := &Bix{bgzf: bgz, path: path, file: b}

	l, err := buf.ReadString('\n')
	if err != nil {
		return tbx, err
	}

	for i := 0; i < int(idx.Skip) || rune(l[0]) == idx.MetaChar; i++ {
		h = append(h, l)
		l, err = buf.ReadString('\n')
		if err != nil {
			return tbx, err
		}
	}
	header := strings.Join(h, "")

	if len(h) > 0 && strings.HasSuffix(tbx.path, ".vcf.gz") {
		var err error
		h := strings.NewReader(header)

		tbx.VReader, err = vcfgo.NewReader(h, true)
		if err != nil {
			return nil, err
		}
	}
	tbx.buf = buf
	tbx.Index = idx
	return tbx, nil
}

func (b *Bix) Close() error {
	//b.bgzf.Close()
	//b.file.Close()
	return nil
}

func (tbx *Bix) toPosition(toks [][]byte) interfaces.Relatable {
	isVCF := tbx.VReader != nil

	if isVCF {
		v := tbx.VReader.Parse(toks)
		return interfaces.AsRelatable(v)
	} else {
		g, _ := newgeneric(toks, int(tbx.Index.NameColumn-1), int(tbx.Index.BeginColumn-1),
			int(tbx.Index.EndColumn-1), tbx.Index.ZeroBased)
		return g
	}
}

func unsafeString(b []byte) string {
	return *(*string)(unsafe.Pointer(&b))
}

// return an interval using the info from the tabix index
func newgeneric(fields [][]byte, chromCol int, startCol int, endCol int, zeroBased bool) (*parsers.Interval, error) {
	s, err := strconv.Atoi(unsafeString(fields[startCol]))
	if err != nil {
		return nil, err
	}
	if !zeroBased {
		s -= 1
	}
	e, err := strconv.Atoi(unsafeString(fields[endCol]))
	if err != nil {
		return nil, err
	}
	return parsers.NewInterval(string(fields[chromCol]), uint32(s), uint32(e), fields, uint32(0), nil), nil
}

func (tbx *Bix) ChunkedReader(r tabix.Record) (io.ReadCloser, error) {
	chunks, err := tbx.Chunks(r)
	if err == index.ErrNoReference {
		l := location{r.RefName(), r.Start(), r.End()}
		if strings.HasPrefix(l.chrom, "chr") {
			l.chrom = l.chrom[3:]
			chunks, err = tbx.Chunks(l)
		} else {
			l.chrom = "chr" + l.chrom
			chunks, err = tbx.Chunks(l)
		}
	}
	if err == index.ErrInvalid {
		return index.NewChunkReader(tbx.bgzf, []bgzf.Chunk{})
	} else if err != nil {
		return nil, err
	}
	cr, err := index.NewChunkReader(tbx.bgzf, chunks)
	if err != nil {
		return nil, err
	}
	return cr, nil
}

// bixerator meets interfaces.RelatableIterator
type bixerator struct {
	rdr io.ReadCloser
	buf *bufio.Reader
	tbx *Bix

	region interfaces.IPosition
}

func (b bixerator) Next() (interfaces.Relatable, error) {

	for {
		line, err := b.buf.ReadBytes('\n')

		if err == io.EOF && len(line) == 0 {
			return nil, io.EOF
		} else if err != nil {
			return nil, err
		}
		if len(line) == 0 {
			return nil, io.EOF
		}
		if line[len(line)-1] == '\n' {
			line = line[:len(line)-1]
		}
		var in bool = true
		var toks [][]byte
		if b.region != nil {
			var err error

			in, err, toks = b.inBounds(line)
			if err != nil {
				return nil, err
			}
		} else {
			if b.tbx.VReader != nil {
				toks = bytes.SplitN(line, []byte{'\t'}, 10)
			} else {
				toks = bytes.Split(line, []byte{'\t'})
			}
		}

		if in {
			return b.tbx.toPosition(toks), nil
		}
	}
	return nil, io.EOF
}

func (b bixerator) Close() error {
	return b.tbx.Close()
}

var _ interfaces.RelatableIterator = bixerator{}

func (tbx *Bix) Query(region interfaces.IPosition) (interfaces.RelatableIterator, error) {
	tbx2, err := newShort(tbx)
	if err != nil {
		return nil, err
	}
	if region == nil {
		var l string
		var err error
		buf := bufio.NewReader(tbx2.bgzf)
		l, err = buf.ReadString('\n')
		for i := 0; i < int(tbx2.Index.Skip) || rune(l[0]) == tbx2.Index.MetaChar; i++ {
			l, err = buf.ReadString('\n')
			if err != nil {
				return nil, err
			}
		}
		if tbx2.Index.Skip == 0 && rune(l[0]) != tbx2.Index.MetaChar {
			buf = bufio.NewReader(io.MultiReader(strings.NewReader(l), buf))
		}
		return bixerator{nil, buf, tbx2, region}, nil
	}
	cr, err := tbx2.ChunkedReader(location{chrom: region.Chrom(), start: int(region.Start()), end: int(region.End())})
	if err != nil {
		return nil, err
	}
	return bixerator{cr, bufio.NewReader(cr), tbx2, region}, nil
}

func (tbx *Bix) AddInfoToHeader(id, number, vtype, desc string) {
	if tbx.VReader == nil {
		return
	}
	tbx.VReader.AddInfoToHeader(id, number, vtype, desc)
}

func (b *bixerator) inBounds(line []byte) (bool, error, [][]byte) {

	var readErr error
	line = bytes.TrimRight(line, "\r\n")
	var toks [][]byte
	if b.tbx.VReader != nil {
		toks = bytes.SplitN(line, []byte{'\t'}, 10)
	} else {
		toks = bytes.Split(line, []byte{'\t'})
	}

	s, err := strconv.Atoi(unsafeString(toks[b.tbx.BeginColumn-1]))
	if err != nil {
		return false, err, toks
	}

	pos := s
	if !b.tbx.ZeroBased {
		pos -= 1
	}
	if pos >= int(b.region.End()) {
		return false, io.EOF, toks
	}

	if b.tbx.EndColumn != 0 {
		e, err := strconv.Atoi(unsafeString(toks[b.tbx.EndColumn-1]))
		if err != nil {
			return false, err, toks
		}
		if e < int(b.region.Start()) {
			return false, readErr, toks
		}
		return true, readErr, toks
	} else if b.tbx.VReader != nil {
		start := int(b.region.Start())
		alt := strings.Split(string(toks[4]), ",")
		lref := len(toks[3])
		if start >= pos+lref {
			for _, a := range alt {
				if a[0] != '<' {
					e := pos + lref
					if e > start {
						return true, readErr, toks
					}
				} else if strings.HasPrefix(a, "<DEL") || strings.HasPrefix(a, "<DUP") || strings.HasPrefix(a, "<INV") || strings.HasPrefix(a, "<CN") {
					info := string(toks[7])
					if idx := strings.Index(info, ";END="); idx != -1 {
						v := info[idx+5 : idx+5+strings.Index(info[idx+5:], ";")]
						e, err := strconv.Atoi(v)
						if err != nil {
							return false, err, toks
						}
						if e > start {
							return true, readErr, toks
						}
					} else {
						log.Println("no end")
					}
				}
			}
		} else {
			return true, readErr, toks
		}
		return false, readErr, toks
	}
	return false, readErr, toks

}
<commit_msg>dont bail without reference, just log warning<commit_after>// Tabix queries for go
package bix

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"unsafe"

	"github.com/biogo/hts/bgzf"
	"github.com/biogo/hts/bgzf/index"
	"github.com/biogo/hts/tabix"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/irelate/parsers"
	"github.com/brentp/vcfgo"
)

const (
	VCF = iota
	BED
	GENERIC
)

type location struct {
	chrom string
	start int
	end   int
}

func (s location) RefName() string {
	return s.chrom
}
func (s location) Start() int {
	return s.start
}
func (s location) End() int {
	return s.end
}

// Bix provides read access to tabix files.
type Bix struct {
	*tabix.Index
	bgzf *bgzf.Reader
	path string

	VReader *vcfgo.Reader

	file *os.File
	buf  *bufio.Reader
}

// create a new bix that does as little as possible from the old bix
func newShort(old *Bix) (*Bix, error) {
	tbx := &Bix{
		Index:   old.Index,
		path:    old.path,
		VReader: old.VReader,
	}
	var err error
	tbx.file, err = os.Open(tbx.path)
	if err != nil {
		return nil, err
	}
	tbx.bgzf, err = bgzf.NewReader(tbx.file, 1)
	if err != nil {
		return nil, err
	}
	return tbx, nil
}

// New returns a &Bix
func New(path string, workers ...int) (*Bix, error) {
	f, err := os.Open(path + ".tbi")
	if err != nil {
		return nil, err
	}
	defer f.Close()

	gz, err := gzip.NewReader(f)
	if err != nil {
		return nil, err
	}
	defer gz.Close()

	idx, err := tabix.ReadFrom(gz)
	if err != nil {
		return nil, err
	}
	n := 1
	if len(workers) > 0 {
		n = workers[0]
	}

	b, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	bgz, err := bgzf.NewReader(b, n)
	if err != nil {
		return nil, err
	}

	/*
		c := cache.NewFIFO(4)
		bgz.SetCache(c)
	*/

	buf := bufio.NewReaderSize(bgz, 16384)
	h := make([]string, 0)
	tbx := &Bix{bgzf: bgz, path: path, file: b}

	l, err := buf.ReadString('\n')
	if err != nil {
		return tbx, err
	}

	for i := 0; i < int(idx.Skip) || rune(l[0]) == idx.MetaChar; i++ {
		h = append(h, l)
		l, err = buf.ReadString('\n')
		if err != nil {
			return tbx, err
		}
	}
	header := strings.Join(h, "")

	if len(h) > 0 && strings.HasSuffix(tbx.path, ".vcf.gz") {
		var err error
		h := strings.NewReader(header)

		tbx.VReader, err = vcfgo.NewReader(h, true)
		if err != nil {
			return nil, err
		}
	}
	tbx.buf = buf
	tbx.Index = idx
	return tbx, nil
}

func (b *Bix) Close() error {
	//b.bgzf.Close()
	//b.file.Close()
	return nil
}

func (tbx *Bix) toPosition(toks [][]byte) interfaces.Relatable {
	isVCF := tbx.VReader != nil

	if isVCF {
		v := tbx.VReader.Parse(toks)
		return interfaces.AsRelatable(v)
	} else {
		g, _ := newgeneric(toks, int(tbx.Index.NameColumn-1), int(tbx.Index.BeginColumn-1),
			int(tbx.Index.EndColumn-1), tbx.Index.ZeroBased)
		return g
	}
}

func unsafeString(b []byte) string {
	return *(*string)(unsafe.Pointer(&b))
}

// return an interval using the info from the tabix index
func newgeneric(fields [][]byte, chromCol int, startCol int, endCol int, zeroBased bool) (*parsers.Interval, error) {
	s, err := strconv.Atoi(unsafeString(fields[startCol]))
	if err != nil {
		return nil, err
	}
	if !zeroBased {
		s -= 1
	}
	e, err := strconv.Atoi(unsafeString(fields[endCol]))
	if err != nil {
		return nil, err
	}
	return parsers.NewInterval(string(fields[chromCol]), uint32(s), uint32(e), fields, uint32(0), nil), nil
}

func (tbx *Bix) ChunkedReader(r tabix.Record) (io.ReadCloser, error) {
	chunks, err := tbx.Chunks(r)
	if err == index.ErrNoReference {
		l := location{r.RefName(), r.Start(), r.End()}
		if strings.HasPrefix(l.chrom, "chr") {
			l.chrom = l.chrom[3:]
			chunks, err = tbx.Chunks(l)
		} else {
			l.chrom = "chr" + l.chrom
			chunks, err = tbx.Chunks(l)
		}
	}
	if err == index.ErrInvalid {
		return index.NewChunkReader(tbx.bgzf, []bgzf.Chunk{})
	} else if err == index.ErrNoReference {
		log.Printf("chromosome %s not found in %s\n", r.RefName(), tbx.path)
		return index.NewChunkReader(tbx.bgzf, []bgzf.Chunk{})
	} else if err != nil {
		return nil, err
	}
	cr, err := index.NewChunkReader(tbx.bgzf, chunks)
	if err != nil {
		return nil, err
	}
	return cr, nil
}

// bixerator meets interfaces.RelatableIterator
type bixerator struct {
	rdr io.ReadCloser
	buf *bufio.Reader
	tbx *Bix

	region interfaces.IPosition
}

func (b bixerator) Next() (interfaces.Relatable, error) {

	for {
		line, err := b.buf.ReadBytes('\n')

		if err == io.EOF && len(line) == 0 {
			return nil, io.EOF
		} else if err != nil {
			return nil, err
		}
		if len(line) == 0 {
			return nil, io.EOF
		}
		if line[len(line)-1] == '\n' {
			line = line[:len(line)-1]
		}
		var in bool = true
		var toks [][]byte
		if b.region != nil {
			var err error

			in, err, toks = b.inBounds(line)
			if err != nil {
				return nil, err
			}
		} else {
			if b.tbx.VReader != nil {
				toks = bytes.SplitN(line, []byte{'\t'}, 10)
			} else {
				toks = bytes.Split(line, []byte{'\t'})
			}
		}

		if in {
			return b.tbx.toPosition(toks), nil
		}
	}
	return nil, io.EOF
}

func (b bixerator) Close() error {
	return b.tbx.Close()
}

var _ interfaces.RelatableIterator = bixerator{}

func (tbx *Bix) Query(region interfaces.IPosition) (interfaces.RelatableIterator, error) {
	tbx2, err := newShort(tbx)
	if err != nil {
		return nil, err
	}
	if region == nil {
		var l string
		var err error
		buf := bufio.NewReader(tbx2.bgzf)
		l, err = buf.ReadString('\n')
		for i := 0; i < int(tbx2.Index.Skip) || rune(l[0]) == tbx2.Index.MetaChar; i++ {
			l, err = buf.ReadString('\n')
			if err != nil {
				return nil, err
			}
		}
		if tbx2.Index.Skip == 0 && rune(l[0]) != tbx2.Index.MetaChar {
			buf = bufio.NewReader(io.MultiReader(strings.NewReader(l), buf))
		}
		return bixerator{nil, buf, tbx2, region}, nil
	}
	cr, err := tbx2.ChunkedReader(location{chrom: region.Chrom(), start: int(region.Start()), end: int(region.End())})
	if err != nil {
		return nil, err
	}
	return bixerator{cr, bufio.NewReader(cr), tbx2, region}, nil
}

func (tbx *Bix) AddInfoToHeader(id, number, vtype, desc string) {
	if tbx.VReader == nil {
		return
	}
	tbx.VReader.AddInfoToHeader(id, number, vtype, desc)
}

func (b *bixerator) inBounds(line []byte) (bool, error, [][]byte) {

	var readErr error
	line = bytes.TrimRight(line, "\r\n")
	var toks [][]byte
	if b.tbx.VReader != nil {
		toks = bytes.SplitN(line, []byte{'\t'}, 10)
	} else {
		toks = bytes.Split(line, []byte{'\t'})
	}

	s, err := strconv.Atoi(unsafeString(toks[b.tbx.BeginColumn-1]))
	if err != nil {
		return false, err, toks
	}

	pos := s
	if !b.tbx.ZeroBased {
		pos -= 1
	}
	if pos >= int(b.region.End()) {
		return false, io.EOF, toks
	}

	if b.tbx.EndColumn != 0 {
		e, err := strconv.Atoi(unsafeString(toks[b.tbx.EndColumn-1]))
		if err != nil {
			return false, err, toks
		}
		if e < int(b.region.Start()) {
			return false, readErr, toks
		}
		return true, readErr, toks
	} else if b.tbx.VReader != nil {
		start := int(b.region.Start())
		alt := strings.Split(string(toks[4]), ",")
		lref := len(toks[3])
		if start >= pos+lref {
			for _, a := range alt {
				if a[0] != '<' {
					e := pos + lref
					if e > start {
						return true, readErr, toks
					}
				} else if strings.HasPrefix(a, "<DEL") || strings.HasPrefix(a, "<DUP") || strings.HasPrefix(a, "<INV") || strings.HasPrefix(a, "<CN") {
					info := string(toks[7])
					if idx := strings.Index(info, ";END="); idx != -1 {
						v := info[idx+5 : idx+5+strings.Index(info[idx+5:], ";")]
						e, err := strconv.Atoi(v)
						if err != nil {
							return false, err, toks
						}
						if e > start {
							return true, readErr, toks
						}
					} else {
						log.Println("no end")
					}
				}
			}
		} else {
			return true, readErr, toks
		}
		return false, readErr, toks
	}
	return false, readErr, toks

}
<|endoftext|>


<commit_before>package unison

import (
	"fmt"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/bwmarrin/discordgo"

	"github.com/s1kx/unison/events"
)

// BotSettings contains the definition of bot behavior.
// It is used for creating the actual bot.
type BotSettings struct {
	Token string

	Commands   []*Command
	EventHooks []*EventHook

	// Every option here is added to an array of accepted prefixes later
	// So you can set values in CommandPrefix and/or CommandPrefixes at the same time
	// This also regards the option CommandInvokedByMention
	CommandPrefix string
	CommandPrefixes []string
	CommandInvokedByMention bool
}

func RunBot(settings *BotSettings) error {
	// TODO: Validate commands

	// discordgo requires "Bot " prefix for Bot applications
	token := settings.Token
	if !strings.HasPrefix(token, "Bot ") {
		token = "Bot " + token
	}

	// Initialize discord client
	ds, err := discordgo.New(token)
	if err != nil {
		return err
	}

	// Initialize and start bot
	bot, err := newBot(settings, ds)
	if err != nil {
		return err
	}
	bot.Run()

	return nil
}

// Bot is an active bot session.
type Bot struct {
	*BotSettings
	Discord *discordgo.Session

	// Lookup map for name/alias => command
	commandMap map[string]*Command
	// Lookup map for name => hook
	eventHookMap    map[string]*EventHook
	eventDispatcher *eventDispatcher

	// Contains a generated array of accepted prefixes based on BotSettings
	commandPrefixes []string
	commandInvokedByMention bool

	readyState *discordgo.Ready
	User       *discordgo.User
}

func newBot(settings *BotSettings, ds *discordgo.Session) (*Bot, error) {
	// Initialize bot
	bot := &Bot{
		BotSettings: 				settings,
		Discord:     				ds,

		commandMap:   				make(map[string]*Command),
		eventHookMap: 				make(map[string]*EventHook),
		eventDispatcher: newEventDispatcher(),

		commandPrefixes:			[]string {},
		commandInvokedByMention: 	settings.CommandInvokedByMention,
	}

	// Register commands
	for _, cmd := range bot.Commands {
		err := bot.RegisterCommand(cmd)
		if err != nil {
			return nil, err
		}
	}

	// Register event hooks
	for _, hook := range bot.EventHooks {
		err := bot.RegisterEventHook(hook)
		if err != nil {
			return nil, err
		}
	}

	// Generate the array of accepted command prefixes.
	// Use a channel to generate one for bot mentions, or add it later
	if settings.CommandPrefix != "" {
		settings.CommandPrefixes = append(settings.CommandPrefixes, settings.CommandPrefix)
	}

	for _, prefix := range settings.CommandPrefixes {
		err := bot.RegisterCommandPrefix(prefix)
		if err != nil {
			return nil, err
		}
	}

	// Make sure at least one prefix exists, let's use "!" as default if none was given.
	if len(bot.CommandPrefixes) == 0 && !settings.CommandInvokedByMention {
		err := bot.RegisterCommandPrefix(DefaultCommandPrefix) // See command.go
		if err != nil {
			return nil, err
		}
	}

	return bot, nil
}

func (bot *Bot) Run() error {
	// Add handler to wait for ready state in order to initialize the bot fully.
	bot.Discord.AddHandler(bot.onReady)

	// Open the websocket and begin listening.
	err := bot.Discord.Open()
	if err != nil {
		return fmt.Errorf("error opening connection: %s", err)
	}

	logrus.Info("Bot is now running.  Press CTRL-C to exit.")

	// Simple way to keep program running until CTRL-C is pressed.
	// TODO: Add signal handler to exit gracefully.
	<-make(chan struct{})

	return nil
}

func (bot *Bot) RegisterCommand(cmd *Command) error {
	name := cmd.Name
	if ex, exists := bot.commandMap[name]; exists {
		return &DuplicateCommandError{Existing: ex, New: cmd, Name: name}
	}
	bot.commandMap[name] = cmd

	// TODO: Register aliases

	return nil
}

func (bot *Bot) RegisterEventHook(hook *EventHook) error {
	name := hook.Name
	if ex, exists := bot.eventHookMap[name]; exists {
		return &DuplicateEventHookError{Existing: ex, New: hook}
	}
	bot.eventHookMap[name] = hook

	if len(hook.Events) == 0 {
		logrus.Warnf("Hook '%s' is not subscribed to any events", name)
	}

	bot.eventDispatcher.AddHook(hook)

	return nil
}

func (bot *Bot) RegisterCommandPrefix(prefix string) error {
	// The prefix must have a length of minimum 1
	if (len(prefix) < 1) {
		return &TooShortCommandPrefixError{Prefix: prefix}
	}

	// Dont add one that already exists
	exists := false
	for _, existingPrefix := range bot.commandPrefixes {
		if existingPrefix == prefix {
			exists = true;
			break;
		}
	}

	if !exists {
		// Add the new prefix entry.
		bot.commandPrefixes = append(bot.commandPrefixes, prefix)
	} else {
		// Was not able to add the prefix because it already exists.
		return &DuplicateCommandPrefixError{Prefix: prefix}
	}

	return nil
}

func (bot *Bot) onReady(ds *discordgo.Session, r *discordgo.Ready) {
	// Set bot state
	bot.readyState = r
	bot.User = r.User

	logrus.WithFields(logrus.Fields{
		"ID":       r.User.ID,
		"Username": r.User.Username,
	}).Infof("Bot is connected and running.")

	// Add a command prefix based on the Bot ID if commandInvokedByMention is set to true
	if bot.commandInvokedByMention {
		bot.RegisterCommandPrefix("<@" + bot.User.ID + ">")
		//TODO[BLOCKER]: What if this fails?
	}

	// Create context for handlers
	ctx := NewContext(bot)

	// Add command handler
	bot.Discord.AddHandler(func(ds *discordgo.Session, m *discordgo.MessageCreate) {
		handleMessageCreate(ctx, ds, m)
	})

	// Add generic handler for event hooks
	// Add command handler
	bot.Discord.AddHandler(func(ds *discordgo.Session, event interface{}) {
		bot.onEvent(ds, event)
	})
}

func (bot *Bot) onEvent(ds *discordgo.Session, dv interface{}) {
	// Inspect and wrap event
	ev, err := events.NewDiscordEvent(dv)
	if err != nil {
		logrus.Errorf("event handler: %s", err)
	}

	// Create context for handlers
	ctx := NewContext(bot, ds)

	// Invoke command handler on new messages
	if ev.Type == events.MessageCreateEvent {
		handleMessageCreate(ctx, ev.Event.(*discordgo.MessageCreate))
	}

	// Invoke event hooks for the hooks that are subscribed to the event type
	bot.eventDispatcher.Dispatch(ctx, ev)
}
<commit_msg>Fix merge error<commit_after>package unison

import (
	"fmt"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/bwmarrin/discordgo"

	"github.com/s1kx/unison/events"
)

// BotSettings contains the definition of bot behavior.
// It is used for creating the actual bot.
type BotSettings struct {
	Token string

	Commands   []*Command
	EventHooks []*EventHook

	// Every option here is added to an array of accepted prefixes later
	// So you can set values in CommandPrefix and/or CommandPrefixes at the same time
	// This also regards the option CommandInvokedByMention
	CommandPrefix           string
	CommandPrefixes         []string
	CommandInvokedByMention bool
}

func RunBot(settings *BotSettings) error {
	// TODO: Validate commands

	// discordgo requires "Bot " prefix for Bot applications
	token := settings.Token
	if !strings.HasPrefix(token, "Bot ") {
		token = "Bot " + token
	}

	// Initialize discord client
	ds, err := discordgo.New(token)
	if err != nil {
		return err
	}

	// Initialize and start bot
	bot, err := newBot(settings, ds)
	if err != nil {
		return err
	}
	bot.Run()

	return nil
}

// Bot is an active bot session.
type Bot struct {
	*BotSettings
	Discord *discordgo.Session

	// Lookup map for name/alias => command
	commandMap map[string]*Command
	// Lookup map for name => hook
	eventHookMap    map[string]*EventHook
	eventDispatcher *eventDispatcher

	// Contains a generated array of accepted prefixes based on BotSettings
	commandPrefixes         []string
	commandInvokedByMention bool

	readyState *discordgo.Ready
	User       *discordgo.User
}

func newBot(settings *BotSettings, ds *discordgo.Session) (*Bot, error) {
	// Initialize bot
	bot := &Bot{
		BotSettings: settings,
		Discord:     ds,

		commandMap:      make(map[string]*Command),
		eventHookMap:    make(map[string]*EventHook),
		eventDispatcher: newEventDispatcher(),

		commandPrefixes:         []string{},
		commandInvokedByMention: settings.CommandInvokedByMention,
	}

	// Register commands
	for _, cmd := range bot.Commands {
		err := bot.RegisterCommand(cmd)
		if err != nil {
			return nil, err
		}
	}

	// Register event hooks
	for _, hook := range bot.EventHooks {
		err := bot.RegisterEventHook(hook)
		if err != nil {
			return nil, err
		}
	}

	// Generate the array of accepted command prefixes.
	// Use a channel to generate one for bot mentions, or add it later
	if settings.CommandPrefix != "" {
		settings.CommandPrefixes = append(settings.CommandPrefixes, settings.CommandPrefix)
	}

	for _, prefix := range settings.CommandPrefixes {
		err := bot.RegisterCommandPrefix(prefix)
		if err != nil {
			return nil, err
		}
	}

	// Make sure at least one prefix exists, let's use "!" as default if none was given.
	if len(bot.CommandPrefixes) == 0 && !settings.CommandInvokedByMention {
		err := bot.RegisterCommandPrefix(DefaultCommandPrefix) // See command.go
		if err != nil {
			return nil, err
		}
	}

	return bot, nil
}

func (bot *Bot) Run() error {
	// Add handler to wait for ready state in order to initialize the bot fully.
	bot.Discord.AddHandler(bot.onReady)

	// Open the websocket and begin listening.
	err := bot.Discord.Open()
	if err != nil {
		return fmt.Errorf("error opening connection: %s", err)
	}

	logrus.Info("Bot is now running.  Press CTRL-C to exit.")

	// Simple way to keep program running until CTRL-C is pressed.
	// TODO: Add signal handler to exit gracefully.
	<-make(chan struct{})

	return nil
}

func (bot *Bot) RegisterCommand(cmd *Command) error {
	name := cmd.Name
	if ex, exists := bot.commandMap[name]; exists {
		return &DuplicateCommandError{Existing: ex, New: cmd, Name: name}
	}
	bot.commandMap[name] = cmd

	// TODO: Register aliases

	return nil
}

func (bot *Bot) RegisterEventHook(hook *EventHook) error {
	name := hook.Name
	if ex, exists := bot.eventHookMap[name]; exists {
		return &DuplicateEventHookError{Existing: ex, New: hook}
	}
	bot.eventHookMap[name] = hook

	if len(hook.Events) == 0 {
		logrus.Warnf("Hook '%s' is not subscribed to any events", name)
	}

	bot.eventDispatcher.AddHook(hook)

	return nil
}

func (bot *Bot) RegisterCommandPrefix(prefix string) error {
	// The prefix must have a length of minimum 1
	if len(prefix) < 1 {
		return &TooShortCommandPrefixError{Prefix: prefix}
	}

	// Dont add one that already exists
	exists := false
	for _, existingPrefix := range bot.commandPrefixes {
		if existingPrefix == prefix {
			exists = true
			break
		}
	}

	if !exists {
		// Add the new prefix entry.
		bot.commandPrefixes = append(bot.commandPrefixes, prefix)
	} else {
		// Was not able to add the prefix because it already exists.
		return &DuplicateCommandPrefixError{Prefix: prefix}
	}

	return nil
}

func (bot *Bot) onReady(ds *discordgo.Session, r *discordgo.Ready) {
	// Set bot state
	bot.readyState = r
	bot.User = r.User

	logrus.WithFields(logrus.Fields{
		"ID":       r.User.ID,
		"Username": r.User.Username,
	}).Infof("Bot is connected and running.")

	// Add a command prefix based on the Bot ID if commandInvokedByMention is set to true
	if bot.commandInvokedByMention {
		bot.RegisterCommandPrefix("<@" + bot.User.ID + ">")
		//TODO[BLOCKER]: What if this fails?
	}

	// Add generic handler for event hooks
	// Add command handler
	bot.Discord.AddHandler(func(ds *discordgo.Session, event interface{}) {
		bot.onEvent(ds, event)
	})
}

func (bot *Bot) onEvent(ds *discordgo.Session, dv interface{}) {
	// Inspect and wrap event
	ev, err := events.NewDiscordEvent(dv)
	if err != nil {
		logrus.Errorf("event handler: %s", err)
	}

	// Create context for handlers
	ctx := NewContext(bot, ds)

	// Invoke command handler on new messages
	if ev.Type == events.MessageCreateEvent {
		handleMessageCreate(ctx, ev.Event.(*discordgo.MessageCreate))
	}

	// Invoke event hooks for the hooks that are subscribed to the event type
	bot.eventDispatcher.Dispatch(ctx, ev)
}
<|endoftext|>