<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/doc"
	"go/format"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"unicode"
)

var (
	flagExportPath     string
	flagDefaultContext bool
	flagCustomContext  string
)

const help = `Export go packages to qlang modules.

Usage:
  qexport [-contexts=""] [-defctx=false] [-outpath="./qlang"] packages

The packages for go package list or std for golang all standard packages.
`

func usage() {
	fmt.Fprintln(os.Stderr, help)
	flag.PrintDefaults()
}

func init() {
	flag.StringVar(&flagExportPath, "outpath", "./qlang", "optional set export root path")
	flag.BoolVar(&flagDefaultContext, "defctx", false, "optional use default context for build, default use all contexts.")
	flag.StringVar(&flagCustomContext, "contexts", "", "optional comma-separated list of <goos>-<goarch>[-cgo] to override default contexts.")
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) == 0 {
		usage()
		return
	}

	if flagCustomContext != "" {
		flagDefaultContext = false
		setCustomContexts(flagCustomContext)
	}

	var outpath string
	if filepath.IsAbs(flagExportPath) {
		outpath = flagExportPath
	} else {
		dir, err := os.Getwd()
		if err != nil {
			log.Fatalln(err)
		}
		outpath = filepath.Join(dir, flagExportPath)
	}

	var pkgs []string
	if args[0] == "std" {
		out, err := exec.Command("go", "list", "-e", args[0]).Output()
		if err != nil {
			log.Fatal(err)
		}
		pkgs = strings.Fields(string(out))
	} else {
		pkgs = args
	}

	for _, pkg := range pkgs {
		err := export(pkg, outpath, true)
		if err != nil {
			log.Printf("export pkg %q error, %s.\n", pkg, err)
		} else {
			log.Printf("export pkg %q success.\n", pkg)
		}
	}
}

var (
	uint64_const_keys = []string{
		"crc64.ECMA",
		"crc64.ISO",
		"math.MaxUint64",
	}
)

func isUint64Const(key string) bool {
	for _, k := range uint64_const_keys {
		if key == k {
			return true
		}
	}
	return false
}

func export(pkg string, outpath string, skipOSArch bool) error {
	p, err := NewPackage(pkg, flagDefaultContext)
	if err != nil {
		return err
	}

	p.Parser()

	bp := p.BuildPackage()
	if bp == nil {
		return errors.New("not find build")
	}

	if p.CommonCount() == 0 {
		return errors.New("empty common exports")
	}

	if pkg == "unsafe" {
		return errors.New("skip unsafe pkg")
	}

	pkgName := bp.Name

	//skip internal
	for _, path := range strings.Split(bp.ImportPath, "/") {
		if path == "internal" {
			return errors.New("skip internal pkg")
		}
	}

	var buf bytes.Buffer
	outf := func(format string, a ...interface{}) (err error) {
		_, err = buf.WriteString(fmt.Sprintf(format, a...))
		return
	}

	//write package
	outf("package %s\n", pkgName)

	//write imports
	outf("import (\n")
	outf("\t%q\n", pkg)
	outf(")\n\n")

	//write exports
	outf(`// Exports is the export table of this module.
//
var Exports = map[string]interface{}{
	"_name": "%s",
`, pkg)

	var addins []string
	//const
	if keys, _ := p.FilterCommon(Const); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			name := toQlangName(v)
			fn := pkgName + "." + v
			if isUint64Const(fn) {
				fn = "uint64(" + fn + ")"
			}
			outf("\t%q:\t%s,\n", name, fn)
		}
	}

	//vars
	if keys, _ := p.FilterCommon(Var); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			name := toQlangName(v)
			fn := pkgName + "." + v
			outf("\t%q:\t%s,\n", name, fn)
		}
	}

	//funcs
	if keys, _ := p.FilterCommon(Func); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			name := toQlangName(v)
			fn := pkgName + "." + v
			outf("\t%q:\t%s,\n", name, fn)
		}
	}

	//structs
	if keys, m := p.FilterCommon(Struct); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			t, ok := m[v]
			if !ok {
				continue
			}
			dt, ok := t.(*doc.Type)
			if !ok {
				continue
			}
			//empty func
			if len(dt.Funcs) == 0 && ast.IsExported(v) {
				//fmt.Println(v)
				name := toQlangName(v)
				var vfn string = "new" + v
				var tname string = pkgName + "." + v
				addins = append(addins, fmt.Sprintf("func %s() *%s {\n\treturn new(%s)\n}",
					vfn, tname, tname,
				))
				var vfns string = "new" + v + "Array"
				addins = append(addins, fmt.Sprintf("func %s(n int) []%s {\n\treturn make([]%s,n)\n}",
					vfns, tname, tname,
				))
				outf("\t%q:\t%s,\n", name, vfn)
				outf("\t%q:\t%s,\n", name+"Array", vfns)
			} else {
				//write factor func and check is common
				var funcs []string
				for _, f := range dt.Funcs {
					if ast.IsExported(f.Name) {
						funcs = append(funcs, f.Name)
					}
				}
				for _, f := range funcs {
					name := toQlangName(f)
					if len(funcs) == 0 {
						name = toQlangName(v)
					}
					fn := pkgName + "." + f
					outf("\t%q:\t%s,\n", name, fn)
				}
			}
		}
	}

	// end exports
	outf("}")

	if len(addins) > 0 {
		for _, addin := range addins {
			outf("\n\n")
			outf(addin)
		}
	}

	// format
	data, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	// write file
	root := filepath.Join(outpath, pkg)
	err = os.MkdirAll(root, 0777)
	if err != nil {
		return err
	}

	file, err := os.Create(filepath.Join(root, pkgName+".go"))
	if err != nil {
		return err
	}
	defer file.Close()
	file.Write(data)

	return nil
}

func toQlangName(s string) string {
	if len(s) <= 1 {
		return s
	}

	if unicode.IsLower(rune(s[1])) {
		return strings.ToLower(s[0:1]) + s[1:]
	}
	return s
}
<commit_msg>qexport fix go1.4.2 skip cmd<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/doc"
	"go/format"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"unicode"
)

var (
	flagExportPath     string
	flagDefaultContext bool
	flagCustomContext  string
)

const help = `Export go packages to qlang modules.

Usage:
  qexport [-contexts=""] [-defctx=false] [-outpath="./qlang"] packages

The packages for go package list or std for golang all standard packages.
`

func usage() {
	fmt.Fprintln(os.Stderr, help)
	flag.PrintDefaults()
}

func init() {
	flag.StringVar(&flagExportPath, "outpath", "./qlang", "optional set export root path")
	flag.BoolVar(&flagDefaultContext, "defctx", false, "optional use default context for build, default use all contexts.")
	flag.StringVar(&flagCustomContext, "contexts", "", "optional comma-separated list of <goos>-<goarch>[-cgo] to override default contexts.")
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) == 0 {
		usage()
		return
	}

	if flagCustomContext != "" {
		flagDefaultContext = false
		setCustomContexts(flagCustomContext)
	}

	var outpath string
	if filepath.IsAbs(flagExportPath) {
		outpath = flagExportPath
	} else {
		dir, err := os.Getwd()
		if err != nil {
			log.Fatalln(err)
		}
		outpath = filepath.Join(dir, flagExportPath)
	}

	var pkgs []string
	if args[0] == "std" {
		out, err := exec.Command("go", "list", "-e", args[0]).Output()
		if err != nil {
			log.Fatal(err)
		}
		pkgs = strings.Fields(string(out))
	} else {
		pkgs = args
	}

	for _, pkg := range pkgs {
		err := export(pkg, outpath, true)
		if err != nil {
			log.Printf("export pkg %q error, %s.\n", pkg, err)
		} else {
			log.Printf("export pkg %q success.\n", pkg)
		}
	}
}

var (
	uint64_const_keys = []string{
		"crc64.ECMA",
		"crc64.ISO",
		"math.MaxUint64",
	}
)

func isUint64Const(key string) bool {
	for _, k := range uint64_const_keys {
		if key == k {
			return true
		}
	}
	return false
}

func export(pkg string, outpath string, skipOSArch bool) error {
	p, err := NewPackage(pkg, flagDefaultContext)
	if err != nil {
		return err
	}

	p.Parser()

	bp := p.BuildPackage()
	if bp == nil {
		return errors.New("not find build")
	}

	pkgName := bp.Name

	if bp.Name == "main" {
		return errors.New("skip main pkg")
	}

	if pkg == "unsafe" {
		return errors.New("skip unsafe pkg")
	}

	if p.CommonCount() == 0 {
		return errors.New("empty common exports")
	}

	//skip internal
	for _, path := range strings.Split(bp.ImportPath, "/") {
		if path == "internal" {
			return errors.New("skip internal pkg")
		}
	}

	var buf bytes.Buffer
	outf := func(format string, a ...interface{}) (err error) {
		_, err = buf.WriteString(fmt.Sprintf(format, a...))
		return
	}

	//write package
	outf("package %s\n", pkgName)

	//write imports
	outf("import (\n")
	outf("\t%q\n", pkg)
	outf(")\n\n")

	//write exports
	outf(`// Exports is the export table of this module.
//
var Exports = map[string]interface{}{
	"_name": "%s",
`, pkg)

	var addins []string
	//const
	if keys, _ := p.FilterCommon(Const); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			name := toQlangName(v)
			fn := pkgName + "." + v
			if isUint64Const(fn) {
				fn = "uint64(" + fn + ")"
			}
			outf("\t%q:\t%s,\n", name, fn)
		}
	}

	//vars
	if keys, _ := p.FilterCommon(Var); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			name := toQlangName(v)
			fn := pkgName + "." + v
			outf("\t%q:\t%s,\n", name, fn)
		}
	}

	//funcs
	if keys, _ := p.FilterCommon(Func); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			name := toQlangName(v)
			fn := pkgName + "." + v
			outf("\t%q:\t%s,\n", name, fn)
		}
	}

	//structs
	if keys, m := p.FilterCommon(Struct); len(keys) > 0 {
		outf("\n")
		for _, v := range keys {
			t, ok := m[v]
			if !ok {
				continue
			}
			dt, ok := t.(*doc.Type)
			if !ok {
				continue
			}
			//empty func
			if len(dt.Funcs) == 0 && ast.IsExported(v) {
				//fmt.Println(v)
				name := toQlangName(v)
				var vfn string = "new" + v
				var tname string = pkgName + "." + v
				addins = append(addins, fmt.Sprintf("func %s() *%s {\n\treturn new(%s)\n}",
					vfn, tname, tname,
				))
				var vfns string = "new" + v + "Array"
				addins = append(addins, fmt.Sprintf("func %s(n int) []%s {\n\treturn make([]%s,n)\n}",
					vfns, tname, tname,
				))
				outf("\t%q:\t%s,\n", name, vfn)
				outf("\t%q:\t%s,\n", name+"Array", vfns)
			} else {
				//write factor func and check is common
				var funcs []string
				for _, f := range dt.Funcs {
					if ast.IsExported(f.Name) {
						funcs = append(funcs, f.Name)
					}
				}
				for _, f := range funcs {
					name := toQlangName(f)
					if len(funcs) == 0 {
						name = toQlangName(v)
					}
					fn := pkgName + "." + f
					outf("\t%q:\t%s,\n", name, fn)
				}
			}
		}
	}

	// end exports
	outf("}")

	if len(addins) > 0 {
		for _, addin := range addins {
			outf("\n\n")
			outf(addin)
		}
	}

	// format
	data, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	// write file
	root := filepath.Join(outpath, pkg)
	err = os.MkdirAll(root, 0777)
	if err != nil {
		return err
	}

	file, err := os.Create(filepath.Join(root, pkgName+".go"))
	if err != nil {
		return err
	}
	defer file.Close()
	file.Write(data)

	return nil
}

func toQlangName(s string) string {
	if len(s) <= 1 {
		return s
	}

	if unicode.IsLower(rune(s[1])) {
		return strings.ToLower(s[0:1]) + s[1:]
	}
	return s
}
<|endoftext|>


<commit_before>package tokens

import (
	"github.com/dustin/go-humanize"
	"math/rand"
	"sync"
	"time"
	"log"
)

type Token struct {
	ValidTo         time.Time
	ExpiresReadable string
	Id              string
	VerifiedCount   int
}

type Tokens struct {
	sync.RWMutex
	tokens []Token
}

func Init() Tokens {
	t := Tokens{}
	return t
}

func (t *Tokens) Generate() string {
	t.Cleanup()

	var token Token
	token.Id = RandomString(8)
	now := time.Now().UTC()
	token.ValidTo = now.Add(1 * time.Minute)

	t.Lock()
	t.tokens = append([]Token{token}, t.tokens...)
	t.Unlock()
	return token.Id
}

func (t *Tokens) Verify(token string) bool {
	t.Lock()
	found := false
	now := time.Now().UTC()
	for i, data := range t.tokens {
		if data.Id == token {
			if now.Before(data.ValidTo) {
				t.tokens[i].VerifiedCount = t.tokens[i].VerifiedCount + 1
				found = true
			}
		}
	}
	t.Unlock()
	return found
}

func (t *Tokens) removeToken(token string) {
	t.Lock()
	for i, data := range t.tokens {
		if data.Id == token {
			t.tokens = append(t.tokens[:i], t.tokens[i+1:]...)
		}
	}
	t.Unlock()
}

func (t *Tokens) Cleanup() {
	if len(t.tokens) > 500 {
		now := time.Now().UTC()
		before := len(t.tokens)
		for _, data := range t.tokens {
			if now.After(data.ValidTo) {
				t.removeToken(data.Id)
			}
		}
		after := len(t.tokens)
		log.Println("Token clean up:", before-after, "tokens have been removed.")
	}
}

func (t *Tokens) GetAllTokens() []Token {
	t.RLock()
	defer t.RUnlock()

	var r []Token
	for _, data := range t.tokens {
		data.ExpiresReadable = humanize.Time(data.ValidTo)
		r = append(r, data)
	}
	return r
}

func RandomString(n int) string {
	var letters = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}
<commit_msg>Tokens are valid for 5 minutes<commit_after>package tokens

import (
	"github.com/dustin/go-humanize"
	"math/rand"
	"sync"
	"time"
	"log"
)

type Token struct {
	ValidTo         time.Time
	ExpiresReadable string
	Id              string
	VerifiedCount   int
}

type Tokens struct {
	sync.RWMutex
	tokens []Token
}

func Init() Tokens {
	t := Tokens{}
	return t
}

func (t *Tokens) Generate() string {
	t.Cleanup()

	var token Token
	token.Id = RandomString(8)
	now := time.Now().UTC()
	token.ValidTo = now.Add(5 * time.Minute)

	t.Lock()
	t.tokens = append([]Token{token}, t.tokens...)
	t.Unlock()
	return token.Id
}

func (t *Tokens) Verify(token string) bool {
	t.Lock()
	found := false
	now := time.Now().UTC()
	for i, data := range t.tokens {
		if data.Id == token {
			if now.Before(data.ValidTo) {
				t.tokens[i].VerifiedCount = t.tokens[i].VerifiedCount + 1
				found = true
			}
		}
	}
	t.Unlock()
	return found
}

func (t *Tokens) removeToken(token string) {
	t.Lock()
	for i, data := range t.tokens {
		if data.Id == token {
			t.tokens = append(t.tokens[:i], t.tokens[i+1:]...)
		}
	}
	t.Unlock()
}

func (t *Tokens) Cleanup() {
	if len(t.tokens) > 500 {
		now := time.Now().UTC()
		before := len(t.tokens)
		for _, data := range t.tokens {
			if now.After(data.ValidTo) {
				t.removeToken(data.Id)
			}
		}
		after := len(t.tokens)
		log.Println("Token clean up:", before-after, "tokens have been removed.")
	}
}

func (t *Tokens) GetAllTokens() []Token {
	t.RLock()
	defer t.RUnlock()

	var r []Token
	for _, data := range t.tokens {
		data.ExpiresReadable = humanize.Time(data.ValidTo)
		r = append(r, data)
	}
	return r
}

func RandomString(n int) string {
	var letters = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}
<|endoftext|>


<commit_before>package tokens

import (
	"github.com/dustin/go-humanize"
	"math/rand"
	"sync"
	"time"
	"log"
)

type Token struct {
	ValidTo         time.Time
	ExpiresReadable string
	Id              string
	VerifiedCount   int
}

type Tokens struct {
	sync.RWMutex
	tokens []Token
}

func Init() Tokens {
	t := Tokens{}
	return t
}

func (t *Tokens) Generate() string {
	t.Cleanup()

	var token Token
	token.Id = RandomString(8)
	now := time.Now().UTC()
	token.ValidTo = now.Add(5 * time.Minute)

	t.Lock()
	t.tokens = append([]Token{token}, t.tokens...)
	t.Unlock()
	return token.Id
}

func (t *Tokens) Verify(token string) bool {
	t.Lock()
	found := false
	now := time.Now().UTC()
	for i, data := range t.tokens {
		if data.Id == token {
			if now.Before(data.ValidTo) {
				t.tokens[i].VerifiedCount = t.tokens[i].VerifiedCount + 1
				found = true
			}
		}
	}
	t.Unlock()
	return found
}

func (t *Tokens) removeToken(token string) {
	t.Lock()
	for i, data := range t.tokens {
		if data.Id == token {
			t.tokens = append(t.tokens[:i], t.tokens[i+1:]...)
		}
	}
	t.Unlock()
}

func (t *Tokens) Cleanup() {
	if len(t.tokens) > 500 {
		now := time.Now().UTC()
		before := len(t.tokens)
		for _, data := range t.tokens {
			if now.After(data.ValidTo) {
				t.removeToken(data.Id)
			}
		}
		after := len(t.tokens)
		log.Println("Token clean up:", before-after, "tokens have been removed.")
	}
}

func (t *Tokens) GetAllTokens() []Token {
	t.RLock()
	defer t.RUnlock()

	var r []Token
	for _, data := range t.tokens {
		data.ExpiresReadable = humanize.Time(data.ValidTo)
		r = append(r, data)
	}
	return r
}

func RandomString(n int) string {
	var letters = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}
<commit_msg>Improve token clean up<commit_after>package tokens

import (
	"github.com/dustin/go-humanize"
	"math/rand"
	"sync"
	"time"
	"log"
)

type Token struct {
	ValidTo         time.Time
	ExpiresReadable string
	Id              string
	VerifiedCount   int
}

type Tokens struct {
	sync.RWMutex
	tokens []Token
}

func Init() Tokens {
	t := Tokens{}
	return t
}

func (t *Tokens) Generate() string {
	t.Cleanup()

	var token Token
	token.Id = RandomString(8)
	now := time.Now().UTC()
	token.ValidTo = now.Add(5 * time.Minute)

	t.Lock()
	t.tokens = append([]Token{token}, t.tokens...)
	t.Unlock()
	return token.Id
}

func (t *Tokens) Verify(token string) bool {
	t.Lock()
	found := false
	now := time.Now().UTC()
	for i, data := range t.tokens {
		if data.Id == token {
			if now.Before(data.ValidTo) {
				t.tokens[i].VerifiedCount = t.tokens[i].VerifiedCount + 1
				found = true
			}
		}
	}
	t.Unlock()
	return found
}

func (t *Tokens) Cleanup() {
	var valid []Token
	t.Lock()
	if len(t.tokens) > 500 {
		now := time.Now().UTC()
		for _, data := range t.tokens {
			if now.Before(data.ValidTo) {
				valid = append(valid, data)
			}
		}
		before := len(t.tokens)
		t.tokens = valid
		after := len(t.tokens)
		log.Println("Token clean up:", before-after, "tokens have been removed.")
	}
	t.Unlock()

}

func (t *Tokens) GetAllTokens() []Token {
	t.RLock()
	defer t.RUnlock()

	var r []Token
	for _, data := range t.tokens {
		data.ExpiresReadable = humanize.Time(data.ValidTo)
		r = append(r, data)
	}
	return r
}

func RandomString(n int) string {
	var letters = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}
<|endoftext|>


<commit_before>package storagenode

import (
	"fmt"
	"net/http"
	"sync/atomic"

	"github.com/jacksontj/dataman/src/query"
	"github.com/julienschmidt/httprouter"
	"github.com/xeipuuv/gojsonschema"

	"github.com/jacksontj/dataman/src/storage_node/metadata"
)

// This node is responsible for handling all of the queries for a specific storage node
// This is also responsible for maintaining schema, indexes, etc. from the metadata store
// and applying them to the actual storage subsystem
type StorageNode struct {
	Config    *Config
	MetaStore *MetadataStore

	storeSchema StorageSchemaInterface
	Store       StorageDataInterface

	meta atomic.Value
}

func NewStorageNode(config *Config) (*StorageNode, error) {

	// Create the meta store
	metaStore, err := NewMetadataStore(config)
	if err != nil {
		return nil, err
	}

	store, err := config.GetStore(metaStore.GetMeta)
	if err != nil {
		return nil, err
	}
	node := &StorageNode{
		Config:    config,
		MetaStore: metaStore,
		Store:     store,
	}

	if storeSchema, ok := store.(StorageSchemaInterface); ok {
		node.storeSchema = storeSchema
	}

	node.RefreshMeta()

	return node, nil
}

func (s *StorageNode) GetMeta() *metadata.Meta {
	return s.meta.Load().(*metadata.Meta)
}

// TODO: handle errors?
func (s *StorageNode) RefreshMeta() {
	s.meta.Store(s.MetaStore.GetMeta())
}

// TODO: have a stop?
func (s *StorageNode) Start() error {
	// initialize the http api (since at this point we are ready to go!
	router := httprouter.New()
	api := NewHTTPApi(s)
	api.Start(router)

	return http.ListenAndServe(s.Config.HTTP.Addr, router)
}

// TODO: switch this to the query.Query struct? If not then we should probably support both query formats? Or remove that Query struct
func (s *StorageNode) HandleQuery(q map[query.QueryType]query.QueryArgs) *query.Result {
	return s.HandleQueries([]map[query.QueryType]query.QueryArgs{q})[0]
}

func (s *StorageNode) HandleQueries(queries []map[query.QueryType]query.QueryArgs) []*query.Result {
	// TODO: we should actually do these in parallel (potentially with some
	// config of *how* parallel)
	results := make([]*query.Result, len(queries))

	// We specifically want to load this once for the batch so we don't have mixed
	// schema information across this batch of queries
	meta := s.MetaStore.GetMeta()

QUERYLOOP:
	for i, queryMap := range queries {
		// We only allow a single method to be defined per item
		if len(queryMap) == 1 {
			for queryType, queryArgs := range queryMap {
				collection, err := meta.GetCollection(queryArgs["db"].(string), queryArgs["collection"].(string))
				// Verify that the table is within our domain
				if err != nil {
					results[i] = &query.Result{
						Error: err.Error(),
					}
					continue
				}

				// If this is a write operation, do whatever schema validation is necessary
				switch queryType {
				case query.Set:
					fallthrough
				case query.Insert:
					fallthrough
				case query.Update:
					// On set, if there is a schema on the table-- enforce the schema
					for name, data := range queryArgs["record"].(map[string]interface{}) {
						// TODO: some datastores can actually do the enforcement on their own. We
						// probably want to leave this up to lower layers, and provide some wrapper
						// that they can call if they can't do it in the datastore itself
						if field, ok := collection.FieldMap[name]; ok && field.Schema != nil {
							result, err := field.Schema.Gschema.Validate(gojsonschema.NewGoLoader(data))
							if err != nil {
								results[i] = &query.Result{Error: err.Error()}
								continue QUERYLOOP
							}
							if !result.Valid() {
								var validationErrors string
								for _, e := range result.Errors() {
									validationErrors += "\n" + e.String()
								}
								results[i] = &query.Result{Error: "data doesn't match table schema" + validationErrors}
								continue QUERYLOOP
							}
						}
					}
				}

				// This will need to get more complex as we support multiple
				// storage interfaces
				switch queryType {
				case query.Get:
					results[i] = s.Store.Get(queryArgs)
				case query.Set:
					results[i] = s.Store.Set(queryArgs)
				case query.Insert:
					results[i] = s.Store.Insert(queryArgs)
				case query.Update:
					results[i] = s.Store.Update(queryArgs)
				case query.Delete:
					results[i] = s.Store.Delete(queryArgs)
				case query.Filter:
					results[i] = s.Store.Filter(queryArgs)
				default:
					results[i] = &query.Result{
						Error: "Unsupported query type " + string(queryType),
					}
				}
			}

		} else {
			results[i] = &query.Result{
				Error: fmt.Sprintf("Only one QueryType supported per query: %v -- %v", queryMap, queries),
			}
		}
	}
	return results
}

// TODO: schema management changes here
func (s *StorageNode) AddDatabase(db *metadata.Database) error {
	if s.storeSchema == nil {
		return fmt.Errorf("store doesn't support schema modification")
	}

	// Add the database in the store
	if err := s.storeSchema.AddDatabase(db); err != nil {
		return err
	}
	// Add it in the meta
	if err := s.MetaStore.AddDatabase(db); err != nil {
		return err
	}

	// Refresh the metadata
	s.RefreshMeta()

	return nil
}

func (s *StorageNode) RemoveDatabase(dbname string) error {
	if s.storeSchema == nil {
		return fmt.Errorf("store doesn't support schema modification")
	}

	// Remove from meta
	if err := s.MetaStore.RemoveDatabase(dbname); err != nil {
		return err
	}
	// Refresh the metadata
	s.RefreshMeta()
	// Remove from the datastore
	if err := s.storeSchema.RemoveDatabase(dbname); err != nil {
		return err
	}

	return nil
}

// TODO: to-implement
func (s *StorageNode) AddCollection(dbname string, collection *metadata.Collection) error { return nil }
func (s *StorageNode) UpdateCollection(dbname string, collection *metadata.Collection) error {
	return nil
}
func (s *StorageNode) RemoveCollection(dbname, collectionname string) error { return nil }

// TODO: move add/get/set schema stuff here (to allow for config contol
<commit_msg>Use the cached atomic one, don't load it every request<commit_after>package storagenode

import (
	"fmt"
	"net/http"
	"sync/atomic"

	"github.com/jacksontj/dataman/src/query"
	"github.com/julienschmidt/httprouter"
	"github.com/xeipuuv/gojsonschema"

	"github.com/jacksontj/dataman/src/storage_node/metadata"
)

// This node is responsible for handling all of the queries for a specific storage node
// This is also responsible for maintaining schema, indexes, etc. from the metadata store
// and applying them to the actual storage subsystem
type StorageNode struct {
	Config    *Config
	MetaStore *MetadataStore

	storeSchema StorageSchemaInterface
	Store       StorageDataInterface

	meta atomic.Value
}

func NewStorageNode(config *Config) (*StorageNode, error) {

	// Create the meta store
	metaStore, err := NewMetadataStore(config)
	if err != nil {
		return nil, err
	}

	store, err := config.GetStore(metaStore.GetMeta)
	if err != nil {
		return nil, err
	}
	node := &StorageNode{
		Config:    config,
		MetaStore: metaStore,
		Store:     store,
	}

	if storeSchema, ok := store.(StorageSchemaInterface); ok {
		node.storeSchema = storeSchema
	}

	node.RefreshMeta()

	return node, nil
}

func (s *StorageNode) GetMeta() *metadata.Meta {
	return s.meta.Load().(*metadata.Meta)
}

// TODO: handle errors?
func (s *StorageNode) RefreshMeta() {
	s.meta.Store(s.MetaStore.GetMeta())
}

// TODO: have a stop?
func (s *StorageNode) Start() error {
	// initialize the http api (since at this point we are ready to go!
	router := httprouter.New()
	api := NewHTTPApi(s)
	api.Start(router)

	return http.ListenAndServe(s.Config.HTTP.Addr, router)
}

// TODO: switch this to the query.Query struct? If not then we should probably support both query formats? Or remove that Query struct
func (s *StorageNode) HandleQuery(q map[query.QueryType]query.QueryArgs) *query.Result {
	return s.HandleQueries([]map[query.QueryType]query.QueryArgs{q})[0]
}

func (s *StorageNode) HandleQueries(queries []map[query.QueryType]query.QueryArgs) []*query.Result {
	// TODO: we should actually do these in parallel (potentially with some
	// config of *how* parallel)
	results := make([]*query.Result, len(queries))

	// We specifically want to load this once for the batch so we don't have mixed
	// schema information across this batch of queries
	meta := s.GetMeta()

QUERYLOOP:
	for i, queryMap := range queries {
		// We only allow a single method to be defined per item
		if len(queryMap) == 1 {
			for queryType, queryArgs := range queryMap {
				collection, err := meta.GetCollection(queryArgs["db"].(string), queryArgs["collection"].(string))
				// Verify that the table is within our domain
				if err != nil {
					results[i] = &query.Result{
						Error: err.Error(),
					}
					continue
				}

				// If this is a write operation, do whatever schema validation is necessary
				switch queryType {
				case query.Set:
					fallthrough
				case query.Insert:
					fallthrough
				case query.Update:
					// On set, if there is a schema on the table-- enforce the schema
					for name, data := range queryArgs["record"].(map[string]interface{}) {
						// TODO: some datastores can actually do the enforcement on their own. We
						// probably want to leave this up to lower layers, and provide some wrapper
						// that they can call if they can't do it in the datastore itself
						if field, ok := collection.FieldMap[name]; ok && field.Schema != nil {
							result, err := field.Schema.Gschema.Validate(gojsonschema.NewGoLoader(data))
							if err != nil {
								results[i] = &query.Result{Error: err.Error()}
								continue QUERYLOOP
							}
							if !result.Valid() {
								var validationErrors string
								for _, e := range result.Errors() {
									validationErrors += "\n" + e.String()
								}
								results[i] = &query.Result{Error: "data doesn't match table schema" + validationErrors}
								continue QUERYLOOP
							}
						}
					}
				}

				// This will need to get more complex as we support multiple
				// storage interfaces
				switch queryType {
				case query.Get:
					results[i] = s.Store.Get(queryArgs)
				case query.Set:
					results[i] = s.Store.Set(queryArgs)
				case query.Insert:
					results[i] = s.Store.Insert(queryArgs)
				case query.Update:
					results[i] = s.Store.Update(queryArgs)
				case query.Delete:
					results[i] = s.Store.Delete(queryArgs)
				case query.Filter:
					results[i] = s.Store.Filter(queryArgs)
				default:
					results[i] = &query.Result{
						Error: "Unsupported query type " + string(queryType),
					}
				}
			}

		} else {
			results[i] = &query.Result{
				Error: fmt.Sprintf("Only one QueryType supported per query: %v -- %v", queryMap, queries),
			}
		}
	}
	return results
}

// TODO: schema management changes here
func (s *StorageNode) AddDatabase(db *metadata.Database) error {
	if s.storeSchema == nil {
		return fmt.Errorf("store doesn't support schema modification")
	}

	// Add the database in the store
	if err := s.storeSchema.AddDatabase(db); err != nil {
		return err
	}
	// Add it in the meta
	if err := s.MetaStore.AddDatabase(db); err != nil {
		return err
	}

	// Refresh the metadata
	s.RefreshMeta()

	return nil
}

func (s *StorageNode) RemoveDatabase(dbname string) error {
	if s.storeSchema == nil {
		return fmt.Errorf("store doesn't support schema modification")
	}

	// Remove from meta
	if err := s.MetaStore.RemoveDatabase(dbname); err != nil {
		return err
	}
	// Refresh the metadata
	s.RefreshMeta()
	// Remove from the datastore
	if err := s.storeSchema.RemoveDatabase(dbname); err != nil {
		return err
	}

	return nil
}

// TODO: to-implement
func (s *StorageNode) AddCollection(dbname string, collection *metadata.Collection) error { return nil }
func (s *StorageNode) UpdateCollection(dbname string, collection *metadata.Collection) error {
	return nil
}
func (s *StorageNode) RemoveCollection(dbname, collectionname string) error { return nil }

// TODO: move add/get/set schema stuff here (to allow for config contol
<|endoftext|>


<commit_before>package stackplan

import (
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"time"

	"koding/db/models"
	"koding/db/mongodb"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/stackstate"
	"koding/kites/kloud/utils/object"

	"github.com/hashicorp/go-multierror"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/yaml.v2"
)

var defaultDatabase Database = &mongoDatabase{
	mongo: modelhelper.Mongo,
}

// mongoDatabase provides an implementation for the Database interface.
//
// It assumes modelhelper.Init was called before calling any of its methods.
type mongoDatabase struct {
	mongo *mongodb.MongoDB
}

var _ Database = (*mongoDatabase)(nil)

// Detach implements the Database interface.
func (db *mongoDatabase) Detach(opts *DetachOptions) error {
	const detachReason = "Stack destroy requested."

	if err := opts.Valid(); err != nil {
		return err
	}

	// 1) Detach stack from user. Failure is critical, as upon return
	//    a user would not be able to create new stack.

	detachStack := modelhelper.Selector{
		"targetName": "JStackTemplate",
		"targetId":   opts.Stack.BaseStackId,
		"sourceName": "JAccount",
		"sourceId":   opts.Stack.OriginId,
		"as":         "user",
	}

	err := modelhelper.DeleteRelationships(detachStack)
	if err != nil && err != mgo.ErrNotFound {
		return err
	}

	// 2) Set stack to "destroying" state.
	id := opts.Stack.Id.Hex()

	err = modelhelper.SetStackState(id, detachReason, stackstate.Destroying)
	if err != nil && err != mgo.ErrNotFound {
		// Stack state update failure is not critical, as jComputeStack
		// is going to be removed either way at the end of destroy op.
		opts.Log.Error("unable to set stack state to %q", stackstate.Destroying)
	}

	// 3) Update counters.

	err = modelhelper.DecrementOrCreateCounter(opts.Stack.Group, modelhelper.CounterStacks, 1)
	if err != nil {
		// Counter update is not crucial, nevertheless we log an error
		// if updating failed for whatever reason.
		opts.Log.Error("failure updating %q counter", modelhelper.CounterStacks)
	}

	err = modelhelper.DecrementOrCreateCounter(opts.Stack.Group, modelhelper.CounterInstances, len(opts.Stack.Machines))
	if err != nil {
		// Counter update is not crucial, nevertheless we log an error
		// if updating failed for whatever reason.
		opts.Log.Error("failure updating %q counter", modelhelper.CounterInstances)
	}

	// 4) Detach machines from user.

	detachMachines := bson.M{
		"$set": bson.M{
			"status.state": "Terminated",
			"users":        []interface{}{},
		},
	}

	err = modelhelper.UpdateMachines(detachMachines, opts.Stack.Machines...)
	if err != nil && err != mgo.ErrNotFound {
		// Detaching users from machines error is not critical, as the jMachine
		// documents are going to be deleted at the end of destroy operation.
		// Nevertheless we log error in case a troubleshooting would be needed.
		opts.Log.Error("detaching users from machines failed: %s", err)
	}

	return nil
}

// Destroy implements the Database interface.
func (db *mongoDatabase) Destroy(opts *DestroyOptions) error {
	if err := opts.Valid(); err != nil {
		return err
	}

	err := new(multierror.Error)

	for _, id := range opts.Stack.Machines {
		if e := modelhelper.DeleteMachine(id); e != nil {
			err = multierror.Append(err, e)
		}
	}

	if e := modelhelper.DeleteComputeStack(opts.Stack.Id.Hex()); e != nil {
		err = multierror.Append(err, e)
	}

	return err.ErrorOrNil()
}

var migrationBuilder = &object.Builder{
	Tag:       "bson",
	Sep:       ".",
	Prefix:    "meta.migration",
	Recursive: true,
}

var machineBuilder = &object.Builder{
	Tag:       "bson",
	Sep:       ".",
	Recursive: true,
}

// UpdateMigration implements the Database interface.
func (db *mongoDatabase) UpdateMigration(opts *UpdateMigrationOptions) error {
	change := bson.M{
		"$set": migrationBuilder.Build(opts.Meta),
	}
	return modelhelper.UpdateMachine(opts.MachineID, change)
}

// Migrate implements the Database interface.
func (db *mongoDatabase) Migrate(opts *MigrateOptions) error {
	stack := models.NewStackTemplate(opts.Provider, opts.Identifier)
	stack.Machines = make([]bson.M, len(opts.Machines))

	for i := range stack.Machines {
		stack.Machines[i] = bson.M(machineBuilder.Build(opts.Machines[i]))
	}

	account, err := modelhelper.GetAccount(opts.Username)
	if err != nil {
		return fmt.Errorf("account lookup failed for %q: %s", opts.Username, err)
	}

	sum := sha1.Sum([]byte(opts.Template))

	stack.Title = opts.StackName
	stack.OriginID = account.Id
	stack.Template.Details = bson.M{
		"lastUpdaterId": account.Id,
	}
	stack.Group = opts.GroupName
	stack.Template.Content = opts.Template
	stack.Template.Sum = hex.EncodeToString(sum[:])

	if s, err := yamlReencode(opts.Template); err == nil {
		stack.Template.RawContent = s
	}

	if err := modelhelper.CreateStackTemplate(stack); err != nil {
		return fmt.Errorf("failed to create stack template: %s", err)
	}

	change := bson.M{
		"meta.migration.modifiedAt":      time.Now(),
		"meta.migration.status":          MigrationMigrated,
		"meta.migration.stackTemplateId": stack.Id,
	}

	for _, id := range opts.MachineIDs {
		if e := modelhelper.UpdateMachine(id, change); e != nil {
			err = multierror.Append(err, fmt.Errorf("failed to update migration details for %q: %s", id.Hex(), err))
		}
	}

	// Failure updating jMachine migration metadata is not critical,
	// just log the error and continue.
	if err != nil {
		opts.Log.Error("%s", err)
	}

	return nil
}

func yamlReencode(template string) (string, error) {
	var m map[string]interface{}

	if err := json.Unmarshal([]byte(template), &m); err != nil {
		return "", err
	}

	p, err := yaml.Marshal(m)
	if err != nil {
		return "", err
	}

	return string(p), nil
}
<commit_msg>stackplan: fix final migration update<commit_after>package stackplan

import (
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"time"

	"koding/db/models"
	"koding/db/mongodb"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/stackstate"
	"koding/kites/kloud/utils/object"

	"github.com/hashicorp/go-multierror"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/yaml.v2"
)

var defaultDatabase Database = &mongoDatabase{
	mongo: modelhelper.Mongo,
}

// mongoDatabase provides an implementation for the Database interface.
//
// It assumes modelhelper.Init was called before calling any of its methods.
type mongoDatabase struct {
	mongo *mongodb.MongoDB
}

var _ Database = (*mongoDatabase)(nil)

// Detach implements the Database interface.
func (db *mongoDatabase) Detach(opts *DetachOptions) error {
	const detachReason = "Stack destroy requested."

	if err := opts.Valid(); err != nil {
		return err
	}

	// 1) Detach stack from user. Failure is critical, as upon return
	//    a user would not be able to create new stack.

	detachStack := modelhelper.Selector{
		"targetName": "JStackTemplate",
		"targetId":   opts.Stack.BaseStackId,
		"sourceName": "JAccount",
		"sourceId":   opts.Stack.OriginId,
		"as":         "user",
	}

	err := modelhelper.DeleteRelationships(detachStack)
	if err != nil && err != mgo.ErrNotFound {
		return err
	}

	// 2) Set stack to "destroying" state.
	id := opts.Stack.Id.Hex()

	err = modelhelper.SetStackState(id, detachReason, stackstate.Destroying)
	if err != nil && err != mgo.ErrNotFound {
		// Stack state update failure is not critical, as jComputeStack
		// is going to be removed either way at the end of destroy op.
		opts.Log.Error("unable to set stack state to %q", stackstate.Destroying)
	}

	// 3) Update counters.

	err = modelhelper.DecrementOrCreateCounter(opts.Stack.Group, modelhelper.CounterStacks, 1)
	if err != nil {
		// Counter update is not crucial, nevertheless we log an error
		// if updating failed for whatever reason.
		opts.Log.Error("failure updating %q counter", modelhelper.CounterStacks)
	}

	err = modelhelper.DecrementOrCreateCounter(opts.Stack.Group, modelhelper.CounterInstances, len(opts.Stack.Machines))
	if err != nil {
		// Counter update is not crucial, nevertheless we log an error
		// if updating failed for whatever reason.
		opts.Log.Error("failure updating %q counter", modelhelper.CounterInstances)
	}

	// 4) Detach machines from user.

	detachMachines := bson.M{
		"$set": bson.M{
			"status.state": "Terminated",
			"users":        []interface{}{},
		},
	}

	err = modelhelper.UpdateMachines(detachMachines, opts.Stack.Machines...)
	if err != nil && err != mgo.ErrNotFound {
		// Detaching users from machines error is not critical, as the jMachine
		// documents are going to be deleted at the end of destroy operation.
		// Nevertheless we log error in case a troubleshooting would be needed.
		opts.Log.Error("detaching users from machines failed: %s", err)
	}

	return nil
}

// Destroy implements the Database interface.
func (db *mongoDatabase) Destroy(opts *DestroyOptions) error {
	if err := opts.Valid(); err != nil {
		return err
	}

	err := new(multierror.Error)

	for _, id := range opts.Stack.Machines {
		if e := modelhelper.DeleteMachine(id); e != nil {
			err = multierror.Append(err, e)
		}
	}

	if e := modelhelper.DeleteComputeStack(opts.Stack.Id.Hex()); e != nil {
		err = multierror.Append(err, e)
	}

	return err.ErrorOrNil()
}

var migrationBuilder = &object.Builder{
	Tag:       "bson",
	Sep:       ".",
	Prefix:    "meta.migration",
	Recursive: true,
}

var machineBuilder = &object.Builder{
	Tag:       "bson",
	Sep:       ".",
	Recursive: true,
}

// UpdateMigration implements the Database interface.
func (db *mongoDatabase) UpdateMigration(opts *UpdateMigrationOptions) error {
	change := bson.M{
		"$set": migrationBuilder.Build(opts.Meta),
	}
	return modelhelper.UpdateMachine(opts.MachineID, change)
}

// Migrate implements the Database interface.
func (db *mongoDatabase) Migrate(opts *MigrateOptions) error {
	stack := models.NewStackTemplate(opts.Provider, opts.Identifier)
	stack.Machines = make([]bson.M, len(opts.Machines))

	for i := range stack.Machines {
		stack.Machines[i] = bson.M(machineBuilder.Build(opts.Machines[i]))
	}

	account, err := modelhelper.GetAccount(opts.Username)
	if err != nil {
		return fmt.Errorf("account lookup failed for %q: %s", opts.Username, err)
	}

	sum := sha1.Sum([]byte(opts.Template))

	stack.Title = opts.StackName
	stack.OriginID = account.Id
	stack.Template.Details = bson.M{
		"lastUpdaterId": account.Id,
	}
	stack.Group = opts.GroupName
	stack.Template.Content = opts.Template
	stack.Template.Sum = hex.EncodeToString(sum[:])

	if s, err := yamlReencode(opts.Template); err == nil {
		stack.Template.RawContent = s
	}

	if err := modelhelper.CreateStackTemplate(stack); err != nil {
		return fmt.Errorf("failed to create stack template: %s", err)
	}

	change := bson.M{
		"$set": bson.M{
			"meta.migration.modifiedAt":      time.Now(),
			"meta.migration.status":          MigrationMigrated,
			"meta.migration.stackTemplateId": stack.Id,
		},
	}

	for _, id := range opts.MachineIDs {
		if e := modelhelper.UpdateMachine(id, change); e != nil {
			err = multierror.Append(err, fmt.Errorf("failed to update migration details for %q: %s", id.Hex(), err))
		}
	}

	// Failure updating jMachine migration metadata is not critical,
	// just log the error and continue.
	if err != nil {
		opts.Log.Error("%s", err)
	}

	return nil
}

func yamlReencode(template string) (string, error) {
	var m map[string]interface{}

	if err := json.Unmarshal([]byte(template), &m); err != nil {
		return "", err
	}

	p, err := yaml.Marshal(m)
	if err != nil {
		return "", err
	}

	return string(p), nil
}
<|endoftext|>


<commit_before>package container551

import (
	"errors"
	"github.com/go51/auth551"
	"github.com/go51/cookie551"
	"github.com/go51/log551"
	"github.com/go51/memcache551"
	"github.com/go51/model551"
	"github.com/go51/mysql551"
	"github.com/go51/repository551"
	"github.com/go51/secure551"
	"github.com/go51/string551"
	"net/http"
	"strconv"
)

type urlFunc func(name string, parameter ...string) string

type Container struct {
	sid         string
	ssid        string
	w           http.ResponseWriter
	r           *http.Request
	logger      *log551.Log551
	cookie      *cookie551.Cookie
	db          *mysql551.Mysql
	session     *memcache551.Memcache
	model       *model551.Model
	auth        *auth551.Auth
	user        *auth551.UserModel
	options     map[string]string
	urlFunction urlFunc
}

func New() *Container {
	return &Container{}
}

func (c *Container) SetSID(sid string) {
	c.sid = sid
	c.ssid = sid[:10]
}

func (c *Container) SID() string {
	return c.sid
}
func (c *Container) SSID() string {
	return c.ssid
}

func (c *Container) SetResponseWriter(w http.ResponseWriter) {
	c.w = w
}

func (c *Container) ResponseWriter() http.ResponseWriter {
	return c.w
}

func (c *Container) SetRequest(r *http.Request) {
	c.r = r
}

func (c *Container) Request() *http.Request {
	return c.r
}

func (c *Container) SetLogger(logger *log551.Log551) {
	c.logger = logger
}

func (c *Container) Logger() *log551.Log551 {
	return c.logger
}

func (c *Container) SetCookie(cookie *cookie551.Cookie) {
	c.cookie = cookie
}

func (c *Container) Cookie() *cookie551.Cookie {
	return c.cookie
}

func (c *Container) SetDb(db *mysql551.Mysql) {
	c.db = db
}

func (c *Container) Db() *mysql551.Mysql {
	return c.db
}

func (c *Container) SetSession(session *memcache551.Memcache) {
	c.session = session
}

func (c *Container) Session() *memcache551.Memcache {
	return c.session
}

func (c *Container) SetModel(modelManager *model551.Model) {
	c.model = modelManager
}

func (c *Container) ModelManager() *model551.Model {
	return c.model
}

func (c *Container) SetAuth(auth *auth551.Auth) {
	c.auth = auth

	if c.user != nil {
		return
	}

	// Load user from session
	c.session.GetModel("reminder_user", &c.user)

	// Get user id from cookie
	id, err := c.getRemindId()
	if err != nil {
		return
	}

	// Get user model from database
	c.user = c.getUser(id)

	// Set user model to session
	c.session.Set("reminder_user", c.user)

	return

}

func (c *Container) getRemindId() (int64, error) {
	cookieId, err := c.cookie.Get(c.auth.CookieKeyName())
	if err != nil {
		return 0, err
	}

	sid := secure551.Decrypted(cookieId, c.auth.MasterKey())
	id, err := strconv.ParseInt(sid, 10, 64)
	if err != nil {
		return 0, err
	}

	return id, nil

}

func (c *Container) getUser(id int64) *auth551.UserModel {
	repo := repository551.Load()
	miUser := c.ModelManager().Get("UserModel")
	mUser := repo.Find(c.db, miUser, id)
	user, ok := mUser.(*auth551.UserModel)
	if !ok {
		return nil
	}

	return user
}

func (c *Container) Auth() *auth551.Auth {
	return c.auth
}

func (c *Container) SignIn(user *auth551.UserModel) {
	// Set remind id to cookie
	id := string551.Right("0000000000000000"+strconv.FormatInt(user.Id, 10), 16)
	secureId := secure551.Encrypted(id, c.auth.MasterKey())
	c.cookie.Set(c.auth.CookieKeyName(), secureId, 60*60*24*365)

	// Set user model to session
	c.session.Set("reminder_user", user)

}

func (c *Container) SignOut() {
	c.cookie.Delete(c.auth.CookieKeyName())
	c.session.Delete("reminder_user")
}

func (c *Container) IsSignIn() bool {
	return c.user != nil
}

func (c *Container) User() *auth551.UserModel {
	return c.user
}

func (c *Container) SetCommandOptions(options map[string]string) {
	c.options = options
}

func (c *Container) CommandOption(name string) string {
	return c.options[name]
}

func (c *Container) Segment(number int) string {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return ""
	}
	return paths[number]
}

func (c *Container) SegmentInt64(number int) (int64, error) {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return 0, errors.New("invalid memory address or nil pointer dereference")
	}
	segment := paths[number]

	return strconv.ParseInt(segment, 10, 64)

}

func (c *Container) SetUrlFunc(urlFunction urlFunc) {
	c.urlFunction = urlFunction
}

func (c *Container) URL(name string, parameter ...string) string {
	return c.urlFunction(name, parameter...)
}
<commit_msg>refs #33 インスタンス内の *auth551.UserModel を更新できるメソッドを実装する<commit_after>package container551

import (
	"errors"
	"github.com/go51/auth551"
	"github.com/go51/cookie551"
	"github.com/go51/log551"
	"github.com/go51/memcache551"
	"github.com/go51/model551"
	"github.com/go51/mysql551"
	"github.com/go51/repository551"
	"github.com/go51/secure551"
	"github.com/go51/string551"
	"net/http"
	"strconv"
)

type urlFunc func(name string, parameter ...string) string

type Container struct {
	sid         string
	ssid        string
	w           http.ResponseWriter
	r           *http.Request
	logger      *log551.Log551
	cookie      *cookie551.Cookie
	db          *mysql551.Mysql
	session     *memcache551.Memcache
	model       *model551.Model
	auth        *auth551.Auth
	user        *auth551.UserModel
	options     map[string]string
	urlFunction urlFunc
}

func New() *Container {
	return &Container{}
}

func (c *Container) SetSID(sid string) {
	c.sid = sid
	c.ssid = sid[:10]
}

func (c *Container) SID() string {
	return c.sid
}
func (c *Container) SSID() string {
	return c.ssid
}

func (c *Container) SetResponseWriter(w http.ResponseWriter) {
	c.w = w
}

func (c *Container) ResponseWriter() http.ResponseWriter {
	return c.w
}

func (c *Container) SetRequest(r *http.Request) {
	c.r = r
}

func (c *Container) Request() *http.Request {
	return c.r
}

func (c *Container) SetLogger(logger *log551.Log551) {
	c.logger = logger
}

func (c *Container) Logger() *log551.Log551 {
	return c.logger
}

func (c *Container) SetCookie(cookie *cookie551.Cookie) {
	c.cookie = cookie
}

func (c *Container) Cookie() *cookie551.Cookie {
	return c.cookie
}

func (c *Container) SetDb(db *mysql551.Mysql) {
	c.db = db
}

func (c *Container) Db() *mysql551.Mysql {
	return c.db
}

func (c *Container) SetSession(session *memcache551.Memcache) {
	c.session = session
}

func (c *Container) Session() *memcache551.Memcache {
	return c.session
}

func (c *Container) SetModel(modelManager *model551.Model) {
	c.model = modelManager
}

func (c *Container) ModelManager() *model551.Model {
	return c.model
}

func (c *Container) SetAuth(auth *auth551.Auth) {
	c.auth = auth

	if c.user != nil {
		return
	}

	// Load user from session
	c.session.GetModel("reminder_user", &c.user)

	// Get user id from cookie
	id, err := c.getRemindId()
	if err != nil {
		return
	}

	// Get user model from database
	c.user = c.getUser(id)

	// Set user model to session
	c.session.Set("reminder_user", c.user)

	return

}

func (c *Container) getRemindId() (int64, error) {
	cookieId, err := c.cookie.Get(c.auth.CookieKeyName())
	if err != nil {
		return 0, err
	}

	sid := secure551.Decrypted(cookieId, c.auth.MasterKey())
	id, err := strconv.ParseInt(sid, 10, 64)
	if err != nil {
		return 0, err
	}

	return id, nil

}

func (c *Container) getUser(id int64) *auth551.UserModel {
	repo := repository551.Load()
	miUser := c.ModelManager().Get("UserModel")
	mUser := repo.Find(c.db, miUser, id)
	user, ok := mUser.(*auth551.UserModel)
	if !ok {
		return nil
	}

	return user
}

func (c *Container) Auth() *auth551.Auth {
	return c.auth
}

func (c *Container) SignIn(user *auth551.UserModel) {
	// Set remind id to cookie
	id := string551.Right("0000000000000000"+strconv.FormatInt(user.Id, 10), 16)
	secureId := secure551.Encrypted(id, c.auth.MasterKey())
	c.cookie.Set(c.auth.CookieKeyName(), secureId, 60*60*24*365)

	// Set user model to session
	c.session.Set("reminder_user", user)

}

func (c *Container) SignOut() {
	c.cookie.Delete(c.auth.CookieKeyName())
	c.session.Delete("reminder_user")
}

func (c *Container) IsSignIn() bool {
	return c.user != nil
}

func (c *Container) User() *auth551.UserModel {
	return c.user
}

func (c *Container) UpdateUser(user *auth551.UserModel) {
	c.user = user
}

func (c *Container) SetCommandOptions(options map[string]string) {
	c.options = options
}

func (c *Container) CommandOption(name string) string {
	return c.options[name]
}

func (c *Container) Segment(number int) string {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return ""
	}
	return paths[number]
}

func (c *Container) SegmentInt64(number int) (int64, error) {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return 0, errors.New("invalid memory address or nil pointer dereference")
	}
	segment := paths[number]

	return strconv.ParseInt(segment, 10, 64)

}

func (c *Container) SetUrlFunc(urlFunction urlFunc) {
	c.urlFunction = urlFunction
}

func (c *Container) URL(name string, parameter ...string) string {
	return c.urlFunction(name, parameter...)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/user"
	"strconv"
	"syscall"

	"github.com/ondevice/ondevice/command"
	"github.com/ondevice/ondevice/config"
	"github.com/ondevice/ondevice/logg"
)

func main() {
	if os.Getuid() == 0 {
		// running as root -> setup the files we need and drop privileges
		uid := _dropPrivileges()
		if err := syscall.Setuid(int(uid)); err != nil {
			logg.Fatal("Failed to drop privileges: ", err)
		}
	}

	if len(os.Args) < 2 {
		logg.Fatal("Missing command! try 'ondevice help'")
	}

	//logg.Debug("-- args: ", os.Args[1:])
	cmd := os.Args[1]
	rc := command.Run(cmd, os.Args[2:])
	os.Exit(rc)
}

func _dropPrivileges() int {
	// first see if there's an 'ondevice' user account
	u, err := user.Lookup("ondevice")
	if err != nil {
		logg.Fatal("Can't run as root - and couldn't find 'ondevice' user")
	}

	// get uid
	uid, err := strconv.ParseInt(u.Uid, 10, 32)
	if err != nil {
		logg.Fatal("Couldn't convert uid string: ", u.Uid)
	}

	gid, err := strconv.ParseInt(u.Gid, 10, 32)
	if err != nil {
		logg.Fatal("Couldn't convert gid string: ", u.Gid)
	}

	// see if ondevice.conf exists
	_, err = os.Stat("/etc/ondevice/ondevice.conf")
	if os.IsNotExist(err) {
		logg.Fatal("Couldn't find /etc/ondevice/ondevice.conf")
	}

	// TODO use other paths for other OSs
	// TODO allow the user to override these paths (e.g. using environment vars or commandline flags)
	_setupFile("ondevice.pid", "/var/run/ondevice.pid", int(uid), int(gid), 0644)
	_setupFile("ondevice.sock", "/var/run/ondevice.sock", int(uid), int(gid), 0664)
	config.SetFilePath("ondevice.conf", "/etc/ondevice/ondevice.conf")

	return int(uid)
}

func _setupFile(filename string, path string, uid int, gid int, mode os.FileMode) {
	config.SetFilePath(filename, path)
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		os.OpenFile(path, os.O_RDONLY|os.O_CREATE, mode)
	} else if err != nil {
		logg.Fatal("Couldn't get file info for ", filename, ": ", err)
	}
	err = os.Chown(path, uid, gid)
	if err != nil {
		logg.Fatalf("Couldn't set file permissions for %s to %d", filename, mode)
	}
}
<commit_msg>using /var/lib/ondevice/ondevice.conf when running as root<commit_after>package main

import (
	"os"
	"os/user"
	"strconv"
	"syscall"

	"github.com/ondevice/ondevice/command"
	"github.com/ondevice/ondevice/config"
	"github.com/ondevice/ondevice/logg"
)

func main() {
	if os.Getuid() == 0 {
		// running as root -> setup the files we need and drop privileges
		uid := _dropPrivileges()
		if err := syscall.Setuid(int(uid)); err != nil {
			logg.Fatal("Failed to drop privileges: ", err)
		}
	}

	if len(os.Args) < 2 {
		logg.Fatal("Missing command! try 'ondevice help'")
	}

	//logg.Debug("-- args: ", os.Args[1:])
	cmd := os.Args[1]
	rc := command.Run(cmd, os.Args[2:])
	os.Exit(rc)
}

func _dropPrivileges() int {
	// first see if there's an 'ondevice' user account
	u, err := user.Lookup("ondevice")
	if err != nil {
		logg.Fatal("Can't run as root - and couldn't find 'ondevice' user")
	}

	// get uid
	uid, err := strconv.ParseInt(u.Uid, 10, 32)
	if err != nil {
		logg.Fatal("Couldn't convert uid string: ", u.Uid)
	}

	gid, err := strconv.ParseInt(u.Gid, 10, 32)
	if err != nil {
		logg.Fatal("Couldn't convert gid string: ", u.Gid)
	}

	// see if ondevice.conf exists
	_, err = os.Stat("/etc/ondevice/ondevice.conf")
	if os.IsNotExist(err) {
		logg.Fatal("Couldn't find /etc/ondevice/ondevice.conf")
	}

	// TODO use other paths for other OSs
	// TODO allow the user to override these paths (e.g. using environment vars or commandline flags)
	_setupFile("ondevice.pid", "/var/run/ondevice.pid", int(uid), int(gid), 0644)
	_setupFile("ondevice.sock", "/var/run/ondevice.sock", int(uid), int(gid), 0664)
	config.SetFilePath("ondevice.conf", "/var/lib/ondevice/ondevice.conf")

	return int(uid)
}

func _setupFile(filename string, path string, uid int, gid int, mode os.FileMode) {
	config.SetFilePath(filename, path)
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		os.OpenFile(path, os.O_RDONLY|os.O_CREATE, mode)
	} else if err != nil {
		logg.Fatal("Couldn't get file info for ", filename, ": ", err)
	}
	err = os.Chown(path, uid, gid)
	if err != nil {
		logg.Fatalf("Couldn't set file permissions for %s to %d", filename, mode)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/fcgi"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
)

var (
	form url.Values
	header http.Header
	cookies []*http.Cookie
	writer http.ResponseWriter
	request http.Request
	exit_error bool
	server *GochanServer
)

type GochanServer struct{
	writer http.ResponseWriter
	request http.Request
	namespaces map[string]func(http.ResponseWriter, *http.Request, interface{})
}

func (s GochanServer) AddNamespace(base_path string, namespace_function func(http.ResponseWriter, *http.Request, interface{})) {
	s.namespaces[base_path] = namespace_function
}

func (s GochanServer) getFileData(writer http.ResponseWriter, url string) ([]byte, bool) {
	var file_bytes []byte
	filepath := path.Join(config.DocumentRoot, url)
	results,err := os.Stat(filepath)
	if err != nil {
		fmt.Println("404 at ", filepath)
		// the requested path isn't a file or directory, 404
		return file_bytes, false
	} else {
		//the file exists, or there is a folder here
		if results.IsDir() {
			found_index := false
			newpath := ""

			//check to see if one of the specified index pages exists
			for i := 0; i < len(config.FirstPage); i++ {
				newpath = path.Join(filepath,config.FirstPage[i])
				_,err := os.Stat(newpath)
				if err == nil {
					// serve the index page
					writer.Header().Add("Cache-Control", "max-age=5, must-revalidate")
					fmt.Println("found index at ", newpath)
					file_bytes,err  = ioutil.ReadFile(newpath)
					return file_bytes, true
					found_index = true
					break
				}
			}

			if !found_index {
				// none of the index pages specified in config.cfg exist
				return file_bytes, false
			}
		} else {
			//the file exists, and is not a folder
			file_bytes, err = ioutil.ReadFile(filepath)
			extension := getFileExtension(url)
			switch {
				case extension == "png":
					writer.Header().Add("Content-Type", "image/png")
					writer.Header().Add("Cache-Control", "max-age=86400")
				case extension == "gif":
					writer.Header().Add("Content-Type", "image/gif")
					writer.Header().Add("Cache-Control", "max-age=86400")
				case extension == "jpg":
					writer.Header().Add("Cache-Control", "max-age=86400")
				case extension == "css":
					writer.Header().Add("Content-Type", "text/css")
					writer.Header().Add("Cache-Control", "max-age=43200")
				case extension == "js":
					writer.Header().Add("Content-Type", "text/javascript")
					writer.Header().Add("Cache-Control", "max-age=43200")
			}
			if extension  == "html" || extension == "htm" {
				writer.Header().Add("Cache-Control", "max-age=5, must-revalidate")
			}
			//http.ServeFile(writer, request, filepath)
			access_log.Print("Success: 200 from " + request.RemoteAddr + " @ " + request.RequestURI)
			return file_bytes, true
		}
	}
	return file_bytes, false
}

func (s GochanServer) Redirect(location string) {
	http.Redirect(writer,&request,location,http.StatusFound)
}

func (s GochanServer) serve404(writer http.ResponseWriter, request *http.Request) {
	error_page, err := ioutil.ReadFile(config.DocumentRoot + "/error/404.html")
	if err != nil {
		writer.Write([]byte("Requested page not found, and 404 error page not found"))
	} else {
		writer.Write(error_page)
	}
	error_log.Print("Error: 404 Not Found from " + request.RemoteAddr + " @ " + request.RequestURI)
}

func (s GochanServer) ServeErrorPage(writer http.ResponseWriter, err string) {
	error_page_bytes,_ := ioutil.ReadFile("templates/error.html")
	error_page := string(error_page_bytes)
	error_page = strings.Replace(error_page,"{ERRORTEXT}", err,-1)
	writer.Write([]byte(error_page))
	exit_error = true
}

func (s GochanServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	for name, namespace_function := range s.namespaces {
		//if len(request.URL)
		if request.URL.Path == "/" + name {
			namespace_function(writer, request, nil)
			return
		}
	}
	fb,found := s.getFileData(writer, request.URL.Path)
	writer.Header().Add("Cache-Control", "max-age=86400")
	if !found {
		s.serve404(writer, request)
		return
	}
	writer.Write(fb)
}

func initServer() {
	listener,err := net.Listen("tcp", config.Domain+":"+strconv.Itoa(config.Port))
	if(err != nil) {
		fmt.Printf("Failed listening on "+config.Domain+":%d, see log for details",config.Port)
		error_log.Fatal(err.Error())
	}
	server = new(GochanServer)
	server.namespaces = make(map[string]func(http.ResponseWriter, *http.Request, interface{}))

	testfunc := func(writer http.ResponseWriter, response *http.Request, data interface{}) {
		if writer != nil {
			writer.Write([]byte("hahahaha"))
		}
	}
	server.AddNamespace("example", testfunc)
	server.AddNamespace("manage", callManageFunction)
	server.AddNamespace("post", makePost)
	server.AddNamespace("util", utilHandler)
	if config.UseFastCGI {
		fcgi.Serve(listener,server)
	} else {
		http.Serve(listener, server)
	}
}

func validReferrer(request http.Request) (valid bool) {
	if request.Referer() == "" || request.Referer()[7:len(config.SiteDomain)+7] != config.SiteDomain {
	// if request.Referer() == "" || request.Referer()[7:len(config.Domain)+7] != config.Domain {
		valid = false
	} else {
		valid = true
	}
	return
}

func utilHandler(writer http.ResponseWriter, request *http.Request, data interface{}) {
	action := request.FormValue("action")
	board := request.FormValue("board")

	if action == "" && request.PostFormValue("delete_btn") != "Delete" && request.PostFormValue("report_btn") != "Report" {
		http.Redirect(writer,request,path.Join(config.SiteWebfolder,"/"),http.StatusFound)
		return
	}
	var posts_arr []string
	for key,_ := range request.PostForm {
		if strings.Index(key,"check") == 0 {
			posts_arr = append(posts_arr,key[5:])
		}
	}
	if request.PostFormValue("delete_btn") == "Delete" {
		file_only := request.FormValue("fileonly") == "on"
		password := md5_sum(request.FormValue("password"))
		rank := getStaffRank()

		if request.FormValue("password") == ""  && rank == 0 {
			server.ServeErrorPage(writer, "Password required for post deletion")
			return
		}

		for _,post := range posts_arr {
			var parent_id int
			var filename string
			var filetype string
			var password_checksum string
			var board_id int
			post_int,err := strconv.Atoi(post)

			err = db.QueryRow("SELECT `parentid`,`filename`,`password` FROM `"+config.DBprefix+"posts` WHERE `id` = "+post).Scan(&parent_id,&filename,&password_checksum)
			if err == sql.ErrNoRows {
				//the post has already been deleted
				fmt.Fprintf(writer, "%s has already been deleted\n",post)
				continue
			}
			if err != nil {
				server.ServeErrorPage(writer,err.Error())
				return
			}

			err = db.QueryRow("SELECT `id` FROM `"+config.DBprefix+"boards` WHERE `dir` = '"+board+"'").Scan(&board_id)
			if err != nil {
				server.ServeErrorPage(writer,err.Error())
				return
			}

			if password != password_checksum && rank == 0 {
				fmt.Fprintf(writer, "Incorrect password for %s\n", post)
				continue
			}

			if file_only {
				if filename != "" {
					filetype = filename[strings.Index(filename,".")+1:]
					filename = filename[:strings.Index(filename,".")]
					err := os.Remove(path.Join(config.DocumentRoot,board,"/src/"+filename+"."+filetype))
					if err != nil {
						server.ServeErrorPage(writer,err.Error())
						return
					}
					err = os.Remove(path.Join(config.DocumentRoot,board,"/thumb/"+filename+"t."+filetype))
					if err != nil {
						server.ServeErrorPage(writer,err.Error())
						return
					}
					_,err = db.Exec("UPDATE `"+config.DBprefix+"posts` SET `filename` = 'deleted' WHERE `id` = "+post)
					if err != nil {
						server.ServeErrorPage(writer,err.Error())
						return
					}
				}
				fmt.Fprintf(writer, "Attached image from %s deleted successfully\n", post)
				writer.Header().Add("refresh", "5;url="+request.Referer())
			} else {
				if parent_id > 0 {
					os.Remove(path.Join(config.DocumentRoot,board,"/res/index.html"))
				}
				_,err = db.Exec("DELETE FROM `"+config.DBprefix+"posts` WHERE `id` = "+post)
				if parent_id == 0 {
					err = buildThread(post_int, board_id)
				} else {
					err = buildThread(parent_id,board_id)
				}

				if err != nil {
					server.ServeErrorPage(writer,err.Error())
					return
				}
				fmt.Fprintf(writer, "%s deleted successfully\n", post)
				writer.Header().Add("refresh", "5;url="+request.Referer())
			}
		}
	}
}
<commit_msg>mime type for jpeg images should be set to image/jpeg<commit_after>package main

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/fcgi"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
)

var (
	form url.Values
	header http.Header
	cookies []*http.Cookie
	writer http.ResponseWriter
	request http.Request
	exit_error bool
	server *GochanServer
)

type GochanServer struct{
	writer http.ResponseWriter
	request http.Request
	namespaces map[string]func(http.ResponseWriter, *http.Request, interface{})
}

func (s GochanServer) AddNamespace(base_path string, namespace_function func(http.ResponseWriter, *http.Request, interface{})) {
	s.namespaces[base_path] = namespace_function
}

func (s GochanServer) getFileData(writer http.ResponseWriter, url string) ([]byte, bool) {
	var file_bytes []byte
	filepath := path.Join(config.DocumentRoot, url)
	results,err := os.Stat(filepath)
	if err != nil {
		fmt.Println("404 at ", filepath)
		// the requested path isn't a file or directory, 404
		return file_bytes, false
	} else {
		//the file exists, or there is a folder here
		if results.IsDir() {
			found_index := false
			newpath := ""

			//check to see if one of the specified index pages exists
			for i := 0; i < len(config.FirstPage); i++ {
				newpath = path.Join(filepath,config.FirstPage[i])
				_,err := os.Stat(newpath)
				if err == nil {
					// serve the index page
					writer.Header().Add("Cache-Control", "max-age=5, must-revalidate")
					fmt.Println("found index at ", newpath)
					file_bytes,err  = ioutil.ReadFile(newpath)
					return file_bytes, true
					found_index = true
					break
				}
			}

			if !found_index {
				// none of the index pages specified in config.cfg exist
				return file_bytes, false
			}
		} else {
			//the file exists, and is not a folder
			file_bytes, err = ioutil.ReadFile(filepath)
			extension := getFileExtension(url)
			switch {
				case extension == "png":
					writer.Header().Add("Content-Type", "image/png")
					writer.Header().Add("Cache-Control", "max-age=86400")
				case extension == "gif":
					writer.Header().Add("Content-Type", "image/gif")
					writer.Header().Add("Cache-Control", "max-age=86400")
				case extension == "jpg":
					writer.Header().Add("Content-Type", "image/jpeg")
					writer.Header().Add("Cache-Control", "max-age=86400")
				case extension == "css":
					writer.Header().Add("Content-Type", "text/css")
					writer.Header().Add("Cache-Control", "max-age=43200")
				case extension == "js":
					writer.Header().Add("Content-Type", "text/javascript")
					writer.Header().Add("Cache-Control", "max-age=43200")
			}
			if extension  == "html" || extension == "htm" {
				writer.Header().Add("Cache-Control", "max-age=5, must-revalidate")
			}
			//http.ServeFile(writer, request, filepath)
			access_log.Print("Success: 200 from " + request.RemoteAddr + " @ " + request.RequestURI)
			return file_bytes, true
		}
	}
	return file_bytes, false
}

func (s GochanServer) Redirect(location string) {
	http.Redirect(writer,&request,location,http.StatusFound)
}

func (s GochanServer) serve404(writer http.ResponseWriter, request *http.Request) {
	error_page, err := ioutil.ReadFile(config.DocumentRoot + "/error/404.html")
	if err != nil {
		writer.Write([]byte("Requested page not found, and 404 error page not found"))
	} else {
		writer.Write(error_page)
	}
	error_log.Print("Error: 404 Not Found from " + request.RemoteAddr + " @ " + request.RequestURI)
}

func (s GochanServer) ServeErrorPage(writer http.ResponseWriter, err string) {
	error_page_bytes,_ := ioutil.ReadFile("templates/error.html")
	error_page := string(error_page_bytes)
	error_page = strings.Replace(error_page,"{ERRORTEXT}", err,-1)
	writer.Write([]byte(error_page))
	exit_error = true
}

func (s GochanServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	for name, namespace_function := range s.namespaces {
		//if len(request.URL)
		if request.URL.Path == "/" + name {
			namespace_function(writer, request, nil)
			return
		}
	}
	fb,found := s.getFileData(writer, request.URL.Path)
	writer.Header().Add("Cache-Control", "max-age=86400")
	if !found {
		s.serve404(writer, request)
		return
	}
	writer.Write(fb)
}

func initServer() {
	listener,err := net.Listen("tcp", config.Domain+":"+strconv.Itoa(config.Port))
	if(err != nil) {
		fmt.Printf("Failed listening on "+config.Domain+":%d, see log for details",config.Port)
		error_log.Fatal(err.Error())
	}
	server = new(GochanServer)
	server.namespaces = make(map[string]func(http.ResponseWriter, *http.Request, interface{}))

	testfunc := func(writer http.ResponseWriter, response *http.Request, data interface{}) {
		if writer != nil {
			writer.Write([]byte("hahahaha"))
		}
	}
	server.AddNamespace("example", testfunc)
	server.AddNamespace("manage", callManageFunction)
	server.AddNamespace("post", makePost)
	server.AddNamespace("util", utilHandler)
	if config.UseFastCGI {
		fcgi.Serve(listener,server)
	} else {
		http.Serve(listener, server)
	}
}

func validReferrer(request http.Request) (valid bool) {
	if request.Referer() == "" || request.Referer()[7:len(config.SiteDomain)+7] != config.SiteDomain {
	// if request.Referer() == "" || request.Referer()[7:len(config.Domain)+7] != config.Domain {
		valid = false
	} else {
		valid = true
	}
	return
}

func utilHandler(writer http.ResponseWriter, request *http.Request, data interface{}) {
	action := request.FormValue("action")
	board := request.FormValue("board")
	if action == "" && request.PostFormValue("delete_btn") != "Delete" && request.PostFormValue("report_btn") != "Report" {
		http.Redirect(writer,request,path.Join(config.SiteWebfolder,"/"),http.StatusFound)
		return
	}
	var posts_arr []string
	for key,_ := range request.PostForm {
		if strings.Index(key,"check") == 0 {
			posts_arr = append(posts_arr,key[5:])
		}
	}
	if request.PostFormValue("delete_btn") == "Delete" {
		file_only := request.FormValue("fileonly") == "on"
		password := md5_sum(request.FormValue("password"))
		rank := getStaffRank()

		if request.FormValue("password") == ""  && rank == 0 {
			server.ServeErrorPage(writer, "Password required for post deletion")
			return
		}

		for _,post := range posts_arr {
			var parent_id int
			var filename string
			var filetype string
			var password_checksum string
			var board_id int
			post_int,err := strconv.Atoi(post)

			err = db.QueryRow("SELECT `parentid`,`filename`,`password` FROM `"+config.DBprefix+"posts` WHERE `id` = "+post).Scan(&parent_id,&filename,&password_checksum)
			if err == sql.ErrNoRows {
				//the post has already been deleted
				fmt.Fprintf(writer, "%s has already been deleted\n",post)
				continue
			}
			if err != nil {
				server.ServeErrorPage(writer,err.Error())
				return
			}

			err = db.QueryRow("SELECT `id` FROM `"+config.DBprefix+"boards` WHERE `dir` = '"+board+"'").Scan(&board_id)
			if err != nil {
				server.ServeErrorPage(writer,err.Error())
				return
			}

			if password != password_checksum && rank == 0 {
				fmt.Fprintf(writer, "Incorrect password for %s\n", post)
				continue
			}

			if file_only {
				if filename != "" {
					filetype = filename[strings.Index(filename,".")+1:]
					filename = filename[:strings.Index(filename,".")]
					err := os.Remove(path.Join(config.DocumentRoot,board,"/src/"+filename+"."+filetype))
					if err != nil {
						server.ServeErrorPage(writer,err.Error())
						return
					}
					err = os.Remove(path.Join(config.DocumentRoot,board,"/thumb/"+filename+"t."+filetype))
					if err != nil {
						server.ServeErrorPage(writer,err.Error())
						return
					}
					_,err = db.Exec("UPDATE `"+config.DBprefix+"posts` SET `filename` = 'deleted' WHERE `id` = "+post)
					if err != nil {
						server.ServeErrorPage(writer,err.Error())
						return
					}
				}
				fmt.Fprintf(writer, "Attached image from %s deleted successfully<br />\n<meta http-equiv=\"refresh\" content=\"1;url=http://lunachan.net/test/\">", post)
			} else {
				if parent_id > 0 {
					os.Remove(path.Join(config.DocumentRoot,board,"/res/index.html"))
				}
				_,err = db.Exec("DELETE FROM `"+config.DBprefix+"posts` WHERE `id` = "+post)
				if parent_id == 0 {
					err = buildThread(post_int, board_id)
				} else {
					err = buildThread(parent_id,board_id)
				}

				if err != nil {
					server.ServeErrorPage(writer,err.Error())
					return
				}
				fmt.Fprintf(writer, "%s deleted successfully\n", post)
				writer.Header().Add("refresh", "5;url="+request.Referer())
			}
		}
	}
}
<|endoftext|>


<commit_before>package spruce

import (
	"fmt"
	"github.com/starkandwayne/goutils/ansi"
	"os"
	"regexp"
	"strconv"

	. "github.com/geofffranks/spruce/log"
	"github.com/starkandwayne/goutils/tree"
)

// Action ...
type Action int

const (
	// Replace ...
	Replace Action = iota

	// Inject ...
	Inject
)

// OperatorPhase ...
type OperatorPhase int

const (
	// MergePhase ...
	MergePhase OperatorPhase = iota
	// EvalPhase ...
	EvalPhase
	// ParamPhase ...
	ParamPhase
)

// Response ...
type Response struct {
	Type  Action
	Value interface{}
}

// Operator ...
type Operator interface {
	// setup whatever global/static state needed -- see (( static_ips ... ))
	Setup() error

	// evaluate the tree and determine what should be done to satisfy caller
	Run(ev *Evaluator, args []*Expr) (*Response, error)

	// returns a set of implicit / inherent dependencies used by Run()
	Dependencies(ev *Evaluator, args []*Expr, locs []*tree.Cursor) []*tree.Cursor

	// what phase does this operator run during?
	Phase() OperatorPhase
}

// OpRegistry ...
var OpRegistry map[string]Operator

// OperatorFor ...
func OperatorFor(name string) Operator {
	if op, ok := OpRegistry[name]; ok {
		return op
	}
	return NullOperator{Missing: name}
}

// RegisterOp ...
func RegisterOp(name string, op Operator) {
	if OpRegistry == nil {
		OpRegistry = map[string]Operator{}
	}
	OpRegistry[name] = op
}

// SetupOperators ...
func SetupOperators(phase OperatorPhase) error {
	errors := MultiError{Errors: []error{}}
	for _, op := range OpRegistry {
		if op.Phase() == phase {
			if err := op.Setup(); err != nil {
				errors.Append(err)
			}
		}
	}
	if len(errors.Errors) > 0 {
		return errors
	}
	return nil
}

// ExprType ...
type ExprType int

const (
	// Reference ...
	Reference ExprType = iota
	// Literal ...
	Literal
	// LogicalOr ...
	LogicalOr
	EnvVar
)

// Expr ...
type Expr struct {
	Type      ExprType
	Reference *tree.Cursor
	Literal   interface{}
	Name      string
	Left      *Expr
	Right     *Expr
}

func (e *Expr) String() string {
	switch e.Type {
	case Literal:
		if e.Literal == nil {
			return "nil"
		}
		if _, ok := e.Literal.(string); ok {
			return fmt.Sprintf(`"%s"`, e.Literal)
		}
		return fmt.Sprintf("%v", e.Literal)

	case EnvVar:
		return fmt.Sprintf("$%s", e.Name)

	case Reference:
		return e.Reference.String()

	case LogicalOr:
		return fmt.Sprintf("%s || %s", e.Left, e.Right)

	default:
		return "<!! unknown !!>"
	}
}

// Reduce ...
func (e *Expr) Reduce() (*Expr, error) {

	var reduce func(*Expr) (*Expr, *Expr, bool)
	reduce = func(e *Expr) (*Expr, *Expr, bool) {
		switch e.Type {
		case Literal:
			return e, e, false
		case EnvVar:
			return e, nil, false
		case Reference:
			return e, nil, false

		case LogicalOr:
			l, short, _ := reduce(e.Left)
			if short != nil {
				return l, short, true
			}

			r, short, more := reduce(e.Right)
			return &Expr{
				Type:  LogicalOr,
				Left:  l,
				Right: r,
			}, short, more
		}
		return nil, nil, false
	}

	reduced, short, more := reduce(e)
	if more && short != nil {
		return reduced, ansi.Errorf("@R{literal} @c{%v} @R{short-circuits expression (}@c{%s}@R{)}", short, e)
	}
	return reduced, nil
}

// Resolve ...
func (e *Expr) Resolve(tree map[interface{}]interface{}) (*Expr, error) {
	switch e.Type {
	case Literal:
		return e, nil

	case EnvVar:
		v := os.Getenv(e.Name)
		if v == "" {
			return nil, ansi.Errorf("@R{Environment variable} @c{$%s} @R{is not set}", e.Name)
		}
		return &Expr{Type: Literal, Literal: v}, nil

	case Reference:
		if _, err := e.Reference.Resolve(tree); err != nil {
			return nil, ansi.Errorf("@R{Unable to resolve `}@c{%s}@R{`: %s}", e.Reference, err)
		}
		return e, nil

	case LogicalOr:
		if o, err := e.Left.Resolve(tree); err == nil {
			return o, nil
		}
		return e.Right.Resolve(tree)
	}
	return nil, ansi.Errorf("@R{unknown expression operand type (}@c{%d}@R{)}", e.Type)
}

// Evaluate ...
func (e *Expr) Evaluate(tree map[interface{}]interface{}) (interface{}, error) {
	final, err := e.Resolve(tree)
	if err != nil {
		return nil, err
	}

	switch final.Type {
	case Literal:
		return final.Literal, nil
	case EnvVar:
		return os.Getenv(final.Name), nil
	case Reference:
		return final.Reference.Resolve(tree)
	case LogicalOr:
		return nil, fmt.Errorf("expression resolved to a logical OR operation (which shouldn't happen)")
	}
	return nil, fmt.Errorf("unknown operand type")
}

// Dependencies ...
func (e *Expr) Dependencies(ev *Evaluator, locs []*tree.Cursor) []*tree.Cursor {
	l := []*tree.Cursor{}

	canonicalize := func(c *tree.Cursor) {
		cc := c.Copy()
		for cc.Depth() > 0 {
			if _, err := cc.Canonical(ev.Tree); err == nil {
				break
			}
			cc.Pop()
		}
		if cc.Depth() > 0 {
			canon, _ := cc.Canonical(ev.Tree)
			l = append(l, canon)
		}
	}

	switch e.Type {
	case Reference:
		canonicalize(e.Reference)

	case LogicalOr:
		for _, c := range e.Left.Dependencies(ev, locs) {
			canonicalize(c)
		}
		for _, c := range e.Right.Dependencies(ev, locs) {
			canonicalize(c)
		}
	}

	return l
}

// Opcall ...
type Opcall struct {
	src       string
	where     *tree.Cursor
	canonical *tree.Cursor
	op        Operator
	args      []*Expr
}

// ParseOpcall ...
func ParseOpcall(phase OperatorPhase, src string) (*Opcall, error) {
	split := func(src string) []string {
		list := make([]string, 0, 0)

		buf := ""
		escaped := false
		quoted := false

		for _, c := range src {
			if escaped {
				switch c {
				case 'n':
					buf += "\n"
				case 'r':
					buf += "\r"
				case 't':
					buf += "\t"
				default:
					buf += string(c)
				}
				escaped = false
				continue
			}

			if c == '\\' {
				escaped = true
				continue
			}

			if c == ' ' || c == '\t' || c == ',' {
				if quoted {
					buf += string(c)
					continue
				} else {
					if buf != "" {
						list = append(list, buf)
						buf = ""
					}
					if c == ',' {
						list = append(list, ",")
					}
				}
				continue
			}

			if c == '"' {
				buf += string(c)
				quoted = !quoted
				continue
			}

			buf += string(c)
		}

		if buf != "" {
			list = append(list, buf)
		}

		return list
	}

	argify := func(src string) (args []*Expr, err error) {
		qstring := regexp.MustCompile(`(?s)^"(.*)"$`)
		integer := regexp.MustCompile(`^[+-]?\d+(\.\d+)?$`)
		float := regexp.MustCompile(`^[+-]?\d*\.\d+$`)
		envvar := regexp.MustCompile(`^\$[a-zA-Z_][a-zA-Z0-9_]*$`)

		var final []*Expr
		var left, op *Expr

		pop := func() {
			if left != nil {
				final = append(final, left)
				left = nil
			}
		}

		push := func(e *Expr) {
			TRACE("expr: pushing data expression `%s' onto stack", e)
			TRACE("expr:   start: left=`%s', op=`%s'", left, op)
			defer func() { TRACE("expr:     end: left=`%s', op=`%s'\n", left, op) }()

			if left == nil {
				left = e
				return
			}
			if op == nil {
				pop()
				left = e
				return
			}
			op.Left = left
			op.Right = e
			left = op
			op = nil
		}

		TRACE("expr: parsing `%s'", src)
		for i, arg := range split(src) {
			switch {
			case arg == ",":
				DEBUG("  #%d: literal comma found; treating what we've seen so far as a complete expression")
				pop()

			case envvar.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted environment variable reference '%s'", i, arg)
				push(&Expr{Type: EnvVar, Name: arg[1:]})

			case qstring.MatchString(arg):
				m := qstring.FindStringSubmatch(arg)
				DEBUG("  #%d: parsed as quoted string literal '%s'", i, m[1])
				push(&Expr{Type: Literal, Literal: m[1]})

			case float.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted floating point literal '%s'", i, arg)
				v, err := strconv.ParseFloat(arg, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parsable as a floating point number: %s", i, arg, err)
					return args, err
				}
				push(&Expr{Type: Literal, Literal: v})

			case integer.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted integer literal '%s'", i, arg)
				v, err := strconv.ParseInt(arg, 10, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parsable as an integer: %s", i, arg, err)
					return args, err
				}
				push(&Expr{Type: Literal, Literal: v})

			case arg == "||":
				DEBUG("  #%d: parsed logical-or operator, '||'", i)

				if left == nil || op != nil {
					return args, fmt.Errorf(`syntax error near: %s`, src)
				}
				TRACE("expr: pushing || expr-op onto the stack")
				op = &Expr{Type: LogicalOr}

			case arg == "nil" || arg == "null" || arg == "~" || arg == "Nil" || arg == "Null" || arg == "NIL" || arg == "NULL":
				DEBUG("  #%d: parsed the nil value token '%s'", i, arg)
				push(&Expr{Type: Literal, Literal: nil})

			case arg == "false" || arg == "False" || arg == "FALSE":
				DEBUG("  #%d: parsed the false value token '%s'", i, arg)
				push(&Expr{Type: Literal, Literal: false})

			case arg == "true" || arg == "True" || arg == "TRUE":
				DEBUG("  #%d: parsed the true value token '%s'", i, arg)
				push(&Expr{Type: Literal, Literal: true})

			default:
				c, err := tree.ParseCursor(arg)
				if err != nil {
					DEBUG("  #%d: %s is a malformed reference: %s", i, arg, err)
					return args, err
				}
				DEBUG("  #%d: parsed as a reference to $.%s", i, c)
				push(&Expr{Type: Reference, Reference: c})
			}
		}
		pop()
		if left != nil || op != nil {
			return nil, fmt.Errorf(`syntax error near: %s`, src)
		}
		DEBUG("")

		for _, e := range final {
			TRACE("expr: pushing expression `%v' onto the operand list", e)
			reduced, err := e.Reduce()
			if err != nil {
				fmt.Fprintf(os.Stdout, "warning: %s\n", err)
			}
			args = append(args, reduced)
		}

		return args, nil
	}

	op := &Opcall{src: src}

	for _, pattern := range []string{
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s*\((.*)\))?\s*\Q))\E$`, // (( op(x,y,z) ))
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s+(.*))?\s*\Q))\E$`,     // (( op x y z ))
	} {
		re := regexp.MustCompile(pattern)
		if !re.MatchString(src) {
			continue
		}

		m := re.FindStringSubmatch(src)
		DEBUG("parsing `%s': looks like a (( %s ... )) operator\n arguments:", src, m[1])

		op.op = OperatorFor(m[1])
		if op.op.Phase() != phase {
			DEBUG("  - skipping (( %s ... )) operation; it belongs to a different phase", m[1])
			return nil, nil
		}

		args, err := argify(m[2])
		if err != nil {
			return nil, err
		}
		if len(args) == 0 {
			DEBUG("  (none)")
		}
		op.args = args
		return op, nil
	}

	return nil, nil
}

// Dependencies ...
func (op *Opcall) Dependencies(ev *Evaluator, locs []*tree.Cursor) []*tree.Cursor {
	l := []*tree.Cursor{}
	for _, arg := range op.args {
		for _, c := range arg.Dependencies(ev, locs) {
			l = append(l, c)
		}
	}

	for _, c := range op.op.Dependencies(ev, op.args, locs) {
		l = append(l, c)
	}
	return l
}

// Run ...
func (op *Opcall) Run(ev *Evaluator) (*Response, error) {
	was := ev.Here
	ev.Here = op.where
	r, err := op.op.Run(ev, op.args)
	ev.Here = was

	if err != nil {
		return nil, ansi.Errorf("@m{$.%s}: @R{%s}", op.where, err)
	}
	return r, nil
}
<commit_msg>Fix missing arg in debug printf<commit_after>package spruce

import (
	"fmt"
	"os"
	"regexp"
	"strconv"

	"github.com/starkandwayne/goutils/ansi"

	. "github.com/geofffranks/spruce/log"
	"github.com/starkandwayne/goutils/tree"
)

// Action ...
type Action int

const (
	// Replace ...
	Replace Action = iota

	// Inject ...
	Inject
)

// OperatorPhase ...
type OperatorPhase int

const (
	// MergePhase ...
	MergePhase OperatorPhase = iota
	// EvalPhase ...
	EvalPhase
	// ParamPhase ...
	ParamPhase
)

// Response ...
type Response struct {
	Type  Action
	Value interface{}
}

// Operator ...
type Operator interface {
	// setup whatever global/static state needed -- see (( static_ips ... ))
	Setup() error

	// evaluate the tree and determine what should be done to satisfy caller
	Run(ev *Evaluator, args []*Expr) (*Response, error)

	// returns a set of implicit / inherent dependencies used by Run()
	Dependencies(ev *Evaluator, args []*Expr, locs []*tree.Cursor) []*tree.Cursor

	// what phase does this operator run during?
	Phase() OperatorPhase
}

// OpRegistry ...
var OpRegistry map[string]Operator

// OperatorFor ...
func OperatorFor(name string) Operator {
	if op, ok := OpRegistry[name]; ok {
		return op
	}
	return NullOperator{Missing: name}
}

// RegisterOp ...
func RegisterOp(name string, op Operator) {
	if OpRegistry == nil {
		OpRegistry = map[string]Operator{}
	}
	OpRegistry[name] = op
}

// SetupOperators ...
func SetupOperators(phase OperatorPhase) error {
	errors := MultiError{Errors: []error{}}
	for _, op := range OpRegistry {
		if op.Phase() == phase {
			if err := op.Setup(); err != nil {
				errors.Append(err)
			}
		}
	}
	if len(errors.Errors) > 0 {
		return errors
	}
	return nil
}

// ExprType ...
type ExprType int

const (
	// Reference ...
	Reference ExprType = iota
	// Literal ...
	Literal
	// LogicalOr ...
	LogicalOr
	EnvVar
)

// Expr ...
type Expr struct {
	Type      ExprType
	Reference *tree.Cursor
	Literal   interface{}
	Name      string
	Left      *Expr
	Right     *Expr
}

func (e *Expr) String() string {
	switch e.Type {
	case Literal:
		if e.Literal == nil {
			return "nil"
		}
		if _, ok := e.Literal.(string); ok {
			return fmt.Sprintf(`"%s"`, e.Literal)
		}
		return fmt.Sprintf("%v", e.Literal)

	case EnvVar:
		return fmt.Sprintf("$%s", e.Name)

	case Reference:
		return e.Reference.String()

	case LogicalOr:
		return fmt.Sprintf("%s || %s", e.Left, e.Right)

	default:
		return "<!! unknown !!>"
	}
}

// Reduce ...
func (e *Expr) Reduce() (*Expr, error) {

	var reduce func(*Expr) (*Expr, *Expr, bool)
	reduce = func(e *Expr) (*Expr, *Expr, bool) {
		switch e.Type {
		case Literal:
			return e, e, false
		case EnvVar:
			return e, nil, false
		case Reference:
			return e, nil, false

		case LogicalOr:
			l, short, _ := reduce(e.Left)
			if short != nil {
				return l, short, true
			}

			r, short, more := reduce(e.Right)
			return &Expr{
				Type:  LogicalOr,
				Left:  l,
				Right: r,
			}, short, more
		}
		return nil, nil, false
	}

	reduced, short, more := reduce(e)
	if more && short != nil {
		return reduced, ansi.Errorf("@R{literal} @c{%v} @R{short-circuits expression (}@c{%s}@R{)}", short, e)
	}
	return reduced, nil
}

// Resolve ...
func (e *Expr) Resolve(tree map[interface{}]interface{}) (*Expr, error) {
	switch e.Type {
	case Literal:
		return e, nil

	case EnvVar:
		v := os.Getenv(e.Name)
		if v == "" {
			return nil, ansi.Errorf("@R{Environment variable} @c{$%s} @R{is not set}", e.Name)
		}
		return &Expr{Type: Literal, Literal: v}, nil

	case Reference:
		if _, err := e.Reference.Resolve(tree); err != nil {
			return nil, ansi.Errorf("@R{Unable to resolve `}@c{%s}@R{`: %s}", e.Reference, err)
		}
		return e, nil

	case LogicalOr:
		if o, err := e.Left.Resolve(tree); err == nil {
			return o, nil
		}
		return e.Right.Resolve(tree)
	}
	return nil, ansi.Errorf("@R{unknown expression operand type (}@c{%d}@R{)}", e.Type)
}

// Evaluate ...
func (e *Expr) Evaluate(tree map[interface{}]interface{}) (interface{}, error) {
	final, err := e.Resolve(tree)
	if err != nil {
		return nil, err
	}

	switch final.Type {
	case Literal:
		return final.Literal, nil
	case EnvVar:
		return os.Getenv(final.Name), nil
	case Reference:
		return final.Reference.Resolve(tree)
	case LogicalOr:
		return nil, fmt.Errorf("expression resolved to a logical OR operation (which shouldn't happen)")
	}
	return nil, fmt.Errorf("unknown operand type")
}

// Dependencies ...
func (e *Expr) Dependencies(ev *Evaluator, locs []*tree.Cursor) []*tree.Cursor {
	l := []*tree.Cursor{}

	canonicalize := func(c *tree.Cursor) {
		cc := c.Copy()
		for cc.Depth() > 0 {
			if _, err := cc.Canonical(ev.Tree); err == nil {
				break
			}
			cc.Pop()
		}
		if cc.Depth() > 0 {
			canon, _ := cc.Canonical(ev.Tree)
			l = append(l, canon)
		}
	}

	switch e.Type {
	case Reference:
		canonicalize(e.Reference)

	case LogicalOr:
		for _, c := range e.Left.Dependencies(ev, locs) {
			canonicalize(c)
		}
		for _, c := range e.Right.Dependencies(ev, locs) {
			canonicalize(c)
		}
	}

	return l
}

// Opcall ...
type Opcall struct {
	src       string
	where     *tree.Cursor
	canonical *tree.Cursor
	op        Operator
	args      []*Expr
}

// ParseOpcall ...
func ParseOpcall(phase OperatorPhase, src string) (*Opcall, error) {
	split := func(src string) []string {
		list := make([]string, 0, 0)

		buf := ""
		escaped := false
		quoted := false

		for _, c := range src {
			if escaped {
				switch c {
				case 'n':
					buf += "\n"
				case 'r':
					buf += "\r"
				case 't':
					buf += "\t"
				default:
					buf += string(c)
				}
				escaped = false
				continue
			}

			if c == '\\' {
				escaped = true
				continue
			}

			if c == ' ' || c == '\t' || c == ',' {
				if quoted {
					buf += string(c)
					continue
				} else {
					if buf != "" {
						list = append(list, buf)
						buf = ""
					}
					if c == ',' {
						list = append(list, ",")
					}
				}
				continue
			}

			if c == '"' {
				buf += string(c)
				quoted = !quoted
				continue
			}

			buf += string(c)
		}

		if buf != "" {
			list = append(list, buf)
		}

		return list
	}

	argify := func(src string) (args []*Expr, err error) {
		qstring := regexp.MustCompile(`(?s)^"(.*)"$`)
		integer := regexp.MustCompile(`^[+-]?\d+(\.\d+)?$`)
		float := regexp.MustCompile(`^[+-]?\d*\.\d+$`)
		envvar := regexp.MustCompile(`^\$[a-zA-Z_][a-zA-Z0-9_]*$`)

		var final []*Expr
		var left, op *Expr

		pop := func() {
			if left != nil {
				final = append(final, left)
				left = nil
			}
		}

		push := func(e *Expr) {
			TRACE("expr: pushing data expression `%s' onto stack", e)
			TRACE("expr:   start: left=`%s', op=`%s'", left, op)
			defer func() { TRACE("expr:     end: left=`%s', op=`%s'\n", left, op) }()

			if left == nil {
				left = e
				return
			}
			if op == nil {
				pop()
				left = e
				return
			}
			op.Left = left
			op.Right = e
			left = op
			op = nil
		}

		TRACE("expr: parsing `%s'", src)
		for i, arg := range split(src) {
			switch {
			case arg == ",":
				DEBUG("  #%d: literal comma found; treating what we've seen so far as a complete expression", i)
				pop()

			case envvar.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted environment variable reference '%s'", i, arg)
				push(&Expr{Type: EnvVar, Name: arg[1:]})

			case qstring.MatchString(arg):
				m := qstring.FindStringSubmatch(arg)
				DEBUG("  #%d: parsed as quoted string literal '%s'", i, m[1])
				push(&Expr{Type: Literal, Literal: m[1]})

			case float.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted floating point literal '%s'", i, arg)
				v, err := strconv.ParseFloat(arg, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parsable as a floating point number: %s", i, arg, err)
					return args, err
				}
				push(&Expr{Type: Literal, Literal: v})

			case integer.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted integer literal '%s'", i, arg)
				v, err := strconv.ParseInt(arg, 10, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parsable as an integer: %s", i, arg, err)
					return args, err
				}
				push(&Expr{Type: Literal, Literal: v})

			case arg == "||":
				DEBUG("  #%d: parsed logical-or operator, '||'", i)

				if left == nil || op != nil {
					return args, fmt.Errorf(`syntax error near: %s`, src)
				}
				TRACE("expr: pushing || expr-op onto the stack")
				op = &Expr{Type: LogicalOr}

			case arg == "nil" || arg == "null" || arg == "~" || arg == "Nil" || arg == "Null" || arg == "NIL" || arg == "NULL":
				DEBUG("  #%d: parsed the nil value token '%s'", i, arg)
				push(&Expr{Type: Literal, Literal: nil})

			case arg == "false" || arg == "False" || arg == "FALSE":
				DEBUG("  #%d: parsed the false value token '%s'", i, arg)
				push(&Expr{Type: Literal, Literal: false})

			case arg == "true" || arg == "True" || arg == "TRUE":
				DEBUG("  #%d: parsed the true value token '%s'", i, arg)
				push(&Expr{Type: Literal, Literal: true})

			default:
				c, err := tree.ParseCursor(arg)
				if err != nil {
					DEBUG("  #%d: %s is a malformed reference: %s", i, arg, err)
					return args, err
				}
				DEBUG("  #%d: parsed as a reference to $.%s", i, c)
				push(&Expr{Type: Reference, Reference: c})
			}
		}
		pop()
		if left != nil || op != nil {
			return nil, fmt.Errorf(`syntax error near: %s`, src)
		}
		DEBUG("")

		for _, e := range final {
			TRACE("expr: pushing expression `%v' onto the operand list", e)
			reduced, err := e.Reduce()
			if err != nil {
				fmt.Fprintf(os.Stdout, "warning: %s\n", err)
			}
			args = append(args, reduced)
		}

		return args, nil
	}

	op := &Opcall{src: src}

	for _, pattern := range []string{
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s*\((.*)\))?\s*\Q))\E$`, // (( op(x,y,z) ))
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s+(.*))?\s*\Q))\E$`,     // (( op x y z ))
	} {
		re := regexp.MustCompile(pattern)
		if !re.MatchString(src) {
			continue
		}

		m := re.FindStringSubmatch(src)
		DEBUG("parsing `%s': looks like a (( %s ... )) operator\n arguments:", src, m[1])

		op.op = OperatorFor(m[1])
		if op.op.Phase() != phase {
			DEBUG("  - skipping (( %s ... )) operation; it belongs to a different phase", m[1])
			return nil, nil
		}

		args, err := argify(m[2])
		if err != nil {
			return nil, err
		}
		if len(args) == 0 {
			DEBUG("  (none)")
		}
		op.args = args
		return op, nil
	}

	return nil, nil
}

// Dependencies ...
func (op *Opcall) Dependencies(ev *Evaluator, locs []*tree.Cursor) []*tree.Cursor {
	l := []*tree.Cursor{}
	for _, arg := range op.args {
		for _, c := range arg.Dependencies(ev, locs) {
			l = append(l, c)
		}
	}

	for _, c := range op.op.Dependencies(ev, op.args, locs) {
		l = append(l, c)
	}
	return l
}

// Run ...
func (op *Opcall) Run(ev *Evaluator) (*Response, error) {
	was := ev.Here
	ev.Here = op.where
	r, err := op.op.Run(ev, op.args)
	ev.Here = was

	if err != nil {
		return nil, ansi.Errorf("@m{$.%s}: @R{%s}", op.where, err)
	}
	return r, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 DB Medialab.  All rights reserved.
// License: MIT

// Package orbitapi provides client access to the Orbit API (http://orbitapi.com/)
package orbitapi

import (
	"encoding/json"
	"net/http"
	"net/url"
	"strings"
)

var (
	// URL on which the Orbit API can be reached
	orbitApiUrl = "http://api.orbitapi.com/"
)

type OrbitApi struct {
	// Key to access the API with
	apiKey string

	// Result will be sent on this channel
	Result chan map[string]interface{}
}

// Create a new Orbit API client
func NewClient(apiKey string) (orbitapi *OrbitApi) {
	orbitapi = new(OrbitApi)
	orbitapi.apiKey = apiKey
	orbitapi.Result = make(chan map[string]interface{})
	return
}

// Send a new GET request to the API
func (o *OrbitApi) Get(uri string) error {
	getUrl := orbitApiUrl + uri
	req, err := http.NewRequest("GET", getUrl, nil)
	if err != nil {
		return err
	}

	// Get requests require the API key to be sent as a header
	req.Header.Add("X-Orbit-API-Key", o.apiKey)
	return o.doRequest(req)
}

// Send a new POST request to the API
func (o *OrbitApi) Post(uri string, args url.Values) error {
	postUrl := orbitApiUrl + uri
	// Post requests require the API key to be sent as a key=value pair
	args.Add("api_key", o.apiKey)
	req, err := http.NewRequest("POST", postUrl, strings.NewReader(args.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return o.doRequest(req)
}

// Do the actual request and return the response on o.Result
func (o *OrbitApi) doRequest(req *http.Request) error {
	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	var data map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return err
	}
	o.Result <- data
	return nil
}
<commit_msg>Structs for concept tagging response.<commit_after>// Copyright 2014 DB Medialab.  All rights reserved.
// License: MIT

// Package orbitapi provides client access to the Orbit API (http://orbitapi.com/ - http://orbit.ai/documentation/introduction)
package orbitapi

import (
	"encoding/json"
	"net/http"
	"net/url"
	"strings"
)

var (
	// URL on which the Orbit API can be reached
	orbitApiUrl = "http://api.orbitapi.com/"
)

type OrbitApi struct {
	// Key to access the API with
	apiKey string

	// Result will be sent on this channel
	Result chan interface{}
}

type OrbitTag struct {
	Entities       map[string]OrbitEntity `json:"entities"`
	Text           []interface{}          `json:"text"`
	RemainingWords int                    `json:"remaining_words"`
}

type OrbitEntity struct {
	Image     string  `json:"image"`
	Label     string  `json:"label"`
	Link      string  `json:"link"`
	Relevance float64 `json:"relevance"`
	Thumbnail string  `json:"thumbnail"`
	Type      string  `json:"type"`
}

// Create a new Orbit API client
func NewClient(apiKey string) (orbitapi *OrbitApi) {
	orbitapi = new(OrbitApi)
	orbitapi.apiKey = apiKey
	orbitapi.Result = make(chan interface{})
	return
}

// Send a new GET request to the API
func (o *OrbitApi) Get(uri string) error {
	getUrl := orbitApiUrl + uri
	req, err := http.NewRequest("GET", getUrl, nil)
	if err != nil {
		return err
	}

	// Get requests require the API key to be sent as a header
	req.Header.Add("X-Orbit-API-Key", o.apiKey)
	data := make(map[string]interface{})
	return o.doRequest(req, data)
}

// Send a new POST request to the API
func (o *OrbitApi) Post(uri string, args url.Values) error {
	postUrl := orbitApiUrl + uri
	// Post requests require the API key to be sent as a key=value pair
	args.Add("api_key", o.apiKey)
	req, err := http.NewRequest("POST", postUrl, strings.NewReader(args.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	var data interface{}
	if uri == "tag" {
		data = new(OrbitTag)
	} else {
		data = make(map[string]interface{})
	}
	return o.doRequest(req, data)
}

// Do the actual request and return the response on o.Result
func (o *OrbitApi) doRequest(req *http.Request, data interface{}) error {
	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return err
	}
	o.Result <- data
	return nil
}
<|endoftext|>