<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package quickfix

import (
	"bytes"
	"nvim-go/context"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/garyburd/neovim-go/vim"
)

// ErrorlistData represents an item in a quickfix and locationlist.
type ErrorlistData struct {
	// Buffer number
	Bufnr int `msgpack:"bufnr,omitempty"`

	// Name of a file; only used when bufnr is not present or it is invalid.
	FileName string `msgpack:"filename,omitempty"`

	// Line number in the file.
	LNum int `msgpack:"lnum,omitempty"`

	// Column number (first column is 1).
	Col int `msgpack:"col,omitempty"`

	// When Vcol is != 0,  Col is visual column.
	VCol int `msgpack:"vcol,omitempty"`

	// Error number.
	Nr int `msgpack:"nr,omitempty"`

	// Search pattern used to locate the error.
	Pattern string `msgpack:"pattern,omitempty"`

	// Description of the error.
	Text string `msgpack:"text,omitempty"`

	// Single-character error type, 'E', 'W', etc.
	Type string `msgpack:"type,omitempty"`

	// Valid is non-zero if this is a recognized error message.
	Valid int `msgpack:"valid,omitempty"`
}

// SetLoclist set the error results data to current buffer's locationlist.
func SetLoclist(p *vim.Pipeline, loclist []*ErrorlistData) error {
	// setloclist({nr}, {list} [, {action}])
	// Call(fname string, result interface{}, args ...interface{})
	if len(loclist) > 0 {
		p.Call("setloclist", nil, 0, loclist)
	} else {
		p.Command("lexpr ''")
	}

	return nil
}

// OpenLoclist open or close the current buffer's locationlist window.
func OpenLoclist(p *vim.Pipeline, w vim.Window, loclist []*ErrorlistData, keep bool) error {
	if len(loclist) > 0 {
		p.Command("lopen")
		if keep {
			p.SetCurrentWindow(w)
		}

		if err := p.Wait(); err != nil {
			return err
		}
	} else {
		p.Command("lclose")
		if err := p.Wait(); err != nil {
			return err
		}
	}

	return nil
}

// CloseLoclist close the current buffer's locationlist window.
func CloseLoclist(v *vim.Vim) error {
	return v.Command("lclose")
}

// SetQuickfix set the error results data to quickfix list.
func SetQuickfix(p *vim.Pipeline, qflist []*ErrorlistData) error {
	p.Call("setqflist", nil, qflist)

	return nil
}

// OpenOuickfix open the quickfix list window.
func OpenOuickfix(p *vim.Pipeline, w vim.Window, keep bool) error {
	p.Command("copen")
	if keep {
		p.SetCurrentWindow(w)
	}
	if err := p.Wait(); err != nil {
		return err
	}

	return nil
}

// CloseQuickfix close the quickfix list window.
func CloseQuickfix(v *vim.Vim) error {
	return v.Command("cclose")
}

// SplitPos parses a string of form 'token.Pos', and return the relative
// filepath from the current working directory path.
func SplitPos(pos string, cwd string) (string, int, int) {
	slc := strings.Split(pos, ":")
	line, err := strconv.ParseInt(slc[1], 10, 64)
	if err != nil {
		line = 0
	}
	col, err := strconv.ParseInt(slc[2], 10, 64)
	if err != nil {
		col = 0
	}

	fname := slc[0]
	frel := strings.TrimPrefix(fname, cwd+string(filepath.Separator))
	if fname == frel {
		return fname, int(line), int(col)
	}

	return frel, int(line), int(col)
}

// ParseError parse a typical output of command written in Go.
func ParseError(errors []byte, cwd string, ctxt *context.Build) ([]*ErrorlistData, error) {
	var (
		errlist []*ErrorlistData
		errPat  = regexp.MustCompile(`^# ([^:]+):(\d+)(?::(\d+))?:\s(.*)`)
		fname   string
	)

	for _, m := range errPat.FindAllSubmatch(errors, -1) {
		fb := bytes.Split(m[1], []byte("\n"))
		fs := string(bytes.Join(fb, []byte(string(filepath.Separator))))
		if ctxt.Tool == "go" {
			sep := ctxt.GOPATH + string(filepath.Separator) + "src" + string(filepath.Separator)
			c := strings.TrimPrefix(cwd, sep)

			fname = strings.TrimPrefix(filepath.Clean(fs), c+string(filepath.Separator))
		} else if ctxt.Tool == "gb" {
			sep := filepath.Base(cwd) + string(filepath.Separator)
			fname = strings.TrimPrefix(filepath.Clean(fs), sep)
		}

		line, _ := strconv.Atoi(string(m[2]))
		col, _ := strconv.Atoi(string(m[3]))

		errlist = append(errlist, &ErrorlistData{
			FileName: fname,
			LNum:     line,
			Col:      col,
			Text:     string(bytes.TrimSpace(m[4])),
		})
	}

	return errlist, nil
}
<commit_msg>quickfix: Fix wrong filepath<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package quickfix

import (
	"bytes"
	"log"
	"nvim-go/context"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/garyburd/neovim-go/vim"
)

// ErrorlistData represents an item in a quickfix and locationlist.
type ErrorlistData struct {
	// Buffer number
	Bufnr int `msgpack:"bufnr,omitempty"`

	// Name of a file; only used when bufnr is not present or it is invalid.
	FileName string `msgpack:"filename,omitempty"`

	// Line number in the file.
	LNum int `msgpack:"lnum,omitempty"`

	// Column number (first column is 1).
	Col int `msgpack:"col,omitempty"`

	// When Vcol is != 0,  Col is visual column.
	VCol int `msgpack:"vcol,omitempty"`

	// Error number.
	Nr int `msgpack:"nr,omitempty"`

	// Search pattern used to locate the error.
	Pattern string `msgpack:"pattern,omitempty"`

	// Description of the error.
	Text string `msgpack:"text,omitempty"`

	// Single-character error type, 'E', 'W', etc.
	Type string `msgpack:"type,omitempty"`

	// Valid is non-zero if this is a recognized error message.
	Valid int `msgpack:"valid,omitempty"`
}

// SetLoclist set the error results data to current buffer's locationlist.
func SetLoclist(p *vim.Pipeline, loclist []*ErrorlistData) error {
	// setloclist({nr}, {list} [, {action}])
	// Call(fname string, result interface{}, args ...interface{})
	if len(loclist) > 0 {
		p.Call("setloclist", nil, 0, loclist)
	} else {
		p.Command("lexpr ''")
	}

	return nil
}

// OpenLoclist open or close the current buffer's locationlist window.
func OpenLoclist(p *vim.Pipeline, w vim.Window, loclist []*ErrorlistData, keep bool) error {
	if len(loclist) > 0 {
		p.Command("lopen")
		if keep {
			p.SetCurrentWindow(w)
		}

		if err := p.Wait(); err != nil {
			return err
		}
	} else {
		p.Command("lclose")
		if err := p.Wait(); err != nil {
			return err
		}
	}

	return nil
}

// CloseLoclist close the current buffer's locationlist window.
func CloseLoclist(v *vim.Vim) error {
	return v.Command("lclose")
}

// SetQuickfix set the error results data to quickfix list.
func SetQuickfix(p *vim.Pipeline, qflist []*ErrorlistData) error {
	p.Call("setqflist", nil, qflist)

	return nil
}

// OpenOuickfix open the quickfix list window.
func OpenOuickfix(p *vim.Pipeline, w vim.Window, keep bool) error {
	p.Command("copen")
	if keep {
		p.SetCurrentWindow(w)
	}
	if err := p.Wait(); err != nil {
		return err
	}

	return nil
}

// CloseQuickfix close the quickfix list window.
func CloseQuickfix(v *vim.Vim) error {
	return v.Command("cclose")
}

// SplitPos parses a string of form 'token.Pos', and return the relative
// filepath from the current working directory path.
func SplitPos(pos string, cwd string) (string, int, int) {
	slc := strings.Split(pos, ":")
	line, err := strconv.ParseInt(slc[1], 10, 64)
	if err != nil {
		line = 0
	}
	col, err := strconv.ParseInt(slc[2], 10, 64)
	if err != nil {
		col = 0
	}

	fname := slc[0]
	frel := strings.TrimPrefix(fname, cwd+string(filepath.Separator))
	if fname == frel {
		return fname, int(line), int(col)
	}

	return frel, int(line), int(col)
}

// ParseError parse a typical output of command written in Go.
func ParseError(errors []byte, cwd string, ctxt *context.Build) ([]*ErrorlistData, error) {
	var (
		errlist []*ErrorlistData
		errPat  = regexp.MustCompile(`^# ([^:]+):(\d+)(?::(\d+))?:\s(.*)`)
		fname   string
	)

	for _, m := range errPat.FindAllSubmatch(errors, -1) {
		fb := bytes.Split(m[1], []byte("\n"))
		fs := string(bytes.Join(fb, []byte(string(filepath.Separator))))

		switch ctxt.Tool {
		case "go":
			sep := filepath.Join(ctxt.GOPATH, "src")
			c := strings.TrimPrefix(cwd, sep)
			fname = strings.TrimPrefix(filepath.Clean(fs), c+string(filepath.Separator))

		case "gb":
			fabs := filepath.Join(ctxt.ProjectDir, "src", fs)
			fname, _ = filepath.Rel(cwd, fabs)
		}

		line, _ := strconv.Atoi(string(m[2]))
		col, _ := strconv.Atoi(string(m[3]))

		errlist = append(errlist, &ErrorlistData{
			FileName: fname,
			LNum:     line,
			Col:      col,
			Text:     string(bytes.TrimSpace(m[4])),
		})
		log.Printf("errlist: %+v\n")
	}

	return errlist, nil
}
<|endoftext|>


<commit_before>package validation

import (
	"fmt"
	"net"
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/structure"
)

// IntBetween returns a SchemaValidateFunc which tests if the provided value
// is of type int and is between min and max (inclusive)
func IntBetween(min, max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(int)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be int", k))
			return
		}

		if v < min || v > max {
			es = append(es, fmt.Errorf("expected %s to be in the range (%d - %d), got %d", k, min, max, v))
			return
		}

		return
	}
}

// IntAtLeast returns a SchemaValidateFunc which tests if the provided value
// is of type int and is at least min (inclusive)
func IntAtLeast(min int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(int)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be int", k))
			return
		}

		if v < min {
			es = append(es, fmt.Errorf("expected %s to be at least (%d), got %d", k, min, v))
			return
		}

		return
	}
}

// IntAtMost returns a SchemaValidateFunc which tests if the provided value
// is of type int and is at most max (inclusive)
func IntAtMost(max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(int)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be int", k))
			return
		}

		if v > max {
			es = append(es, fmt.Errorf("expected %s to be at most (%d), got %d", k, max, v))
			return
		}

		return
	}
}

// StringInSlice returns a SchemaValidateFunc which tests if the provided value
// is of type string and matches the value of an element in the valid slice
// will test with in lower case if ignoreCase is true
func StringInSlice(valid []string, ignoreCase bool) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(string)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be string", k))
			return
		}

		for _, str := range valid {
			if v == str || (ignoreCase && strings.ToLower(v) == strings.ToLower(str)) {
				return
			}
		}

		es = append(es, fmt.Errorf("expected %s to be one of %v, got %s", k, valid, v))
		return
	}
}

// StringLenBetween returns a SchemaValidateFunc which tests if the provided value
// is of type string and has length between min and max (inclusive)
func StringLenBetween(min, max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(string)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be string", k))
			return
		}
		if len(v) < min || len(v) > max {
			es = append(es, fmt.Errorf("expected length of %s to be in the range (%d - %d), got %s", k, min, max, v))
		}
		return
	}
}

// CIDRNetwork returns a SchemaValidateFunc which tests if the provided value
// is of type string, is in valid CIDR network notation, and has significant bits between min and max (inclusive)
func CIDRNetwork(min, max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(string)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be string", k))
			return
		}

		_, ipnet, err := net.ParseCIDR(v)
		if err != nil {
			es = append(es, fmt.Errorf(
				"expected %s to contain a valid CIDR, got: %s with err: %s", k, v, err))
			return
		}

		if ipnet == nil || v != ipnet.String() {
			es = append(es, fmt.Errorf(
				"expected %s to contain a valid network CIDR, expected %s, got %s",
				k, ipnet, v))
		}

		sigbits, _ := ipnet.Mask.Size()
		if sigbits < min || sigbits > max {
			es = append(es, fmt.Errorf(
				"expected %q to contain a network CIDR with between %d and %d significant bits, got: %d",
				k, min, max, sigbits))
		}

		return
	}
}

// ValidateJsonString is a SchemaValidateFunc which tests to make sure the
// supplied string is valid JSON.
func ValidateJsonString(v interface{}, k string) (ws []string, errors []error) {
	if _, err := structure.NormalizeJsonString(v); err != nil {
		errors = append(errors, fmt.Errorf("%q contains an invalid JSON: %s", k, err))
	}
	return
}

// ValidateListUniqueStrings is a ValidateFunc that ensures a list has no
// duplicate items in it. It's useful for when a list is needed over a set
// because order matters, yet the items still need to be unique.
func ValidateListUniqueStrings(v interface{}, k string) (ws []string, errors []error) {
	for n1, v1 := range v.([]interface{}) {
		for n2, v2 := range v.([]interface{}) {
			if v1.(string) == v2.(string) && n1 != n2 {
				errors = append(errors, fmt.Errorf("%q: duplicate entry - %s", k, v1.(string)))
			}
		}
	}
}

// ValidateRegexp returns a SchemaValidateFunc which tests to make sure the
// supplied string is a valid regular expression.
func ValidateRegexp(v interface{}, k string) (ws []string, errors []error) {
	if _, err := regexp.Compile(v.(string)); err != nil {
		errors = append(errors, fmt.Errorf("%q: %s", k, err))
	}
	return
}<commit_msg>Add newline to end of validation.go<commit_after>package validation

import (
	"fmt"
	"net"
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/structure"
)

// IntBetween returns a SchemaValidateFunc which tests if the provided value
// is of type int and is between min and max (inclusive)
func IntBetween(min, max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(int)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be int", k))
			return
		}

		if v < min || v > max {
			es = append(es, fmt.Errorf("expected %s to be in the range (%d - %d), got %d", k, min, max, v))
			return
		}

		return
	}
}

// IntAtLeast returns a SchemaValidateFunc which tests if the provided value
// is of type int and is at least min (inclusive)
func IntAtLeast(min int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(int)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be int", k))
			return
		}

		if v < min {
			es = append(es, fmt.Errorf("expected %s to be at least (%d), got %d", k, min, v))
			return
		}

		return
	}
}

// IntAtMost returns a SchemaValidateFunc which tests if the provided value
// is of type int and is at most max (inclusive)
func IntAtMost(max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(int)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be int", k))
			return
		}

		if v > max {
			es = append(es, fmt.Errorf("expected %s to be at most (%d), got %d", k, max, v))
			return
		}

		return
	}
}

// StringInSlice returns a SchemaValidateFunc which tests if the provided value
// is of type string and matches the value of an element in the valid slice
// will test with in lower case if ignoreCase is true
func StringInSlice(valid []string, ignoreCase bool) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(string)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be string", k))
			return
		}

		for _, str := range valid {
			if v == str || (ignoreCase && strings.ToLower(v) == strings.ToLower(str)) {
				return
			}
		}

		es = append(es, fmt.Errorf("expected %s to be one of %v, got %s", k, valid, v))
		return
	}
}

// StringLenBetween returns a SchemaValidateFunc which tests if the provided value
// is of type string and has length between min and max (inclusive)
func StringLenBetween(min, max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(string)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be string", k))
			return
		}
		if len(v) < min || len(v) > max {
			es = append(es, fmt.Errorf("expected length of %s to be in the range (%d - %d), got %s", k, min, max, v))
		}
		return
	}
}

// CIDRNetwork returns a SchemaValidateFunc which tests if the provided value
// is of type string, is in valid CIDR network notation, and has significant bits between min and max (inclusive)
func CIDRNetwork(min, max int) schema.SchemaValidateFunc {
	return func(i interface{}, k string) (s []string, es []error) {
		v, ok := i.(string)
		if !ok {
			es = append(es, fmt.Errorf("expected type of %s to be string", k))
			return
		}

		_, ipnet, err := net.ParseCIDR(v)
		if err != nil {
			es = append(es, fmt.Errorf(
				"expected %s to contain a valid CIDR, got: %s with err: %s", k, v, err))
			return
		}

		if ipnet == nil || v != ipnet.String() {
			es = append(es, fmt.Errorf(
				"expected %s to contain a valid network CIDR, expected %s, got %s",
				k, ipnet, v))
		}

		sigbits, _ := ipnet.Mask.Size()
		if sigbits < min || sigbits > max {
			es = append(es, fmt.Errorf(
				"expected %q to contain a network CIDR with between %d and %d significant bits, got: %d",
				k, min, max, sigbits))
		}

		return
	}
}

// ValidateJsonString is a SchemaValidateFunc which tests to make sure the
// supplied string is valid JSON.
func ValidateJsonString(v interface{}, k string) (ws []string, errors []error) {
	if _, err := structure.NormalizeJsonString(v); err != nil {
		errors = append(errors, fmt.Errorf("%q contains an invalid JSON: %s", k, err))
	}
	return
}

// ValidateListUniqueStrings is a ValidateFunc that ensures a list has no
// duplicate items in it. It's useful for when a list is needed over a set
// because order matters, yet the items still need to be unique.
func ValidateListUniqueStrings(v interface{}, k string) (ws []string, errors []error) {
	for n1, v1 := range v.([]interface{}) {
		for n2, v2 := range v.([]interface{}) {
			if v1.(string) == v2.(string) && n1 != n2 {
				errors = append(errors, fmt.Errorf("%q: duplicate entry - %s", k, v1.(string)))
			}
		}
	}
}

// ValidateRegexp returns a SchemaValidateFunc which tests to make sure the
// supplied string is a valid regular expression.
func ValidateRegexp(v interface{}, k string) (ws []string, errors []error) {
	if _, err := regexp.Compile(v.(string)); err != nil {
		errors = append(errors, fmt.Errorf("%q: %s", k, err))
	}
	return
}
<|endoftext|>


<commit_before>package resolves

import (
	"strconv"
	"strings"
	"time"

	"github.com/hysios/apiai-go"
	"github.com/wanliu/brain_data/database"
	"github.com/wanliu/flow/builtin/ai"

	. "github.com/wanliu/flow/context"

	client "github.com/wanliu/flow/builtin/graphql_client"
)

// 处理开单的逻辑结构, 不需要是组件
// 作为context的一个部分，或者存在一个Value中
type OrderResolve struct {
	AiParams  ai.AiOrder
	Products  ItemsResolve
	Gifts     ItemsResolve
	Address   string
	Customer  string
	Time      time.Time
	DefTime   string
	Current   Resolve
	Note      string
	UpdatedAt time.Time
	Editing   bool
	Canceled  bool

	User *database.User
}

func NewOrderResolve(ctx Context) *OrderResolve {
	resolve := new(OrderResolve)
	resolve.Touch()

	aiResult := ctx.Value("Result").(apiai.Result)

	resolve.AiParams = ai.ApiAiOrder{AiResult: aiResult}
	resolve.ExtractFromParams()

	if viewer := ctx.Value("Viewer"); viewer != nil {
		user := viewer.(*database.User)
		resolve.User = user
	}

	return resolve
}

func (r *OrderResolve) Solve(aiResult apiai.Result) string {
	return r.Answer()
}

func (r *OrderResolve) Touch() {
	r.UpdatedAt = time.Now()
}

func (r OrderResolve) Modifable(expireMin int) bool {
	return !r.Expired(expireMin) || r.Submited()
}

// TODO
func (r OrderResolve) Cancelable() bool {
	return true
}

// TODO
func (r *OrderResolve) Cancel() bool {
	r.Canceled = true
	return true
}

func (r OrderResolve) Fulfiled() bool {
	return len(r.Products.Products) > 0 && (r.Address != "" || r.Customer != "")
}

func (r OrderResolve) Expired(expireMin int) bool {
	return r.UpdatedAt.Add(time.Duration(expireMin)*time.Minute).UnixNano() < time.Now().UnixNano()
}

// TODO
func (r OrderResolve) Submited() bool {
	return false
}

// 从ｌｕｉｓ数据构造结构数据
func (r *OrderResolve) ExtractFromParams() {
	r.ExtractItems()
	r.ExtractGiftItems()
	r.ExtractAddress()
	r.ExtractCustomer()
	r.ExtractTime()
	r.ExtractNote()
}

func (r *OrderResolve) ExtractItems() {
	for _, i := range r.AiParams.Items() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Products.Products = append(r.Products.Products, item)
	}
}

func (r *OrderResolve) ExtractGiftItems() {
	for _, i := range r.AiParams.GiftItems() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Gifts.Products = append(r.Gifts.Products, item)
	}
}

func (r *OrderResolve) ExtractAddress() {
	r.Address = r.AiParams.Address()
}

func (r *OrderResolve) ExtractCustomer() {
	r.Customer = r.AiParams.Customer()
}

func (r *OrderResolve) ExtractTime() {
	r.Time = r.AiParams.Time()
}

func (r *OrderResolve) ExtractNote() {
	r.Note = r.AiParams.Note()
}

func (r *OrderResolve) SetDefTime(t string) {
	r.DefTime = t

	if r.Time.IsZero() && r.DefTime != "" {
		r.SetTimeByDef()
	}
}

func (r *OrderResolve) SetTimeByDef() {
	if r.DefTime == "今天" {
		r.Time = time.Now()
	} else if r.DefTime == "明天" {
		r.Time = time.Now().Add(24 * time.Hour)
	}
}

func (r OrderResolve) EmptyProducts() bool {
	return len(r.Products.Products) == 0
}

func (r OrderResolve) Answer() string {
	if r.Fulfiled() {
		return r.PostOrderAndAnswer()
	} else {
		return r.AnswerHead() + r.AnswerFooter("")
	}
}

func (r *OrderResolve) PostOrderAndAnswer() string {
	mutationStr := `mutation createOrderMutation($input: OrderInput!) {
        createOrder(input: $input) {
            __typename
            order {
                address
                note
                id
                no
                deliveryTime
                items {
                    product {
                        name
                        picUrl
                        price
                    }
                    price
                    quantity
                }
                gifts {
                    product {
                        name
                        picUrl
                        price
                    }
                    quantity
                }
                saler {
                    id
                    name
                }
            }
        }
    }`

	variables := `{"input": {"address":"` + r.Address + `","deliveryTime":"` + r.Time.Format(time.RFC3339) + `","clientMutationId":0` + `,"items":[`

	itemStrs := []string{}
	for _, item := range r.Products.Products {
		iStr := `{"quantity":` + strconv.Itoa(item.Quantity) + `,"productName":"` + item.Product + `"}`
		itemStrs = append(itemStrs, iStr)
	}
	variables = variables + strings.Join(itemStrs, ",") + `]`

	if len(r.Gifts.Products) > 0 {
		variables = variables + `,"gifts":[`
		giftStrs := []string{}
		for _, gift := range r.Gifts.Products {
			iStr := `{"quantity":` + strconv.Itoa(gift.Quantity) + `,"productName":"` + gift.Product + `"}`
			giftStrs = append(giftStrs, iStr)
		}
		variables = variables + strings.Join(giftStrs, ",") + `]}`
	}

	if r.Note != "" {
		variables = variables + `,"note":"` + r.Note + `"`
	}

	variables = variables + "}}"

	requstSt := client.QueryToRequest(mutationStr, variables)
	r.User.CreateSaledOrder()
	// res, _ := client.MakeGraphqlRequest(requstStr)
	return requstSt
	// return r.AnswerHead() + res.AnswerBody() + r.AnswerFooter(res.OrderNo())
}

func (r OrderResolve) AddressInfo() string {
	if r.Address != "" && r.Customer != "" {
		return "地址:" + r.Address + r.Customer + "\n"
	} else if r.Address != "" {
		return "地址:" + r.Address + "\n"
	} else if r.Customer != "" {
		return "客户:" + r.Customer + "\n"
	} else {
		return ""
	}
}

func (r OrderResolve) AnswerHead() string {
	desc := "订单正在处理, 已经添加" + CnNum(len(r.Products.Products)) + "种产品"

	if r.Fulfiled() {
		desc = "订单已经生成, 共" + CnNum(len(r.Products.Products)) + "种产品"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + ", " + CnNum(len(r.Gifts.Products)) + "种赠品" + "\n"
	} else {
		desc = desc + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerBody() string {
	desc := ""

	for _, p := range r.Products.Products {
		desc = desc + p.Product + " " + strconv.Itoa(p.Quantity) + "件\n"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + "申请的赠品:\n"

		for _, g := range r.Gifts.Products {
			desc = desc + g.Product + " " + strconv.Itoa(g.Quantity) + "件\n"
		}
	}

	desc = desc + "时间:" + r.Time.Format("2006年01月02日") + "\n"

	if r.Note != "" {
		desc = desc + "备注：" + r.Note + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerFooter(no string) string {
	desc := ""

	if r.Fulfiled() {
		desc = desc + r.AddressInfo()
		desc = desc + "订单已经生成，订单号为：" + no + "\n"
		desc = desc + "订单入口: http://wanliu.biz/orders/" + no
	} else {
		desc = desc + "还缺少收货地址或客户信息\n"
	}

	return desc
}

func CnNum(num int) string {
	switch num {
	case 1:
		return "一"
	case 2:
		return "两"
	case 3:
		return "三"
	case 4:
		return "四"
	case 5:
		return "五"
	case 6:
		return "六"
	case 7:
		return "七"
	case 8:
		return "八"
	case 9:
		return "九"
	case 10:
		return "十"
	default:
		return strconv.Itoa(num)
	}
}
<commit_msg>create order for user<commit_after>package resolves

import (
	"strconv"
	"strings"
	"time"

	"github.com/hysios/apiai-go"
	"github.com/wanliu/brain_data/database"
	"github.com/wanliu/flow/builtin/ai"

	. "github.com/wanliu/flow/context"

	client "github.com/wanliu/flow/builtin/graphql_client"
)

// 处理开单的逻辑结构, 不需要是组件
// 作为context的一个部分，或者存在一个Value中
type OrderResolve struct {
	AiParams  ai.AiOrder
	Products  ItemsResolve
	Gifts     ItemsResolve
	Address   string
	Customer  string
	Time      time.Time
	DefTime   string
	Current   Resolve
	Note      string
	UpdatedAt time.Time
	Editing   bool
	Canceled  bool

	User *database.User
}

func NewOrderResolve(ctx Context) *OrderResolve {
	resolve := new(OrderResolve)
	resolve.Touch()

	aiResult := ctx.Value("Result").(apiai.Result)

	resolve.AiParams = ai.ApiAiOrder{AiResult: aiResult}
	resolve.ExtractFromParams()

	if viewer := ctx.Value("Viewer"); viewer != nil {
		user := viewer.(*database.User)
		resolve.User = user
	}

	return resolve
}

func (r *OrderResolve) Solve(aiResult apiai.Result) string {
	return r.Answer()
}

func (r *OrderResolve) Touch() {
	r.UpdatedAt = time.Now()
}

func (r OrderResolve) Modifable(expireMin int) bool {
	return !r.Expired(expireMin) || r.Submited()
}

// TODO
func (r OrderResolve) Cancelable() bool {
	return true
}

// TODO
func (r *OrderResolve) Cancel() bool {
	r.Canceled = true
	return true
}

func (r OrderResolve) Fulfiled() bool {
	return len(r.Products.Products) > 0 && (r.Address != "" || r.Customer != "")
}

func (r OrderResolve) Expired(expireMin int) bool {
	return r.UpdatedAt.Add(time.Duration(expireMin)*time.Minute).UnixNano() < time.Now().UnixNano()
}

// TODO
func (r OrderResolve) Submited() bool {
	return false
}

// 从ｌｕｉｓ数据构造结构数据
func (r *OrderResolve) ExtractFromParams() {
	r.ExtractItems()
	r.ExtractGiftItems()
	r.ExtractAddress()
	r.ExtractCustomer()
	r.ExtractTime()
	r.ExtractNote()
}

func (r *OrderResolve) ExtractItems() {
	for _, i := range r.AiParams.Items() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Products.Products = append(r.Products.Products, item)
	}
}

func (r *OrderResolve) ExtractGiftItems() {
	for _, i := range r.AiParams.GiftItems() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Gifts.Products = append(r.Gifts.Products, item)
	}
}

func (r *OrderResolve) ExtractAddress() {
	r.Address = r.AiParams.Address()
}

func (r *OrderResolve) ExtractCustomer() {
	r.Customer = r.AiParams.Customer()
}

func (r *OrderResolve) ExtractTime() {
	r.Time = r.AiParams.Time()
}

func (r *OrderResolve) ExtractNote() {
	r.Note = r.AiParams.Note()
}

func (r *OrderResolve) SetDefTime(t string) {
	r.DefTime = t

	if r.Time.IsZero() && r.DefTime != "" {
		r.SetTimeByDef()
	}
}

func (r *OrderResolve) SetTimeByDef() {
	if r.DefTime == "今天" {
		r.Time = time.Now()
	} else if r.DefTime == "明天" {
		r.Time = time.Now().Add(24 * time.Hour)
	}
}

func (r OrderResolve) EmptyProducts() bool {
	return len(r.Products.Products) == 0
}

func (r OrderResolve) Answer() string {
	if r.Fulfiled() {
		return r.PostOrderAndAnswer()
	} else {
		return r.AnswerHead() + r.AnswerFooter("", "")
	}
}

func (r *OrderResolve) PostOrderAndAnswer() string {
	items := make([]database.OrderItem, 0, 0)
	gifts := make([]database.GiftItem, 0, 0)

	for _, pr := range r.Products.Products {
		item = database.NewOrderItem("", pr.Product, pr.Product, pr.Price)
		items = append(items, item)
	}

	for _, pr := range r.Gifts.Products {
		gift = database.NewGiftItem("", pr.Product, pr.Quantity)
		gifts = append(items, gift)
	}

	order, err := r.User.CreateSaledOrder(r.Address, r.Note, r.Time, 0, items, gifts)

	if err != nil {
		return err.Error()
	} else {
		return r.AnswerHead() + r.AnswerBody() + r.AnswerFooter(order.No, order.ID)
	}
}

func (r OrderResolve) AddressInfo() string {
	if r.Address != "" && r.Customer != "" {
		return "地址:" + r.Address + r.Customer + "\n"
	} else if r.Address != "" {
		return "地址:" + r.Address + "\n"
	} else if r.Customer != "" {
		return "客户:" + r.Customer + "\n"
	} else {
		return ""
	}
}

func (r OrderResolve) AnswerHead() string {
	desc := "订单正在处理, 已经添加" + CnNum(len(r.Products.Products)) + "种产品"

	if r.Fulfiled() {
		desc = "订单已经生成, 共" + CnNum(len(r.Products.Products)) + "种产品"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + ", " + CnNum(len(r.Gifts.Products)) + "种赠品" + "\n"
	} else {
		desc = desc + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerBody() string {
	desc := ""

	for _, p := range r.Products.Products {
		desc = desc + p.Product + " " + strconv.Itoa(p.Quantity) + "件\n"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + "申请的赠品:\n"

		for _, g := range r.Gifts.Products {
			desc = desc + g.Product + " " + strconv.Itoa(g.Quantity) + "件\n"
		}
	}

	desc = desc + "时间:" + r.Time.Format("2006年01月02日") + "\n"

	if r.Note != "" {
		desc = desc + "备注：" + r.Note + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerFooter(no, id interface{}) string {
	desc := ""

	if r.Fulfiled() {
		desc = desc + r.AddressInfo()
		desc = desc + "订单已经生成，订单号为：" + fmt.Sprint(no) + "\n"
		desc = desc + "订单入口: http://wanliu.biz/orders/" + fmt.Sprint(id)
	} else {
		desc = desc + "还缺少收货地址或客户信息\n"
	}

	return desc
}

func CnNum(num int) string {
	switch num {
	case 1:
		return "一"
	case 2:
		return "两"
	case 3:
		return "三"
	case 4:
		return "四"
	case 5:
		return "五"
	case 6:
		return "六"
	case 7:
		return "七"
	case 8:
		return "八"
	case 9:
		return "九"
	case 10:
		return "十"
	default:
		return strconv.Itoa(num)
	}
}
<|endoftext|>


<commit_before>package categories

// Category ...
type Category struct{
	name string
}<commit_msg>Go fmt<commit_after>package categories

// Category ...
type Category struct {
	name string
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"testing"
	"net/http"
)

var edgeHost = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	sourceUrl := fmt.Sprintf("http://%s/", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/", *edgeHost)

	client := &http.Transport{}
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Error(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %s", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}

	t.Error("Not implemented test to confirm that it doesn't hit origin")
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}
<commit_msg>Use t.Fatal() on request error<commit_after>package main

import (
	"flag"
	"fmt"
	"testing"
	"net/http"
)

var edgeHost = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	sourceUrl := fmt.Sprintf("http://%s/", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/", *edgeHost)

	client := &http.Transport{}
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %s", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}

	t.Error("Not implemented test to confirm that it doesn't hit origin")
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}
<|endoftext|>


<commit_before>package dbActions

import (
	"errors"
	"github.com/HRODEV/project7_8/models"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
)

func GetUserByID(id uint, user *models.User, db *gorm.DB) {
	db.First(user, id)
}

func CreateUser(user *models.User, db *gorm.DB) error {
	valid, err := user.IsValid()

	if valid {
		db.Create(user)
		return nil
	} else {
		return err
	}
}
<commit_msg>Remove unused import (sorry Travis)<commit_after>package dbActions

import (
	"github.com/HRODEV/project7_8/models"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
)

func GetUserByID(id uint, user *models.User, db *gorm.DB) {
	db.First(user, id)
}

func CreateUser(user *models.User, db *gorm.DB) error {
	valid, err := user.IsValid()

	if valid {
		db.Create(user)
		return nil
	} else {
		return err
	}
}
<|endoftext|>


<commit_before>package arn

import (
	"errors"
	"fmt"
	"reflect"

	"github.com/aerogo/aero"
	"github.com/aerogo/api"
)

// Force interface implementations
var (
	_ Publishable            = (*AMV)(nil)
	_ Likeable               = (*AMV)(nil)
	_ LikeEventReceiver      = (*AMV)(nil)
	_ fmt.Stringer           = (*AMV)(nil)
	_ api.Newable            = (*AMV)(nil)
	_ api.Editable           = (*AMV)(nil)
	_ api.Deletable          = (*AMV)(nil)
	_ api.ArrayEventListener = (*AMV)(nil)
)

// Actions
func init() {
	API.RegisterActions("AMV", []*api.Action{
		// Publish
		PublishAction(),

		// Unpublish
		UnpublishAction(),

		// Like
		LikeAction(),

		// Unlike
		UnlikeAction(),
	})
}

// Create sets the data for a new AMV with data we received from the API request.
func (amv *AMV) Create(ctx *aero.Context) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	amv.ID = GenerateID("AMV")
	amv.Created = DateTimeUTC()
	amv.CreatedBy = user.ID

	// Write log entry
	logEntry := NewEditLogEntry(user.ID, "create", "AMV", amv.ID, "", "", "")
	logEntry.Save()

	return amv.Unpublish()
}

// Edit updates the external media object.
func (amv *AMV) Edit(ctx *aero.Context, key string, value reflect.Value, newValue reflect.Value) (bool, error) {
	user := GetUserFromContext(ctx)

	// Write log entry
	logEntry := NewEditLogEntry(user.ID, "edit", "AMV", amv.ID, key, fmt.Sprint(value.Interface()), fmt.Sprint(newValue.Interface()))
	logEntry.Save()

	return false, nil
}

// OnAppend saves a log entry.
func (amv *AMV) OnAppend(ctx *aero.Context, key string, index int, obj interface{}) {
	user := GetUserFromContext(ctx)
	logEntry := NewEditLogEntry(user.ID, "arrayAppend", "AMV", amv.ID, fmt.Sprintf("%s[%d]", key, index), "", fmt.Sprint(obj))
	logEntry.Save()
}

// OnRemove saves a log entry.
func (amv *AMV) OnRemove(ctx *aero.Context, key string, index int, obj interface{}) {
	user := GetUserFromContext(ctx)
	logEntry := NewEditLogEntry(user.ID, "arrayRemove", "AMV", amv.ID, fmt.Sprintf("%s[%d]", key, index), fmt.Sprint(obj), "")
	logEntry.Save()
}

// AfterEdit updates the metadata.
func (amv *AMV) AfterEdit(ctx *aero.Context) error {
	amv.Edited = DateTimeUTC()
	amv.EditedBy = GetUserFromContext(ctx).ID
	return nil
}

// DeleteInContext deletes the amv in the given context.
func (amv *AMV) DeleteInContext(ctx *aero.Context) error {
	user := GetUserFromContext(ctx)

	// Write log entry
	logEntry := NewEditLogEntry(user.ID, "delete", "AMV", amv.ID, "", fmt.Sprint(amv), "")
	logEntry.Save()

	return amv.Delete()
}

// Delete deletes the object from the database.
func (amv *AMV) Delete() error {
	if amv.IsDraft {
		draftIndex := amv.Creator().DraftIndex()
		draftIndex.AMVID = ""
		draftIndex.Save()
	}

	DB.Delete("AMV", amv.ID)
	return nil
}

// Authorize returns an error if the given API POST request is not authorized.
func (amv *AMV) Authorize(ctx *aero.Context, action string) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	if action == "delete" {
		if user.Role != "editor" && user.Role != "admin" {
			return errors.New("Insufficient permissions")
		}
	}

	return nil
}

// Save saves the amv object in the database.
func (amv *AMV) Save() {
	DB.Set("AMV", amv.ID, amv)
}
<commit_msg>Fixed AMV deletion<commit_after>package arn

import (
	"errors"
	"fmt"
	"os"
	"path"
	"reflect"

	"github.com/aerogo/aero"
	"github.com/aerogo/api"
)

// Force interface implementations
var (
	_ Publishable            = (*AMV)(nil)
	_ Likeable               = (*AMV)(nil)
	_ LikeEventReceiver      = (*AMV)(nil)
	_ fmt.Stringer           = (*AMV)(nil)
	_ api.Newable            = (*AMV)(nil)
	_ api.Editable           = (*AMV)(nil)
	_ api.Deletable          = (*AMV)(nil)
	_ api.ArrayEventListener = (*AMV)(nil)
)

// Actions
func init() {
	API.RegisterActions("AMV", []*api.Action{
		// Publish
		PublishAction(),

		// Unpublish
		UnpublishAction(),

		// Like
		LikeAction(),

		// Unlike
		UnlikeAction(),
	})
}

// Create sets the data for a new AMV with data we received from the API request.
func (amv *AMV) Create(ctx *aero.Context) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	amv.ID = GenerateID("AMV")
	amv.Created = DateTimeUTC()
	amv.CreatedBy = user.ID

	// Write log entry
	logEntry := NewEditLogEntry(user.ID, "create", "AMV", amv.ID, "", "", "")
	logEntry.Save()

	return amv.Unpublish()
}

// Edit updates the external media object.
func (amv *AMV) Edit(ctx *aero.Context, key string, value reflect.Value, newValue reflect.Value) (bool, error) {
	user := GetUserFromContext(ctx)

	// Write log entry
	logEntry := NewEditLogEntry(user.ID, "edit", "AMV", amv.ID, key, fmt.Sprint(value.Interface()), fmt.Sprint(newValue.Interface()))
	logEntry.Save()

	return false, nil
}

// OnAppend saves a log entry.
func (amv *AMV) OnAppend(ctx *aero.Context, key string, index int, obj interface{}) {
	user := GetUserFromContext(ctx)
	logEntry := NewEditLogEntry(user.ID, "arrayAppend", "AMV", amv.ID, fmt.Sprintf("%s[%d]", key, index), "", fmt.Sprint(obj))
	logEntry.Save()
}

// OnRemove saves a log entry.
func (amv *AMV) OnRemove(ctx *aero.Context, key string, index int, obj interface{}) {
	user := GetUserFromContext(ctx)
	logEntry := NewEditLogEntry(user.ID, "arrayRemove", "AMV", amv.ID, fmt.Sprintf("%s[%d]", key, index), fmt.Sprint(obj), "")
	logEntry.Save()
}

// AfterEdit updates the metadata.
func (amv *AMV) AfterEdit(ctx *aero.Context) error {
	amv.Edited = DateTimeUTC()
	amv.EditedBy = GetUserFromContext(ctx).ID
	return nil
}

// DeleteInContext deletes the amv in the given context.
func (amv *AMV) DeleteInContext(ctx *aero.Context) error {
	user := GetUserFromContext(ctx)

	// Write log entry
	logEntry := NewEditLogEntry(user.ID, "delete", "AMV", amv.ID, "", fmt.Sprint(amv), "")
	logEntry.Save()

	return amv.Delete()
}

// Delete deletes the object from the database.
func (amv *AMV) Delete() error {
	if amv.IsDraft {
		draftIndex := amv.Creator().DraftIndex()
		draftIndex.AMVID = ""
		draftIndex.Save()
	}

	if amv.File != "" {
		err := os.Remove(path.Join(Root, "videos", "amvs", amv.File))

		if err != nil {
			return err
		}
	}

	DB.Delete("AMV", amv.ID)
	return nil
}

// Authorize returns an error if the given API POST request is not authorized.
func (amv *AMV) Authorize(ctx *aero.Context, action string) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	if action == "delete" {
		if user.Role != "editor" && user.Role != "admin" {
			return errors.New("Insufficient permissions")
		}
	}

	return nil
}

// Save saves the amv object in the database.
func (amv *AMV) Save() {
	DB.Set("AMV", amv.ID, amv)
}
<|endoftext|>


<commit_before><commit_msg>fix magnet test<commit_after><|endoftext|>


<commit_before>package htcat

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"sync"
)

const (
	_        = iota
	kB int64 = 1 << (10 * iota)
	mB
	gB
	tB
	pB
	eB
)

type HtCat struct {
	io.WriterTo
	d     defrag
	u     *url.URL
	cl    *http.Client
	tasks chan *httpFrag

	// Protect httpFragGen with a Mutex.
	httpFragGenMu sync.Mutex
	hfg           httpFragGen
}

type HttpStatusError struct {
	error
	Status string
}

func (cat *HtCat) startup(parallelism int) {
	req := http.Request{
		Method:     "GET",
		URL:        cat.u,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Body:       nil,
		Host:       cat.u.Host,
	}

	resp, err := cat.cl.Do(&req)
	if err != nil {
		go cat.d.cancel(err)
		return
	}

	// Check for non-200 OK response codes from the startup-GET.
	if resp.Status != "200 OK" {
		err = HttpStatusError{
			error: fmt.Errorf(
				"Expected HTTP Status 200, received: %q",
				resp.Status),
			Status: resp.Status}
		go cat.d.cancel(err)
		return
	}

	l := resp.Header.Get("Content-Length")

	// Some kinds of small or indeterminate-length files will
	// receive no parallelism.  This procedure helps prepare the
	// HtCat value for a one-HTTP-Request GET.
	noParallel := func(wtc writerToCloser) {
		f := cat.d.nextFragment()
		cat.d.setLast(cat.d.lastAllocated())
		f.contents = wtc
		cat.d.register(f)
	}

	if l == "" {
		// No Content-Length, stream without parallelism nor
		// assumptions about the length of the stream.
		go noParallel(struct {
			io.WriterTo
			io.Closer
		}{
			WriterTo: bufio.NewReader(resp.Body),
			Closer:   resp.Body,
		})
		return
	}

	length, err := strconv.ParseInt(l, 10, 64)
	if err != nil {
		// Invalid integer for Content-Length, defer reporting
		// the error until a WriteTo call is made.
		go cat.d.cancel(err)
		return
	}

	// Set up httpFrag generator state.
	cat.hfg.totalSize = length
	cat.hfg.targetFragSize = length / int64(parallelism)
	if cat.hfg.targetFragSize > 20*mB {
		cat.hfg.targetFragSize = 20 * mB
	}

	// Very small fragments are probably not worthwhile to start
	// up new requests for, but it in this case it was possible to
	// ascertain the size, so take advantage of that to start
	// reading in the background as eagerly as possible.
	if cat.hfg.targetFragSize < 1*mB {
		cat.hfg.curPos = cat.hfg.totalSize
		er := newEagerReader(resp.Body, cat.hfg.totalSize)
		go noParallel(er)
		go er.WaitClosed()
		return
	}

	// None of the other special short-circuit cases have been
	// triggered, so begin preparation for full-blown parallel
	// GET.  One GET worker is started here to take advantage of
	// the already pending response (which has no determinate
	// length, so it must be limited).
	hf := cat.nextFragment()
	go func() {
		er := newEagerReader(
			struct {
				io.Reader
				io.Closer
			}{
				Reader: io.LimitReader(resp.Body, hf.size),
				Closer: resp.Body,
			},
			hf.size)

		hf.fragment.contents = er
		cat.d.register(hf.fragment)
		er.WaitClosed()

		// Chain into being a regular worker, having finished
		// the special start-up segment.
		cat.get()
	}()

}

func New(client *http.Client, u *url.URL, parallelism int) *HtCat {
	cat := HtCat{
		u:  u,
		cl: client,
	}

	cat.d.initDefrag()
	cat.WriterTo = &cat.d
	cat.startup(parallelism)

	if cat.hfg.curPos == cat.hfg.totalSize {
		return &cat
	}

	// Start background workers.
	//
	// "startup" starts one worker that is specially constructed
	// to deal with the first request, so back off by one to
	// prevent performing with too much parallelism.
	for i := 1; i < parallelism; i += 1 {
		go cat.get()
	}

	return &cat
}

func (cat *HtCat) nextFragment() *httpFrag {
	cat.httpFragGenMu.Lock()
	defer cat.httpFragGenMu.Unlock()

	var hf *httpFrag

	if cat.hfg.hasNext() {
		f := cat.d.nextFragment()
		hf = cat.hfg.nextFragment(f)
	} else {
		cat.d.setLast(cat.d.lastAllocated())
	}

	return hf
}

func (cat *HtCat) get() {
	for {
		hf := cat.nextFragment()
		if hf == nil {
			return
		}

		req := http.Request{
			Method:     "GET",
			URL:        cat.u,
			Proto:      "HTTP/1.1",
			ProtoMajor: 1,
			ProtoMinor: 1,
			Header:     hf.header,
			Body:       nil,
			Host:       cat.u.Host,
		}

		resp, err := cat.cl.Do(&req)
		if err != nil {
			cat.d.cancel(err)
			return
		}

		// Check for an acceptable HTTP status code.
		if !(resp.Status == "206 Partial Content" ||
			resp.Status == "200 OK") {
			err = HttpStatusError{
				error: fmt.Errorf("Expected HTTP Status "+
					"206 or 200, received: %q",
					resp.Status),
				Status: resp.Status}
			go cat.d.cancel(err)
			return
		}

		er := newEagerReader(resp.Body, hf.size)
		hf.fragment.contents = er
		cat.d.register(hf.fragment)
		er.WaitClosed()
	}
}
<commit_msg>Remove line break<commit_after>package htcat

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"sync"
)

const (
	_        = iota
	kB int64 = 1 << (10 * iota)
	mB
	gB
	tB
	pB
	eB
)

type HtCat struct {
	io.WriterTo
	d     defrag
	u     *url.URL
	cl    *http.Client
	tasks chan *httpFrag

	// Protect httpFragGen with a Mutex.
	httpFragGenMu sync.Mutex
	hfg           httpFragGen
}

type HttpStatusError struct {
	error
	Status string
}

func (cat *HtCat) startup(parallelism int) {
	req := http.Request{
		Method:     "GET",
		URL:        cat.u,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Body:       nil,
		Host:       cat.u.Host,
	}

	resp, err := cat.cl.Do(&req)
	if err != nil {
		go cat.d.cancel(err)
		return
	}

	// Check for non-200 OK response codes from the startup-GET.
	if resp.Status != "200 OK" {
		err = HttpStatusError{
			error: fmt.Errorf(
				"Expected HTTP Status 200, received: %q",
				resp.Status),
			Status: resp.Status}
		go cat.d.cancel(err)
		return
	}

	l := resp.Header.Get("Content-Length")

	// Some kinds of small or indeterminate-length files will
	// receive no parallelism.  This procedure helps prepare the
	// HtCat value for a one-HTTP-Request GET.
	noParallel := func(wtc writerToCloser) {
		f := cat.d.nextFragment()
		cat.d.setLast(cat.d.lastAllocated())
		f.contents = wtc
		cat.d.register(f)
	}

	if l == "" {
		// No Content-Length, stream without parallelism nor
		// assumptions about the length of the stream.
		go noParallel(struct {
			io.WriterTo
			io.Closer
		}{
			WriterTo: bufio.NewReader(resp.Body),
			Closer:   resp.Body,
		})
		return
	}

	length, err := strconv.ParseInt(l, 10, 64)
	if err != nil {
		// Invalid integer for Content-Length, defer reporting
		// the error until a WriteTo call is made.
		go cat.d.cancel(err)
		return
	}

	// Set up httpFrag generator state.
	cat.hfg.totalSize = length
	cat.hfg.targetFragSize = length / int64(parallelism)
	if cat.hfg.targetFragSize > 20*mB {
		cat.hfg.targetFragSize = 20 * mB
	}

	// Very small fragments are probably not worthwhile to start
	// up new requests for, but it in this case it was possible to
	// ascertain the size, so take advantage of that to start
	// reading in the background as eagerly as possible.
	if cat.hfg.targetFragSize < 1*mB {
		cat.hfg.curPos = cat.hfg.totalSize
		er := newEagerReader(resp.Body, cat.hfg.totalSize)
		go noParallel(er)
		go er.WaitClosed()
		return
	}

	// None of the other special short-circuit cases have been
	// triggered, so begin preparation for full-blown parallel
	// GET.  One GET worker is started here to take advantage of
	// the already pending response (which has no determinate
	// length, so it must be limited).
	hf := cat.nextFragment()
	go func() {
		er := newEagerReader(
			struct {
				io.Reader
				io.Closer
			}{
				Reader: io.LimitReader(resp.Body, hf.size),
				Closer: resp.Body,
			},
			hf.size)

		hf.fragment.contents = er
		cat.d.register(hf.fragment)
		er.WaitClosed()

		// Chain into being a regular worker, having finished
		// the special start-up segment.
		cat.get()
	}()
}

func New(client *http.Client, u *url.URL, parallelism int) *HtCat {
	cat := HtCat{
		u:  u,
		cl: client,
	}

	cat.d.initDefrag()
	cat.WriterTo = &cat.d
	cat.startup(parallelism)

	if cat.hfg.curPos == cat.hfg.totalSize {
		return &cat
	}

	// Start background workers.
	//
	// "startup" starts one worker that is specially constructed
	// to deal with the first request, so back off by one to
	// prevent performing with too much parallelism.
	for i := 1; i < parallelism; i += 1 {
		go cat.get()
	}

	return &cat
}

func (cat *HtCat) nextFragment() *httpFrag {
	cat.httpFragGenMu.Lock()
	defer cat.httpFragGenMu.Unlock()

	var hf *httpFrag

	if cat.hfg.hasNext() {
		f := cat.d.nextFragment()
		hf = cat.hfg.nextFragment(f)
	} else {
		cat.d.setLast(cat.d.lastAllocated())
	}

	return hf
}

func (cat *HtCat) get() {
	for {
		hf := cat.nextFragment()
		if hf == nil {
			return
		}

		req := http.Request{
			Method:     "GET",
			URL:        cat.u,
			Proto:      "HTTP/1.1",
			ProtoMajor: 1,
			ProtoMinor: 1,
			Header:     hf.header,
			Body:       nil,
			Host:       cat.u.Host,
		}

		resp, err := cat.cl.Do(&req)
		if err != nil {
			cat.d.cancel(err)
			return
		}

		// Check for an acceptable HTTP status code.
		if !(resp.Status == "206 Partial Content" ||
			resp.Status == "200 OK") {
			err = HttpStatusError{
				error: fmt.Errorf("Expected HTTP Status "+
					"206 or 200, received: %q",
					resp.Status),
				Status: resp.Status}
			go cat.d.cancel(err)
			return
		}

		er := newEagerReader(resp.Body, hf.size)
		hf.fragment.contents = er
		cat.d.register(hf.fragment)
		er.WaitClosed()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/dullgiulio/kuradns/cfg"
	"github.com/dullgiulio/kuradns/gen"
)

var errUnhandledURL = errors.New("unhandled URL")

func (s *server) handleHttpError(w http.ResponseWriter, r *http.Request, err error) {
	http.Error(w, "An error occurred; please refer to the logs for more information", 500)
	log.Printf("[error] http: %s %s %s: %s", r.RemoteAddr, r.Method, r.URL.Path, err)
}

func (s *server) handleSourceAdd(name, gentype string, conf *cfg.Config) error {
	src := newSource(name, conf)
	gen, err := gen.MakeGenerator(gentype, conf)
	if err != nil {
		return fmt.Errorf("cannot start generator: %s", err)
	}
	src.gen = gen

	req := makeRequest(src, reqtypeAdd)

	if err := req.send(s.requests); err != nil {
		return fmt.Errorf("cannot process %s: %s", req.String(), err)
	}
	err = <-req.resp
	if err != nil {
		return fmt.Errorf("cannot add source: %s", err)
	}
	return nil
}

func (s *server) handleSourceDelete(name string) error {
	src := newSource(name, nil)
	req := makeRequest(src, reqtypeDel)

	if err := req.send(s.requests); err != nil {
		return fmt.Errorf("cannot process %s: %s", req.String(), err)
	}
	err := <-req.resp
	if err != nil {
		return fmt.Errorf("cannot remove source: %s", err)
	}
	return err
}

func (s *server) handleSourceUpdate(name string) error {
	src := newSource(name, nil)
	req := makeRequest(src, reqtypeUp)

	if err := req.send(s.requests); err != nil {
		return fmt.Errorf("cannot process %s: %s", req.String(), err)
	}
	err := <-req.resp
	if err != nil {
		return fmt.Errorf("cannot update source: %s", err)
	}
	return nil
}

func (s *server) handleDnsDump(w http.ResponseWriter, r *http.Request) error {
	w.Header().Set("Content-Type", "text/plain")
	wb := bufio.NewWriter(w)

	s.mux.RLock()
	defer s.mux.RUnlock()

	if err := s.repo.WriteTo(wb); err != nil {
		return err
	}

	return wb.Flush()
}

func (s *server) handleSourceList(w http.ResponseWriter, r *http.Request) error {
	w.Header().Set("Content-Type", "text/plain")
	wb := bufio.NewWriter(w)

	s.mux.RLock()
	defer s.mux.RUnlock()

	for _, src := range s.srcs {
		fmt.Fprintf(wb, "%s %s\n", src.name, src.conf.GetVal("source.type", "unknown"))
	}

	return wb.Flush()
}

// take last value in case of duplicates
func (s *server) configFromForm(cf *cfg.Config, form url.Values) error {
	for k, vs := range form {
		if strings.HasPrefix(k, "config.") || strings.HasPrefix(k, "source.") {
			cf.Put(k, vs[len(vs)-1])
		}
	}
	return nil
}

func (s *server) configFromJSON(cf *cfg.Config, r io.Reader) error {
	if err := cf.FromJSON(r); err != nil {
		return fmt.Errorf("cannot parse JSON: %s", err)
	}
	return nil
}

func (s *server) getFromConf(cf *cfg.Config, key string) (string, error) {
	if v, ok := cf.Get(key); ok {
		return v, nil
	}
	return "", fmt.Errorf("required parameter %s not found", key)
}

func (s *server) parseBodyData(w http.ResponseWriter, r *http.Request) (*cfg.Config, error) {
	cf := cfg.NewConfig()
	// JSON data as POST body
	if r.Header.Get("Content-Type") == "application/json" {
		return cf, s.configFromJSON(cf, r.Body)
	}
	// Normal URL-encoded form
	if err := r.ParseForm(); err != nil {
		return cf, fmt.Errorf("cannot parse form: %s", err)
	}
	return cf, s.configFromForm(cf, r.Form)
}

func (s *server) httpHandlePOST(w http.ResponseWriter, r *http.Request) error {
	conf, err := s.parseBodyData(w, r)
	if err != nil {
		return err
	}
	conf.Put("dns.zone", s.zone.browser())
	conf.Put("dns.self", s.self.browser())

	switch r.URL.Path {
	case "/source/list":
		return s.handleSourceList(w, r)
	case "/source/add":
		sname, err := s.getFromConf(conf, "source.name")
		if err != nil {
			return err
		}
		stype, err := s.getFromConf(conf, "source.type")
		if err != nil {
			return err
		}
		return s.handleSourceAdd(sname, stype, conf)
	case "/source/delete":
		sname, err := s.getFromConf(conf, "source.name")
		if err != nil {
			return err
		}
		return s.handleSourceDelete(sname)
	case "/source/update":
		sname, err := s.getFromConf(conf, "source.name")
		if err != nil {
			return err
		}
		return s.handleSourceUpdate(sname)
	}
	return errUnhandledURL
}

func (s *server) httpHandleGET(w http.ResponseWriter, r *http.Request) error {
	// TODO: Help, status, etc.
	switch r.URL.Path {
	case "/dns/dump":
		return s.handleDnsDump(w, r)
	case "/favicon.ico":
		// Shut up on bogus requests
		http.NotFound(w, r)
		return nil
	}
	return errUnhandledURL
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var err error
	if s.verbose {
		log.Printf("[info] http: tcp(%s): request %s %s", r.RemoteAddr, r.Method, r.URL.Path)
	}
	switch r.Method {
	case "POST", "PUT":
		err = s.httpHandlePOST(w, r)
	default:
		err = s.httpHandleGET(w, r)
	}
	if err != nil {
		s.handleHttpError(w, r, err)
	}
}
<commit_msg>http: sources/list is a GET method<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/dullgiulio/kuradns/cfg"
	"github.com/dullgiulio/kuradns/gen"
)

var errUnhandledURL = errors.New("unhandled URL")

func (s *server) handleHttpError(w http.ResponseWriter, r *http.Request, err error) {
	http.Error(w, "An error occurred; please refer to the logs for more information", 500)
	log.Printf("[error] http: %s %s %s: %s", r.RemoteAddr, r.Method, r.URL.Path, err)
}

func (s *server) handleSourceAdd(name, gentype string, conf *cfg.Config) error {
	src := newSource(name, conf)
	gen, err := gen.MakeGenerator(gentype, conf)
	if err != nil {
		return fmt.Errorf("cannot start generator: %s", err)
	}
	src.gen = gen

	req := makeRequest(src, reqtypeAdd)

	if err := req.send(s.requests); err != nil {
		return fmt.Errorf("cannot process %s: %s", req.String(), err)
	}
	err = <-req.resp
	if err != nil {
		return fmt.Errorf("cannot add source: %s", err)
	}
	return nil
}

func (s *server) handleSourceDelete(name string) error {
	src := newSource(name, nil)
	req := makeRequest(src, reqtypeDel)

	if err := req.send(s.requests); err != nil {
		return fmt.Errorf("cannot process %s: %s", req.String(), err)
	}
	err := <-req.resp
	if err != nil {
		return fmt.Errorf("cannot remove source: %s", err)
	}
	return err
}

func (s *server) handleSourceUpdate(name string) error {
	src := newSource(name, nil)
	req := makeRequest(src, reqtypeUp)

	if err := req.send(s.requests); err != nil {
		return fmt.Errorf("cannot process %s: %s", req.String(), err)
	}
	err := <-req.resp
	if err != nil {
		return fmt.Errorf("cannot update source: %s", err)
	}
	return nil
}

func (s *server) handleDnsDump(w http.ResponseWriter, r *http.Request) error {
	w.Header().Set("Content-Type", "text/plain")
	wb := bufio.NewWriter(w)

	s.mux.RLock()
	defer s.mux.RUnlock()

	if err := s.repo.WriteTo(wb); err != nil {
		return err
	}

	return wb.Flush()
}

func (s *server) handleSourceList(w http.ResponseWriter, r *http.Request) error {
	w.Header().Set("Content-Type", "text/plain")
	wb := bufio.NewWriter(w)

	s.mux.RLock()
	defer s.mux.RUnlock()

	for _, src := range s.srcs {
		fmt.Fprintf(wb, "%s %s\n", src.name, src.conf.GetVal("source.type", "unknown"))
	}

	return wb.Flush()
}

// take last value in case of duplicates
func (s *server) configFromForm(cf *cfg.Config, form url.Values) error {
	for k, vs := range form {
		if strings.HasPrefix(k, "config.") || strings.HasPrefix(k, "source.") {
			cf.Put(k, vs[len(vs)-1])
		}
	}
	return nil
}

func (s *server) configFromJSON(cf *cfg.Config, r io.Reader) error {
	if err := cf.FromJSON(r); err != nil {
		return fmt.Errorf("cannot parse JSON: %s", err)
	}
	return nil
}

func (s *server) getFromConf(cf *cfg.Config, key string) (string, error) {
	if v, ok := cf.Get(key); ok {
		return v, nil
	}
	return "", fmt.Errorf("required parameter %s not found", key)
}

func (s *server) parseBodyData(w http.ResponseWriter, r *http.Request) (*cfg.Config, error) {
	cf := cfg.NewConfig()
	// JSON data as POST body
	if r.Header.Get("Content-Type") == "application/json" {
		return cf, s.configFromJSON(cf, r.Body)
	}
	// Normal URL-encoded form
	if err := r.ParseForm(); err != nil {
		return cf, fmt.Errorf("cannot parse form: %s", err)
	}
	return cf, s.configFromForm(cf, r.Form)
}

func (s *server) httpHandlePOST(w http.ResponseWriter, r *http.Request) error {
	conf, err := s.parseBodyData(w, r)
	if err != nil {
		return err
	}
	conf.Put("dns.zone", s.zone.browser())
	conf.Put("dns.self", s.self.browser())

	switch r.URL.Path {
	case "/source/add":
		sname, err := s.getFromConf(conf, "source.name")
		if err != nil {
			return err
		}
		stype, err := s.getFromConf(conf, "source.type")
		if err != nil {
			return err
		}
		return s.handleSourceAdd(sname, stype, conf)
	case "/source/delete":
		sname, err := s.getFromConf(conf, "source.name")
		if err != nil {
			return err
		}
		return s.handleSourceDelete(sname)
	case "/source/update":
		sname, err := s.getFromConf(conf, "source.name")
		if err != nil {
			return err
		}
		return s.handleSourceUpdate(sname)
	}
	return errUnhandledURL
}

func (s *server) httpHandleGET(w http.ResponseWriter, r *http.Request) error {
	// TODO: Help, status, etc.
	switch r.URL.Path {
	case "/source/list":
		return s.handleSourceList(w, r)
	case "/dns/dump":
		return s.handleDnsDump(w, r)
	case "/favicon.ico":
		// Shut up on bogus requests
		http.NotFound(w, r)
		return nil
	}
	return errUnhandledURL
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var err error
	if s.verbose {
		log.Printf("[info] http: tcp(%s): request %s %s", r.RemoteAddr, r.Method, r.URL.Path)
	}
	switch r.Method {
	case "POST", "PUT":
		err = s.httpHandlePOST(w, r)
	default:
		err = s.httpHandleGET(w, r)
	}
	if err != nil {
		s.handleHttpError(w, r, err)
	}
}
<|endoftext|>


<commit_before>package crawler

import (
	"gopkg.in/mgo.v2/bson"
	"strconv"
	"time"

	"github.com/mindcastio/mindcastio/backend"

	"github.com/mindcastio/mindcastio/backend/datastore"
	"github.com/mindcastio/mindcastio/backend/logger"
	"github.com/mindcastio/mindcastio/backend/metrics"
	"github.com/mindcastio/mindcastio/backend/util"
)

func SchedulePodcastCrawling() {
	logger.Log("mindcast.crawler.schedule_podcast_crawling")

	// search for podcasts that are candidates for crawling
	expired := searchExpiredPodcasts(backend.DEFAULT_UPDATE_BATCH)
	count := len(expired)

	logger.Log("mindcast.crawler.schedule_podcast_crawling.scheduling", strconv.FormatInt((int64)(count), 10))

	if count > 0 {
		for i := 0; i < count; i++ {
			//go CrawlPodcastFeed(expired[i].Uid)
			CrawlPodcastFeed(expired[i].Uid)
			// FIXME use go routine or not ?
		}
		metrics.Count("crawler.scheduled", count)
	}

	logger.Log("mindcast.crawler.schedule_podcast_crawling.done")
}

func CrawlPodcastFeed(uid string) {

	start_1 := time.Now()
	logger.Log("crawl_podcast_feed", uid)

	idx := backend.IndexLookup(uid)
	if idx == nil {
		logger.Error("crawl_podcast_feed.error.1", nil, uid)
		metrics.Error("crawl_podcast_feed.error", "", []string{uid})
		return
	}

	// HINT: ignore the fact that the item might be disables idx.erros > ...

	// fetch the podcast feed
	start_2 := time.Now()
	podcast, err := ParsePodcastFeed(idx.Feed)
	metrics.Histogram("crawler.parse_feed", (float64)(util.ElapsedTimeSince(start_2)))

	if err != nil {
		suspended, _ := backend.IndexBackoff(uid)

		if suspended {
			logger.Error("crawl_podcast_feed.suspended", err, uid, idx.Feed)
			metrics.Error("crawl_podcast_feed.suspended", err.Error(), []string{uid, idx.Feed})
		}

		return
	}

	// add to podcast metadata index
	is_new, err := podcastAdd(podcast)
	if err != nil {
		logger.Error("crawl_podcast_feed.error.3", err, uid, idx.Feed)
		metrics.Error("crawl_podcast_feed.error", err.Error(), []string{uid, idx.Feed})

		return
	}

	// add to the episodes metadata index
	count, err := episodesAddAll(podcast)
	if err != nil {
		logger.Error("crawl_podcast_feed.error.4", err, uid, idx.Feed)
		metrics.Error("crawl_podcast_feed.error", err.Error(), []string{uid, idx.Feed})

		return
	} else {
		// update main metadata index
		backend.IndexUpdate(uid)

		if count > 0 {
			// update stats and metrics
			if is_new {
				metrics.Count("crawler.podcast.new", 1)
				metrics.Count("crawler.episodes.new", count)
			} else {
				// new episodes added -> update the podcast.published timestamp
				podcastUpdateTimestamp(podcast)
				metrics.Count("crawler.episodes.update", count)
			}
		}

		logger.Log("crawl_podcast_feed.done", uid, idx.Feed, strconv.FormatInt((int64)(count), 10))
		metrics.Histogram("crawler.podcast_feed.duration", (float64)(util.ElapsedTimeSince(start_1)))
	}
}

func podcastAdd(podcast *Podcast) (bool, error) {
	p := backend.PodcastLookup(podcast.Uid)
	if p != nil {
		return false, nil
	}

	ds := datastore.GetDataStore()
	defer ds.Close()

	podcast_metadata := ds.Collection(datastore.PODCASTS_COL)

	meta := podcastDetailsToMetadata(podcast)

	// fix the published timestamp
	now := util.Timestamp()
	if podcast.Published > now {
		meta.Published = now // prevents dates in the future
	}

	err := podcast_metadata.Insert(&meta)

	if err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func podcastUpdateTimestamp(podcast *Podcast) (bool, error) {

	ds := datastore.GetDataStore()
	defer ds.Close()

	podcast_metadata := ds.Collection(datastore.PODCASTS_COL)

	p := backend.PodcastMetadata{}
	podcast_metadata.Find(bson.M{"uid": podcast.Uid}).One(&p)

	if p.Uid == "" {
		return false, nil
	} else {
		now := util.Timestamp()
		p.Updated = now
		if podcast.Published > now {
			p.Published = now // prevents dates in the future
		} else {
			p.Published = podcast.Published
		}

		// update the DB
		err := podcast_metadata.Update(bson.M{"uid": podcast.Uid}, &p)
		if err != nil {
			return false, err
		}
	}

	return true, nil
}

func episodeAdd(episode *Episode, puid string) (bool, error) {
	e := backend.EpisodeLookup(episode.Uid)
	if e != nil {
		return false, nil
	}

	ds := datastore.GetDataStore()
	defer ds.Close()

	episodes_metadata := ds.Collection(datastore.EPISODES_COL)

	meta := episodeDetailsToMetadata(episode, puid)
	err := episodes_metadata.Insert(&meta)

	if err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func episodesAddAll(podcast *Podcast) (int, error) {
	count := 0

	for i := 0; i < len(podcast.Episodes); i++ {
		added, err := episodeAdd(&podcast.Episodes[i], podcast.Uid)
		if err != nil {
			return 0, err
		}
		if added {
			count++
		}
	}
	return count, nil
}

func searchExpiredPodcasts(limit int) []backend.PodcastIndex {

	ds := datastore.GetDataStore()
	defer ds.Close()

	main_index := ds.Collection(datastore.META_COL)

	results := []backend.PodcastIndex{}
	q := bson.M{"next": bson.M{"$lte": util.Timestamp()}, "errors": bson.M{"$lte": backend.MAX_ERRORS}}

	if limit <= 0 {
		// return all
		main_index.Find(q).All(&results)
	} else {
		// with a limit
		main_index.Find(q).Limit(limit).All(&results)
	}

	return results
}

func podcastDetailsToMetadata(podcast *Podcast) *backend.PodcastMetadata {
	meta := backend.PodcastMetadata{
		podcast.Uid,
		podcast.Title,
		podcast.Subtitle,
		podcast.Url,
		podcast.Feed,
		podcast.Description,
		podcast.Published,
		podcast.Language,
		podcast.Image,
		podcast.Owner.Name,
		podcast.Owner.Email,
		"",
		0,
		0,
		0,
		0,
		util.Timestamp(),
		0,
	}
	return &meta
}

func episodeDetailsToMetadata(episode *Episode, puid string) *backend.EpisodeMetadata {
	meta := backend.EpisodeMetadata{
		episode.Uid,
		episode.Title,
		episode.Url,
		episode.Description,
		episode.Published,
		episode.Duration,
		episode.Author,
		episode.Content.Url,
		episode.Content.Type,
		episode.Content.Size,
		puid,
		0,
		util.Timestamp(),
		0,
	}
	return &meta
}
<commit_msg>fixed logger prefix<commit_after>package crawler

import (
	"gopkg.in/mgo.v2/bson"
	"strconv"
	"time"

	"github.com/mindcastio/mindcastio/backend"

	"github.com/mindcastio/mindcastio/backend/datastore"
	"github.com/mindcastio/mindcastio/backend/logger"
	"github.com/mindcastio/mindcastio/backend/metrics"
	"github.com/mindcastio/mindcastio/backend/util"
)

func SchedulePodcastCrawling() {
	logger.Log("mindcast.crawler.schedule_podcast_crawling")

	// search for podcasts that are candidates for crawling
	expired := searchExpiredPodcasts(backend.DEFAULT_UPDATE_BATCH)
	count := len(expired)

	logger.Log("crawler.schedule_podcast_crawling.scheduling", strconv.FormatInt((int64)(count), 10))

	if count > 0 {
		for i := 0; i < count; i++ {
			//go CrawlPodcastFeed(expired[i].Uid)
			CrawlPodcastFeed(expired[i].Uid)
			// FIXME use go routine or not ?
		}
		metrics.Count("crawler.scheduled", count)
	}

	logger.Log("crawler.schedule_podcast_crawling.done")
}

func CrawlPodcastFeed(uid string) {

	start_1 := time.Now()
	logger.Log("crawl_podcast_feed", uid)

	idx := backend.IndexLookup(uid)
	if idx == nil {
		logger.Error("crawl_podcast_feed.error.1", nil, uid)
		metrics.Error("crawl_podcast_feed.error", "", []string{uid})
		return
	}

	// HINT: ignore the fact that the item might be disables idx.erros > ...

	// fetch the podcast feed
	start_2 := time.Now()
	podcast, err := ParsePodcastFeed(idx.Feed)
	metrics.Histogram("crawler.parse_feed", (float64)(util.ElapsedTimeSince(start_2)))

	if err != nil {
		suspended, _ := backend.IndexBackoff(uid)

		if suspended {
			logger.Error("crawl_podcast_feed.suspended", err, uid, idx.Feed)
			metrics.Error("crawl_podcast_feed.suspended", err.Error(), []string{uid, idx.Feed})
		}

		return
	}

	// add to podcast metadata index
	is_new, err := podcastAdd(podcast)
	if err != nil {
		logger.Error("crawl_podcast_feed.error.3", err, uid, idx.Feed)
		metrics.Error("crawl_podcast_feed.error", err.Error(), []string{uid, idx.Feed})

		return
	}

	// add to the episodes metadata index
	count, err := episodesAddAll(podcast)
	if err != nil {
		logger.Error("crawl_podcast_feed.error.4", err, uid, idx.Feed)
		metrics.Error("crawl_podcast_feed.error", err.Error(), []string{uid, idx.Feed})

		return
	} else {
		// update main metadata index
		backend.IndexUpdate(uid)

		if count > 0 {
			// update stats and metrics
			if is_new {
				metrics.Count("crawler.podcast.new", 1)
				metrics.Count("crawler.episodes.new", count)
			} else {
				// new episodes added -> update the podcast.published timestamp
				podcastUpdateTimestamp(podcast)
				metrics.Count("crawler.episodes.update", count)
			}
		}

		logger.Log("crawl_podcast_feed.done", uid, idx.Feed, strconv.FormatInt((int64)(count), 10))
		metrics.Histogram("crawler.podcast_feed.duration", (float64)(util.ElapsedTimeSince(start_1)))
	}
}

func podcastAdd(podcast *Podcast) (bool, error) {
	p := backend.PodcastLookup(podcast.Uid)
	if p != nil {
		return false, nil
	}

	ds := datastore.GetDataStore()
	defer ds.Close()

	podcast_metadata := ds.Collection(datastore.PODCASTS_COL)

	meta := podcastDetailsToMetadata(podcast)

	// fix the published timestamp
	now := util.Timestamp()
	if podcast.Published > now {
		meta.Published = now // prevents dates in the future
	}

	err := podcast_metadata.Insert(&meta)

	if err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func podcastUpdateTimestamp(podcast *Podcast) (bool, error) {

	ds := datastore.GetDataStore()
	defer ds.Close()

	podcast_metadata := ds.Collection(datastore.PODCASTS_COL)

	p := backend.PodcastMetadata{}
	podcast_metadata.Find(bson.M{"uid": podcast.Uid}).One(&p)

	if p.Uid == "" {
		return false, nil
	} else {
		now := util.Timestamp()
		p.Updated = now
		if podcast.Published > now {
			p.Published = now // prevents dates in the future
		} else {
			p.Published = podcast.Published
		}

		// update the DB
		err := podcast_metadata.Update(bson.M{"uid": podcast.Uid}, &p)
		if err != nil {
			return false, err
		}
	}

	return true, nil
}

func episodeAdd(episode *Episode, puid string) (bool, error) {
	e := backend.EpisodeLookup(episode.Uid)
	if e != nil {
		return false, nil
	}

	ds := datastore.GetDataStore()
	defer ds.Close()

	episodes_metadata := ds.Collection(datastore.EPISODES_COL)

	meta := episodeDetailsToMetadata(episode, puid)
	err := episodes_metadata.Insert(&meta)

	if err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func episodesAddAll(podcast *Podcast) (int, error) {
	count := 0

	for i := 0; i < len(podcast.Episodes); i++ {
		added, err := episodeAdd(&podcast.Episodes[i], podcast.Uid)
		if err != nil {
			return 0, err
		}
		if added {
			count++
		}
	}
	return count, nil
}

func searchExpiredPodcasts(limit int) []backend.PodcastIndex {

	ds := datastore.GetDataStore()
	defer ds.Close()

	main_index := ds.Collection(datastore.META_COL)

	results := []backend.PodcastIndex{}
	q := bson.M{"next": bson.M{"$lte": util.Timestamp()}, "errors": bson.M{"$lte": backend.MAX_ERRORS}}

	if limit <= 0 {
		// return all
		main_index.Find(q).All(&results)
	} else {
		// with a limit
		main_index.Find(q).Limit(limit).All(&results)
	}

	return results
}

func podcastDetailsToMetadata(podcast *Podcast) *backend.PodcastMetadata {
	meta := backend.PodcastMetadata{
		podcast.Uid,
		podcast.Title,
		podcast.Subtitle,
		podcast.Url,
		podcast.Feed,
		podcast.Description,
		podcast.Published,
		podcast.Language,
		podcast.Image,
		podcast.Owner.Name,
		podcast.Owner.Email,
		"",
		0,
		0,
		0,
		0,
		util.Timestamp(),
		0,
	}
	return &meta
}

func episodeDetailsToMetadata(episode *Episode, puid string) *backend.EpisodeMetadata {
	meta := backend.EpisodeMetadata{
		episode.Uid,
		episode.Title,
		episode.Url,
		episode.Description,
		episode.Published,
		episode.Duration,
		episode.Author,
		episode.Content.Url,
		episode.Content.Type,
		episode.Content.Size,
		puid,
		0,
		util.Timestamp(),
		0,
	}
	return &meta
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/base32"
	"encoding/json"
	"errors"
	"github.com/devel/dnsmapper/storeapi"
	"io"
	"log"
	"net"
	"net/http"
	"strings"
)

type ipResponse struct {
	DNS  string
	EDNS string
	HTTP string
}

var uuidCh chan string

func uuidFactory() {
	uuidCh = make(chan string, 100)

	enc := base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567")

	length := 20

	buf := make([]byte, length)
	uuid := make([]byte, enc.EncodedLen(length))

	for {
		rand.Read(buf)
		enc.Encode(uuid, buf)
		uuidCh <- string(uuid)
	}
}

func uuid() string {
	return <-uuidCh
}

func jsonData(req *http.Request) (string, error) {
	ip, _, _ := net.SplitHostPort(req.RemoteAddr)

	resp := &ipResponse{HTTP: ip, DNS: ""}

	uuid := getUuidFromDomain(req.Host)
	get := Redis.Get("dns-" + uuid)
	if err := get.Err(); err != nil {
		return "", errors.New("UUID not found")
	}

	v := strings.Split(get.Val(), " ")

	resp.DNS = v[0]
	if len(v) > 1 {
		resp.EDNS = v[1]
	}

	js, err := json.Marshal(resp)
	if err != nil {
		log.Print("JSON ERROR:", err)
		return "", err
	}

	data := storeapi.RequestData{
		TestIP:   *flagip,
		ServerIP: resp.DNS,
		ClientIP: resp.HTTP,
		EnumNet:  resp.EDNS,
	}
	select {
	case ch <- &data:
	default:
		log.Println("dropped log data, queue full")
	}

	return string(js), nil
}

func redirectUuid(w http.ResponseWriter, req *http.Request) {
	uuid := uuid()
	host := uuid + "." + *flagdomain

	proto := "http"

	if req.TLS != nil {
		proto = "https"
	}

	http.Redirect(w, req, proto+"://"+host+req.RequestURI, 302)
	return
}

func mainServer(w http.ResponseWriter, req *http.Request) {

	log.Println("HTTP request from", req.RemoteAddr, req.Host)

	if req.URL.Path == "/jsonp" || req.URL.Path == "/json" || req.URL.Path == "/none" {

		w.Header().Set("Cache-Control", "private, no-cache, no-store, must-revalidate")

		uuid := getUuidFromDomain(req.Host)
		if uuid == "www" {
			redirectUuid(w, req)
			return
		}

		js, err := jsonData(req)
		if err != nil {
			redirectUuid(w, req)
			return
		}

		if req.URL.Path == "/none" {
			w.WriteHeader(204)
			return
		}

		if jsonp := req.FormValue("jsonp"); len(jsonp) > 0 {
			w.Header().Set("Content-Type", "text/javascript")
			io.WriteString(w, jsonp+"("+js+");\n")
			return
		}

		// not jsonp
		w.Header().Set("Content-Type", "application/json")
		io.WriteString(w, js+"\n")
		return

	}

	if req.URL.Path == "/version" {
		io.WriteString(w, `<html><head><title>DNS Mapper `+
			VERSION+`</title><body>`+
			`Hello`+
			`</body></html>`)
		return
	}

	http.NotFound(w, req)
	return
}

func httpHandler() {

	go uuidFactory()

	http.HandleFunc("/", mainServer)

	if len(*flagtlskeyfile) > 0 {

		log.Printf("Starting TLS with key='%s' and cert='%s'",
			*flagtlskeyfile,
			*flagtlscrtfile,
		)

		go func() {
			tlslisten := *flagip + ":" + *flaghttpsport
			log.Println("Going to listen for TLS requests on port", tlslisten)
			log.Fatal(http.ListenAndServeTLS(
				tlslisten,
				*flagtlscrtfile,
				*flagtlskeyfile,
				nil,
			))
		}()
	}

	listen := *flagip + ":" + *flaghttpport
	log.Println("HTTP listen on", listen)
	log.Fatal(http.ListenAndServe(listen, nil))

}
<commit_msg>Enum/Edns in dnsmapper server<commit_after>package main

import (
	"crypto/rand"
	"encoding/base32"
	"encoding/json"
	"errors"
	"github.com/devel/dnsmapper/storeapi"
	"io"
	"log"
	"net"
	"net/http"
	"strings"
)

type ipResponse struct {
	DNS  string
	EDNS string
	HTTP string
}

var uuidCh chan string

func uuidFactory() {
	uuidCh = make(chan string, 100)

	enc := base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567")

	length := 20

	buf := make([]byte, length)
	uuid := make([]byte, enc.EncodedLen(length))

	for {
		rand.Read(buf)
		enc.Encode(uuid, buf)
		uuidCh <- string(uuid)
	}
}

func uuid() string {
	return <-uuidCh
}

func jsonData(req *http.Request) (string, error) {
	ip, _, _ := net.SplitHostPort(req.RemoteAddr)

	resp := &ipResponse{HTTP: ip, DNS: ""}

	uuid := getUuidFromDomain(req.Host)
	get := Redis.Get("dns-" + uuid)
	if err := get.Err(); err != nil {
		return "", errors.New("UUID not found")
	}

	v := strings.Split(get.Val(), " ")

	resp.DNS = v[0]
	if len(v) > 1 {
		resp.EDNS = v[1]
	}

	js, err := json.Marshal(resp)
	if err != nil {
		log.Print("JSON ERROR:", err)
		return "", err
	}

	data := storeapi.RequestData{
		TestIP:   *flagip,
		ServerIP: resp.DNS,
		ClientIP: resp.HTTP,
		EdnsNet:  resp.EDNS,
	}
	select {
	case ch <- &data:
	default:
		log.Println("dropped log data, queue full")
	}

	return string(js), nil
}

func redirectUuid(w http.ResponseWriter, req *http.Request) {
	uuid := uuid()
	host := uuid + "." + *flagdomain

	proto := "http"

	if req.TLS != nil {
		proto = "https"
	}

	http.Redirect(w, req, proto+"://"+host+req.RequestURI, 302)
	return
}

func mainServer(w http.ResponseWriter, req *http.Request) {

	log.Println("HTTP request from", req.RemoteAddr, req.Host)

	if req.URL.Path == "/jsonp" || req.URL.Path == "/json" || req.URL.Path == "/none" {

		w.Header().Set("Cache-Control", "private, no-cache, no-store, must-revalidate")

		uuid := getUuidFromDomain(req.Host)
		if uuid == "www" {
			redirectUuid(w, req)
			return
		}

		js, err := jsonData(req)
		if err != nil {
			redirectUuid(w, req)
			return
		}

		if req.URL.Path == "/none" {
			w.WriteHeader(204)
			return
		}

		if jsonp := req.FormValue("jsonp"); len(jsonp) > 0 {
			w.Header().Set("Content-Type", "text/javascript")
			io.WriteString(w, jsonp+"("+js+");\n")
			return
		}

		// not jsonp
		w.Header().Set("Content-Type", "application/json")
		io.WriteString(w, js+"\n")
		return

	}

	if req.URL.Path == "/version" {
		io.WriteString(w, `<html><head><title>DNS Mapper `+
			VERSION+`</title><body>`+
			`Hello`+
			`</body></html>`)
		return
	}

	http.NotFound(w, req)
	return
}

func httpHandler() {

	go uuidFactory()

	http.HandleFunc("/", mainServer)

	if len(*flagtlskeyfile) > 0 {

		log.Printf("Starting TLS with key='%s' and cert='%s'",
			*flagtlskeyfile,
			*flagtlscrtfile,
		)

		go func() {
			tlslisten := *flagip + ":" + *flaghttpsport
			log.Println("Going to listen for TLS requests on port", tlslisten)
			log.Fatal(http.ListenAndServeTLS(
				tlslisten,
				*flagtlscrtfile,
				*flagtlskeyfile,
				nil,
			))
		}()
	}

	listen := *flagip + ":" + *flaghttpport
	log.Println("HTTP listen on", listen)
	log.Fatal(http.ListenAndServe(listen, nil))

}
<|endoftext|>


<commit_before>package crawler

import (
	"errors"
	"net/http"
	"time"

	log "github.com/meifamily/logrus"

	"github.com/meifamily/ptt-alertor/models/ptt/article"

	"regexp"

	"strings"

	"golang.org/x/net/html"
)

const pttHostURL = "https://www.ptt.cc"

// BuildArticles makes board's index articles to a article slice
func BuildArticles(board string) (article.Articles, error) {

	reqURL := makeBoardURL(board)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Articles{}, err
	}
	htmlNodes := parseHTML(rsp)

	articleBlocks := traverseHTMLNode(htmlNodes, findArticleBlocks)
	initialTargetNodes()
	articles := make(article.Articles, len(articleBlocks))
	for index, articleBlock := range articleBlocks {
		for _, titleDiv := range traverseHTMLNode(articleBlock, findTitleDiv) {
			initialTargetNodes()

			anchors := traverseHTMLNode(titleDiv, findAnchor)

			if len(anchors) == 0 {
				articles[index].Title = titleDiv.FirstChild.Data
				articles[index].Link = ""
				continue
			}

			for _, anchor := range traverseHTMLNode(titleDiv, findAnchor) {
				articles[index].Title = anchor.FirstChild.Data
				link := pttHostURL + getAnchorLink(anchor)
				articles[index].Link = link
				articles[index].ID = articles[index].ParseID(link)
			}
		}
		for _, metaDiv := range traverseHTMLNode(articleBlock, findMetaDiv) {
			initialTargetNodes()

			for _, date := range traverseHTMLNode(metaDiv, findDateDiv) {
				articles[index].Date = date.FirstChild.Data
			}
			for _, author := range traverseHTMLNode(metaDiv, findAuthorDiv) {
				articles[index].Author = author.FirstChild.Data
			}
		}
	}
	return articles, nil
}

// BuildArticle build article object from html
func BuildArticle(board, articleCode string) (article.Article, error) {

	reqURL := makeArticleURL(board, articleCode)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Article{}, err
	}
	htmlNodes := parseHTML(rsp)
	atcl := article.Article{
		Link:  reqURL,
		Code:  articleCode,
		Board: board,
	}
	nodes := traverseHTMLNode(htmlNodes, findOgTitleMeta)
	if len(nodes) > 0 {
		atcl.Title = getMetaContent(nodes[0])
	} else {
		atcl.Title = "[內文標題已被刪除]"
	}
	atcl.ID = atcl.ParseID(reqURL)
	pushBlocks := traverseHTMLNode(htmlNodes, findPushBlocks)
	initialTargetNodes()
	pushes := make([]article.Push, len(pushBlocks))
	for index, pushBlock := range pushBlocks {
		for _, pushTag := range traverseHTMLNode(pushBlock, findPushTag) {
			initialTargetNodes()
			pushes[index].Tag = pushTag.FirstChild.Data
		}
		for _, pushUserID := range traverseHTMLNode(pushBlock, findPushUserID) {
			initialTargetNodes()
			pushes[index].UserID = pushUserID.FirstChild.Data
		}
		for _, pushContent := range traverseHTMLNode(pushBlock, findPushContent) {
			initialTargetNodes()
			content := pushContent.FirstChild.Data
			for n := pushContent.FirstChild.NextSibling; n != nil; n = n.NextSibling {
				if n.FirstChild != nil {
					content += n.FirstChild.Data
				}
				if n.NextSibling != nil {
					content += n.NextSibling.Data
				}
			}
			pushes[index].Content = content
		}
		for _, pushIPDateTime := range traverseHTMLNode(pushBlock, findPushIPDateTime) {
			initialTargetNodes()
			ipdatetime := pushIPDateTime.FirstChild.Data
			if ipdatetime == "" {
				break
			}
			dateTime, err := parseDateTime(ipdatetime)
			if err != nil {
				log.WithFields(log.Fields{
					"ipdatetime": ipdatetime,
					"board":      board,
					"code":       articleCode,
				}).WithError(err).Error("Parse DateTime Error")
			}
			pushes[index].DateTime = dateTime
			if index == len(pushBlocks)-1 {
				atcl.LastPushDateTime = pushes[index].DateTime
			}
		}
	}
	atcl.PushList = pushes
	return atcl, nil
}

func parseDateTime(ipdatetime string) (time.Time, error) {
	re, _ := regexp.Compile("(\\d+\\.\\d+\\.\\d+\\.\\d+)?\\s*(.*)")
	ipdatetime = strings.TrimSpace(ipdatetime)
	subMatches := re.FindStringSubmatch(ipdatetime)
	dateTime := strings.TrimSpace(subMatches[len(subMatches)-1])
	loc, err := time.LoadLocation("UTC")
	if err != nil {
		return time.Time{}, err
	}
	t, err := time.ParseInLocation("01/02 15:04", dateTime, loc)
	if err != nil {
		return time.Time{}, err
	}
	t = t.AddDate(getYear(t), 0, 0)
	return t, nil
}

func getYear(pushTime time.Time) int {
	t := time.Now()
	if t.Month() == 1 && pushTime.Month() == 12 {
		return t.Year() - 1
	}
	return t.Year()
}

// CheckBoardExist use for checking board exist or not
func CheckBoardExist(board string) bool {
	reqURL := makeBoardURL(board)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

// CheckArticleExist user for checking article exist or not
func CheckArticleExist(board, articleCode string) bool {
	reqURL := makeArticleURL(board, articleCode)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

func makeBoardURL(board string) string {
	return pttHostURL + "/bbs/" + board + "/index.html"
}

func makeArticleURL(board, articleCode string) string {
	return pttHostURL + "/bbs/" + board + "/" + articleCode + ".html"
}

type URLNotFoundError struct {
	URL string
}

func (u URLNotFoundError) Error() string {
	return "Fetched URL Not Found"
}

func fetchHTML(reqURL string) (response *http.Response, err error) {

	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return errors.New("Redirect")
		},
	}

	response, err = client.Get(reqURL)

	if response.StatusCode == http.StatusNotFound {
		err = URLNotFoundError{reqURL}
	}

	if err != nil && response.StatusCode == http.StatusFound {
		req := passR18(reqURL)
		response, err = client.Do(req)
	}

	if err != nil {
		log.WithField("url", reqURL).WithError(err).Error("Fetch URL Failed")
	}

	return response, err
}

func passR18(reqURL string) (req *http.Request) {

	req, _ = http.NewRequest("GET", reqURL, nil)

	over18Cookie := http.Cookie{
		Name:       "over18",
		Value:      "1",
		Domain:     "www.ptt.cc",
		Path:       "/",
		RawExpires: "Session",
		MaxAge:     0,
		HttpOnly:   false,
	}

	req.AddCookie(&over18Cookie)

	return req
}

func parseHTML(response *http.Response) *html.Node {
	doc, err := html.Parse(response.Body)
	if err != nil {
		log.Error(err)
	}
	return doc
}
<commit_msg>trimspace before parse datetime<commit_after>package crawler

import (
	"errors"
	"net/http"
	"time"

	log "github.com/meifamily/logrus"

	"github.com/meifamily/ptt-alertor/models/ptt/article"

	"regexp"

	"strings"

	"golang.org/x/net/html"
)

const pttHostURL = "https://www.ptt.cc"

// BuildArticles makes board's index articles to a article slice
func BuildArticles(board string) (article.Articles, error) {

	reqURL := makeBoardURL(board)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Articles{}, err
	}
	htmlNodes := parseHTML(rsp)

	articleBlocks := traverseHTMLNode(htmlNodes, findArticleBlocks)
	initialTargetNodes()
	articles := make(article.Articles, len(articleBlocks))
	for index, articleBlock := range articleBlocks {
		for _, titleDiv := range traverseHTMLNode(articleBlock, findTitleDiv) {
			initialTargetNodes()

			anchors := traverseHTMLNode(titleDiv, findAnchor)

			if len(anchors) == 0 {
				articles[index].Title = titleDiv.FirstChild.Data
				articles[index].Link = ""
				continue
			}

			for _, anchor := range traverseHTMLNode(titleDiv, findAnchor) {
				articles[index].Title = anchor.FirstChild.Data
				link := pttHostURL + getAnchorLink(anchor)
				articles[index].Link = link
				articles[index].ID = articles[index].ParseID(link)
			}
		}
		for _, metaDiv := range traverseHTMLNode(articleBlock, findMetaDiv) {
			initialTargetNodes()

			for _, date := range traverseHTMLNode(metaDiv, findDateDiv) {
				articles[index].Date = date.FirstChild.Data
			}
			for _, author := range traverseHTMLNode(metaDiv, findAuthorDiv) {
				articles[index].Author = author.FirstChild.Data
			}
		}
	}
	return articles, nil
}

// BuildArticle build article object from html
func BuildArticle(board, articleCode string) (article.Article, error) {

	reqURL := makeArticleURL(board, articleCode)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Article{}, err
	}
	htmlNodes := parseHTML(rsp)
	atcl := article.Article{
		Link:  reqURL,
		Code:  articleCode,
		Board: board,
	}
	nodes := traverseHTMLNode(htmlNodes, findOgTitleMeta)
	if len(nodes) > 0 {
		atcl.Title = getMetaContent(nodes[0])
	} else {
		atcl.Title = "[內文標題已被刪除]"
	}
	atcl.ID = atcl.ParseID(reqURL)
	pushBlocks := traverseHTMLNode(htmlNodes, findPushBlocks)
	initialTargetNodes()
	pushes := make([]article.Push, len(pushBlocks))
	for index, pushBlock := range pushBlocks {
		for _, pushTag := range traverseHTMLNode(pushBlock, findPushTag) {
			initialTargetNodes()
			pushes[index].Tag = pushTag.FirstChild.Data
		}
		for _, pushUserID := range traverseHTMLNode(pushBlock, findPushUserID) {
			initialTargetNodes()
			pushes[index].UserID = pushUserID.FirstChild.Data
		}
		for _, pushContent := range traverseHTMLNode(pushBlock, findPushContent) {
			initialTargetNodes()
			content := pushContent.FirstChild.Data
			for n := pushContent.FirstChild.NextSibling; n != nil; n = n.NextSibling {
				if n.FirstChild != nil {
					content += n.FirstChild.Data
				}
				if n.NextSibling != nil {
					content += n.NextSibling.Data
				}
			}
			pushes[index].Content = content
		}
		for _, pushIPDateTime := range traverseHTMLNode(pushBlock, findPushIPDateTime) {
			initialTargetNodes()
			ipdatetime := strings.TrimSpace(pushIPDateTime.FirstChild.Data)
			if ipdatetime == "" {
				break
			}
			dateTime, err := parseDateTime(ipdatetime)
			if err != nil {
				log.WithFields(log.Fields{
					"ipdatetime": ipdatetime,
					"board":      board,
					"code":       articleCode,
				}).WithError(err).Error("Parse DateTime Error")
			}
			pushes[index].DateTime = dateTime
			if index == len(pushBlocks)-1 {
				atcl.LastPushDateTime = pushes[index].DateTime
			}
		}
	}
	atcl.PushList = pushes
	return atcl, nil
}

func parseDateTime(ipdatetime string) (time.Time, error) {
	re, _ := regexp.Compile("(\\d+\\.\\d+\\.\\d+\\.\\d+)?\\s*(.*)")
	subMatches := re.FindStringSubmatch(ipdatetime)
	dateTime := strings.TrimSpace(subMatches[len(subMatches)-1])
	loc, err := time.LoadLocation("UTC")
	if err != nil {
		return time.Time{}, err
	}
	t, err := time.ParseInLocation("01/02 15:04", dateTime, loc)
	if err != nil {
		return time.Time{}, err
	}
	t = t.AddDate(getYear(t), 0, 0)
	return t, nil
}

func getYear(pushTime time.Time) int {
	t := time.Now()
	if t.Month() == 1 && pushTime.Month() == 12 {
		return t.Year() - 1
	}
	return t.Year()
}

// CheckBoardExist use for checking board exist or not
func CheckBoardExist(board string) bool {
	reqURL := makeBoardURL(board)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

// CheckArticleExist user for checking article exist or not
func CheckArticleExist(board, articleCode string) bool {
	reqURL := makeArticleURL(board, articleCode)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

func makeBoardURL(board string) string {
	return pttHostURL + "/bbs/" + board + "/index.html"
}

func makeArticleURL(board, articleCode string) string {
	return pttHostURL + "/bbs/" + board + "/" + articleCode + ".html"
}

type URLNotFoundError struct {
	URL string
}

func (u URLNotFoundError) Error() string {
	return "Fetched URL Not Found"
}

func fetchHTML(reqURL string) (response *http.Response, err error) {

	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return errors.New("Redirect")
		},
	}

	response, err = client.Get(reqURL)

	if response.StatusCode == http.StatusNotFound {
		err = URLNotFoundError{reqURL}
	}

	if err != nil && response.StatusCode == http.StatusFound {
		req := passR18(reqURL)
		response, err = client.Do(req)
	}

	if err != nil {
		log.WithField("url", reqURL).WithError(err).Error("Fetch URL Failed")
	}

	return response, err
}

func passR18(reqURL string) (req *http.Request) {

	req, _ = http.NewRequest("GET", reqURL, nil)

	over18Cookie := http.Cookie{
		Name:       "over18",
		Value:      "1",
		Domain:     "www.ptt.cc",
		Path:       "/",
		RawExpires: "Session",
		MaxAge:     0,
		HttpOnly:   false,
	}

	req.AddCookie(&over18Cookie)

	return req
}

func parseHTML(response *http.Response) *html.Node {
	doc, err := html.Parse(response.Body)
	if err != nil {
		log.Error(err)
	}
	return doc
}
<|endoftext|>