<commit_before>package controllers

import (
	"net/url"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/astaxie/beego"

	"github.com/shwinpiocess/cc/models"
	"github.com/shwinpiocess/cc/utils"
)

type BaseController struct {
	beego.Controller

	userId         int
	userName       string

	requestPath string
	today string

	firstApp bool
	firstSet bool
	firtModule bool

	defaultApp *models.App
}

func (this *BaseController) Prepare() {
	this.requestPath = this.Ctx.Input.URL()
	this.today = time.Now().Format("20060102")

	this.auth()

	var fields []string
	var sortby []string
	var order []string
	var query map[string]string = make(map[string]string)
	var limit int64 = 0
	var offset int64 = 0

	query["owner_id"] = strconv.Itoa(this.userId)

	apps, _ := models.GetAllApp(query, fields, sortby, order, offset, limit)

	if len(apps) > 0 {
		defaultAppId := this.Ctx.GetCookie("defaultAppId")

		if defaultAppId, err := strconv.Atoi(defaultAppId); err == nil {
			if app, err := models.GetAppById(defaultAppId); err != nil {
				defaultApp := apps[0].(models.App)
				this.defaultApp = &defaultApp
				this.Ctx.SetCookie("defaultAppId", strconv.Itoa(this.defaultApp.Id))
				this.Ctx.SetCookie("defaultAppName", url.QueryEscape(this.defaultApp.ApplicationName))
			} else {
				this.defaultApp = app
			}
		}
	} else {
		this.firstApp = true
		this.firstSet = true
		this.firtModule = true
	}
	fmt.Println("this.defaultApp=", this.defaultApp)
	this.Data["requestPath"] = this.requestPath
	this.Data["today"] = this.today

	this.Data["defaultApp"] = this.defaultApp
	this.Data["apps"] = apps
	this.Data["firstApp"] = this.firstApp
	this.Data["firstSet"] = this.firstSet
	this.Data["firstModule"] = this.firtModule
}

func (this *BaseController) getClientIP() string {
	return strings.Split(this.Ctx.Request.RemoteAddr, ":")[0]
}

func (this *BaseController) redirect(url string) {
	this.Redirect(url, 302)
	this.StopRun()
}

func (this *BaseController) isPost() bool {
	return this.Ctx.Request.Method == "POST"
}

func (this *BaseController) jsonResult(out interface{}) {
	this.Data["json"] = out
	this.ServeJSON()
	this.StopRun()
}

func (this *BaseController) auth() {
	arrs := strings.Split(this.Ctx.GetCookie("auth"), "|")
	if len(arrs) == 2 {
		idstr, password := arrs[0], arrs[1]
		userId, _ := strconv.Atoi(idstr)
		if userId > 0 {
			user, err := models.GetUserById(userId)
			if err == nil && password == utils.Md5([]byte(this.getClientIP()+"|"+user.Password+user.Salt)) {
				this.userId = user.Id
				this.userName = user.UserName
			}
		}
	}

	if this.userId == 0 && this.requestPath != beego.URLFor("MainController.Login") && this.requestPath != beego.URLFor("MainController.Logout") {
		this.redirect(beego.URLFor("MainController.Login"))
	}
}
<commit_msg>修复cookies不包含默认app信息的bug<commit_after>package controllers

import (
	"net/url"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/astaxie/beego"

	"github.com/shwinpiocess/cc/models"
	"github.com/shwinpiocess/cc/utils"
)

type BaseController struct {
	beego.Controller

	userId         int
	userName       string

	requestPath string
	today string

	firstApp bool
	firstSet bool
	firtModule bool

	defaultApp *models.App
}

func (this *BaseController) Prepare() {
	this.requestPath = this.Ctx.Input.URL()
	this.today = time.Now().Format("20060102")

	this.auth()

	var fields []string
	var sortby []string
	var order []string
	var query map[string]string = make(map[string]string)
	var limit int64 = 0
	var offset int64 = 0

	query["owner_id"] = strconv.Itoa(this.userId)

	apps, _ := models.GetAllApp(query, fields, sortby, order, offset, limit)

	if len(apps) > 0 {
		defaultAppId := this.Ctx.GetCookie("defaultAppId")

		if defaultAppId, err := strconv.Atoi(defaultAppId); err == nil {
			if app, err := models.GetAppById(defaultAppId); err != nil {
				defaultApp := apps[0].(models.App)
				this.defaultApp = &defaultApp
				this.Ctx.SetCookie("defaultAppId", strconv.Itoa(this.defaultApp.Id))
				this.Ctx.SetCookie("defaultAppName", url.QueryEscape(this.defaultApp.ApplicationName))
			} else {
				this.defaultApp = app
			}
		} else {
			defaultApp := apps[0].(models.App)
			this.defaultApp = &defaultApp
			this.Ctx.SetCookie("defaultAppId", strconv.Itoa(this.defaultApp.Id))
			this.Ctx.SetCookie("defaultAppName", url.QueryEscape(this.defaultApp.ApplicationName))
		}
	} else {
		this.firstApp = true
		this.firstSet = true
		this.firtModule = true
	}
	fmt.Println("this.defaultApp=", this.defaultApp)
	this.Data["requestPath"] = this.requestPath
	this.Data["today"] = this.today

	this.Data["defaultApp"] = this.defaultApp
	this.Data["apps"] = apps
	this.Data["firstApp"] = this.firstApp
	this.Data["firstSet"] = this.firstSet
	this.Data["firstModule"] = this.firtModule
}

func (this *BaseController) getClientIP() string {
	return strings.Split(this.Ctx.Request.RemoteAddr, ":")[0]
}

func (this *BaseController) redirect(url string) {
	this.Redirect(url, 302)
	this.StopRun()
}

func (this *BaseController) isPost() bool {
	return this.Ctx.Request.Method == "POST"
}

func (this *BaseController) jsonResult(out interface{}) {
	this.Data["json"] = out
	this.ServeJSON()
	this.StopRun()
}

func (this *BaseController) auth() {
	arrs := strings.Split(this.Ctx.GetCookie("auth"), "|")
	if len(arrs) == 2 {
		idstr, password := arrs[0], arrs[1]
		userId, _ := strconv.Atoi(idstr)
		if userId > 0 {
			user, err := models.GetUserById(userId)
			if err == nil && password == utils.Md5([]byte(this.getClientIP()+"|"+user.Password+user.Salt)) {
				this.userId = user.Id
				this.userName = user.UserName
			}
		}
	}

	if this.userId == 0 && this.requestPath != beego.URLFor("MainController.Login") && this.requestPath != beego.URLFor("MainController.Logout") {
		this.redirect(beego.URLFor("MainController.Login"))
	}
}
<|endoftext|>


<commit_before>package responses

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/perthgophers/puddle/messagerouter"
	"os/exec"
	"strings"
	"sync"
)

var lock = new(sync.Mutex)

// Checkout pulls & checks out a branch
func Checkout(cr *messagerouter.CommandRequest, w messagerouter.ResponseWriter) error {
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	words := strings.Split(cr.Text, " ")

	if len(words) < 2 {
		w.Write("I need a branch to continue with this operation")
		return nil
	}
	branch := words[1]

	cmd := exec.Command("git", "pull", "origin", branch)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	cmd = exec.Command("git", "checkout", branch)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	w.Write(stdout.String())
	if stderr.Len() > 0 {
		w.Write(stdout.String())
	}

	return nil
}

// Build will pull the latest git master and rebuild. It will then restart puddlebot
// Syntax: `!build <branch-name>`
// Example: `!build master`
func Build(cr *messagerouter.CommandRequest, w messagerouter.ResponseWriter) error {
	out, _ := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	currentBranch := string(out)
	w.Write("Current branch: " + currentBranch)
	w.Write("Rebuild requested. Locked.")
	lock.Lock()
	defer lock.Unlock()

	words := strings.Split(cr.Text, " ")
	branch := "master"
	if len(words) > 1 {
		branch = words[1]
	}
	w.Write(fmt.Sprintf("Selecting branch/%s...", branch))

	Checkout(cr, w)

	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command("go", "install")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	cmd.Run()

	w.Write(stdout.String())
	if stderr.Len() > 0 {
		w.WriteError("Error building:" + stderr.String())
	}

	Restart(cr, w)
	return nil
}

func handleErr(stderr *bytes.Buffer, w messagerouter.ResponseWriter) error {
	if stderr.Len() > 0 {
		errString := stderr.String()
		stderr.Reset()
		w.WriteError(errString)
		return errors.New(errString)
	}
	return nil
}

func init() {
	Handle("!checkout", Checkout)
	Handle("!build", Build)
}
<commit_msg>run glide up before go install<commit_after>package responses

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/perthgophers/puddle/messagerouter"
	"os/exec"
	"strings"
	"sync"
)

var lock = new(sync.Mutex)

// Checkout pulls & checks out a branch
func Checkout(cr *messagerouter.CommandRequest, w messagerouter.ResponseWriter) error {
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	words := strings.Split(cr.Text, " ")

	if len(words) < 2 {
		w.Write("I need a branch to continue with this operation")
		return nil
	}
	branch := words[1]

	cmd := exec.Command("git", "pull", "origin", branch)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	cmd = exec.Command("git", "checkout", branch)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	w.Write(stdout.String())
	if stderr.Len() > 0 {
		w.Write(stdout.String())
	}

	return nil
}

// Build will pull the latest git master and rebuild. It will then restart puddlebot
// Syntax: `!build <branch-name>`
// Example: `!build master`
func Build(cr *messagerouter.CommandRequest, w messagerouter.ResponseWriter) error {
	out, _ := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	currentBranch := string(out)
	w.Write("Current branch: " + currentBranch)
	w.Write("Rebuild requested. Locked.")
	lock.Lock()
	defer lock.Unlock()

	words := strings.Split(cr.Text, " ")
	branch := "master"
	if len(words) > 1 {
		branch = words[1]
	}
	w.Write(fmt.Sprintf("Selecting branch/%s...", branch))

	Checkout(cr, w)

	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command("glide", "up")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	cmd.Run()

	if stderr.Len() > 0 {
		w.WriteError("Error running glide up.\n" + stderr.String())
	}

	stdout.Reset()
	stderr.Reset()

	cmd = exec.Command("go", "install")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	cmd.Run()

	w.Write(stdout.String())
	if stderr.Len() > 0 {
		w.WriteError("Error building:" + stderr.String())
	} else {
		w.Write(":champagne: Go install completed successfully. :champagne: ")
	}

	Restart(cr, w)
	return nil
}

func handleErr(stderr *bytes.Buffer, w messagerouter.ResponseWriter) error {
	if stderr.Len() > 0 {
		errString := stderr.String()
		stderr.Reset()
		w.WriteError(errString)
		return errors.New(errString)
	}
	return nil
}

func init() {
	Handle("!checkout", Checkout)
	Handle("!build", Build)
}
<|endoftext|>


<commit_before>package leveldb

/*
#cgo LDFLAGS: -lleveldb
#include <leveldb/c.h>
*/
import "C"

import (
	"encoding/json"
	"os"
	"unsafe"
)

const defaultFilterBits int = 10

type Config struct {
	Path string `json:"path"`

	Compression     bool `json:"compression"`
	BlockSize       int  `json:"block_size"`
	WriteBufferSize int  `json:"write_buffer_size"`
	CacheSize       int  `json:"cache_size"`
	MaxOpenFiles    int  `json:"max_open_files"`
}

type DB struct {
	cfg *Config

	db *C.leveldb_t

	opts *Options

	//for default read and write options
	readOpts     *ReadOptions
	writeOpts    *WriteOptions
	iteratorOpts *ReadOptions

	syncWriteOpts *WriteOptions

	cache *Cache

	filter *FilterPolicy
}

func Open(configJson json.RawMessage) (*DB, error) {
	cfg := new(Config)
	err := json.Unmarshal(configJson, cfg)
	if err != nil {
		return nil, err
	}

	return OpenWithConfig(cfg)
}

func OpenWithConfig(cfg *Config) (*DB, error) {
	if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil {
		return nil, err
	}

	db := new(DB)
	db.cfg = cfg

	if err := db.open(); err != nil {
		return nil, err
	}

	return db, nil
}

func (db *DB) open() error {
	db.initOptions(db.cfg)

	var errStr *C.char
	ldbname := C.CString(db.cfg.Path)
	defer C.leveldb_free(unsafe.Pointer(ldbname))

	db.db = C.leveldb_open(db.opts.Opt, ldbname, &errStr)
	if errStr != nil {
		db.db = nil
		return saveError(errStr)
	}
	return nil
}

func Repair(cfg *Config) error {
	db := new(DB)
	db.cfg = cfg

	err := db.open()
	defer db.Close()

	//open ok, do not need repair
	if err == nil {
		return nil
	}

	var errStr *C.char
	ldbname := C.CString(db.cfg.Path)
	defer C.leveldb_free(unsafe.Pointer(ldbname))

	C.leveldb_repair_db(db.opts.Opt, ldbname, &errStr)
	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}

func (db *DB) initOptions(cfg *Config) {
	opts := NewOptions()

	opts.SetCreateIfMissing(true)

	if cfg.CacheSize <= 0 {
		cfg.CacheSize = 4 * 1024 * 1024
	}

	db.cache = NewLRUCache(cfg.CacheSize)
	opts.SetCache(db.cache)

	//we must use bloomfilter
	db.filter = NewBloomFilter(defaultFilterBits)
	opts.SetFilterPolicy(db.filter)

	if !cfg.Compression {
		opts.SetCompression(NoCompression)
	}

	if cfg.BlockSize <= 0 {
		cfg.BlockSize = 4 * 1024
	}

	opts.SetBlockSize(cfg.BlockSize)

	if cfg.WriteBufferSize <= 0 {
		cfg.WriteBufferSize = 4 * 1024 * 1024
	}

	opts.SetWriteBufferSize(cfg.WriteBufferSize)

	if cfg.MaxOpenFiles < 1024 {
		cfg.MaxOpenFiles = 1024
	}

	opts.SetMaxOpenFiles(cfg.MaxOpenFiles)

	db.opts = opts

	db.readOpts = NewReadOptions()
	db.writeOpts = NewWriteOptions()

	db.iteratorOpts = NewReadOptions()
	db.iteratorOpts.SetFillCache(false)

	db.syncWriteOpts = NewWriteOptions()
	db.syncWriteOpts.SetSync(true)
}

func (db *DB) Close() {
	if db.db != nil {
		C.leveldb_close(db.db)
		db.db = nil
	}

	db.opts.Close()

	if db.cache != nil {
		db.cache.Close()
	}

	if db.filter != nil {
		db.filter.Close()
	}

	db.readOpts.Close()
	db.writeOpts.Close()
	db.iteratorOpts.Close()
	db.syncWriteOpts.Close()
}

func (db *DB) Destroy() error {
	path := db.cfg.Path

	db.Close()

	opts := NewOptions()
	defer opts.Close()

	var errStr *C.char
	ldbname := C.CString(path)
	defer C.leveldb_free(unsafe.Pointer(ldbname))

	C.leveldb_destroy_db(opts.Opt, ldbname, &errStr)
	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}

func (db *DB) Clear() error {
	bc := db.NewWriteBatch()
	defer bc.Close()

	var err error
	it := db.NewIterator()
	it.SeekToFirst()

	num := 0
	for ; it.Valid(); it.Next() {
		bc.Delete(it.Key())
		num++
		if num == 1000 {
			num = 0
			if err = bc.Commit(); err != nil {
				return err
			}
		}
	}

	err = bc.Commit()

	return err
}

func (db *DB) Put(key, value []byte) error {
	return db.put(db.writeOpts, key, value)
}

func (db *DB) SyncPut(key, value []byte) error {
	return db.put(db.syncWriteOpts, key, value)
}

func (db *DB) Get(key []byte) ([]byte, error) {
	return db.get(db.readOpts, key)
}

func (db *DB) Delete(key []byte) error {
	return db.delete(db.writeOpts, key)
}

func (db *DB) SyncDelete(key []byte) error {
	return db.delete(db.syncWriteOpts, key)
}

func (db *DB) NewWriteBatch() *WriteBatch {
	wb := &WriteBatch{
		db:     db,
		wbatch: C.leveldb_writebatch_create(),
	}
	return wb
}

func (db *DB) NewSnapshot() *Snapshot {
	s := &Snapshot{
		db:           db,
		snap:         C.leveldb_create_snapshot(db.db),
		readOpts:     NewReadOptions(),
		iteratorOpts: NewReadOptions(),
	}

	s.readOpts.SetSnapshot(s)
	s.iteratorOpts.SetSnapshot(s)
	s.iteratorOpts.SetFillCache(false)

	return s
}

func (db *DB) NewIterator() *Iterator {
	it := new(Iterator)

	it.it = C.leveldb_create_iterator(db.db, db.iteratorOpts.Opt)

	return it
}

func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, 0, -1, IteratorForward)
}

func (db *DB) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, 0, -1, IteratorBackward)
}

//limit < 0, unlimit
//offset must >= 0, if < 0, will get nothing
func (db *DB) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, offset, limit, IteratorForward)
}

//limit < 0, unlimit
//offset must >= 0, if < 0, will get nothing
func (db *DB) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, offset, limit, IteratorBackward)
}

func (db *DB) put(wo *WriteOptions, key, value []byte) error {
	var errStr *C.char
	var k, v *C.char
	if len(key) != 0 {
		k = (*C.char)(unsafe.Pointer(&key[0]))
	}
	if len(value) != 0 {
		v = (*C.char)(unsafe.Pointer(&value[0]))
	}

	lenk := len(key)
	lenv := len(value)
	C.leveldb_put(
		db.db, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr)

	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}

func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) {
	var errStr *C.char
	var vallen C.size_t
	var k *C.char
	if len(key) != 0 {
		k = (*C.char)(unsafe.Pointer(&key[0]))
	}

	value := C.leveldb_get(
		db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr)

	if errStr != nil {
		return nil, saveError(errStr)
	}

	if value == nil {
		return nil, nil
	}

	defer C.leveldb_free(unsafe.Pointer(value))
	return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil
}

func (db *DB) delete(wo *WriteOptions, key []byte) error {
	var errStr *C.char
	var k *C.char
	if len(key) != 0 {
		k = (*C.char)(unsafe.Pointer(&key[0]))
	}

	C.leveldb_delete(
		db.db, wo.Opt, k, C.size_t(len(key)), &errStr)

	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}
<commit_msg>close add error return<commit_after>package leveldb

/*
#cgo LDFLAGS: -lleveldb
#include <leveldb/c.h>
*/
import "C"

import (
	"encoding/json"
	"os"
	"unsafe"
)

const defaultFilterBits int = 10

type Config struct {
	Path string `json:"path"`

	Compression     bool `json:"compression"`
	BlockSize       int  `json:"block_size"`
	WriteBufferSize int  `json:"write_buffer_size"`
	CacheSize       int  `json:"cache_size"`
	MaxOpenFiles    int  `json:"max_open_files"`
}

type DB struct {
	cfg *Config

	db *C.leveldb_t

	opts *Options

	//for default read and write options
	readOpts     *ReadOptions
	writeOpts    *WriteOptions
	iteratorOpts *ReadOptions

	syncWriteOpts *WriteOptions

	cache *Cache

	filter *FilterPolicy
}

func Open(configJson json.RawMessage) (*DB, error) {
	cfg := new(Config)
	err := json.Unmarshal(configJson, cfg)
	if err != nil {
		return nil, err
	}

	return OpenWithConfig(cfg)
}

func OpenWithConfig(cfg *Config) (*DB, error) {
	if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil {
		return nil, err
	}

	db := new(DB)
	db.cfg = cfg

	if err := db.open(); err != nil {
		return nil, err
	}

	return db, nil
}

func (db *DB) open() error {
	db.initOptions(db.cfg)

	var errStr *C.char
	ldbname := C.CString(db.cfg.Path)
	defer C.leveldb_free(unsafe.Pointer(ldbname))

	db.db = C.leveldb_open(db.opts.Opt, ldbname, &errStr)
	if errStr != nil {
		db.db = nil
		return saveError(errStr)
	}
	return nil
}

func Repair(cfg *Config) error {
	db := new(DB)
	db.cfg = cfg

	err := db.open()
	defer db.Close()

	//open ok, do not need repair
	if err == nil {
		return nil
	}

	var errStr *C.char
	ldbname := C.CString(db.cfg.Path)
	defer C.leveldb_free(unsafe.Pointer(ldbname))

	C.leveldb_repair_db(db.opts.Opt, ldbname, &errStr)
	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}

func (db *DB) initOptions(cfg *Config) {
	opts := NewOptions()

	opts.SetCreateIfMissing(true)

	if cfg.CacheSize <= 0 {
		cfg.CacheSize = 4 * 1024 * 1024
	}

	db.cache = NewLRUCache(cfg.CacheSize)
	opts.SetCache(db.cache)

	//we must use bloomfilter
	db.filter = NewBloomFilter(defaultFilterBits)
	opts.SetFilterPolicy(db.filter)

	if !cfg.Compression {
		opts.SetCompression(NoCompression)
	}

	if cfg.BlockSize <= 0 {
		cfg.BlockSize = 4 * 1024
	}

	opts.SetBlockSize(cfg.BlockSize)

	if cfg.WriteBufferSize <= 0 {
		cfg.WriteBufferSize = 4 * 1024 * 1024
	}

	opts.SetWriteBufferSize(cfg.WriteBufferSize)

	if cfg.MaxOpenFiles < 1024 {
		cfg.MaxOpenFiles = 1024
	}

	opts.SetMaxOpenFiles(cfg.MaxOpenFiles)

	db.opts = opts

	db.readOpts = NewReadOptions()
	db.writeOpts = NewWriteOptions()

	db.iteratorOpts = NewReadOptions()
	db.iteratorOpts.SetFillCache(false)

	db.syncWriteOpts = NewWriteOptions()
	db.syncWriteOpts.SetSync(true)
}

func (db *DB) Close() error {
	if db.db != nil {
		C.leveldb_close(db.db)
		db.db = nil
	}

	db.opts.Close()

	if db.cache != nil {
		db.cache.Close()
	}

	if db.filter != nil {
		db.filter.Close()
	}

	db.readOpts.Close()
	db.writeOpts.Close()
	db.iteratorOpts.Close()
	db.syncWriteOpts.Close()

	return nil
}

func (db *DB) Destroy() error {
	path := db.cfg.Path

	db.Close()

	opts := NewOptions()
	defer opts.Close()

	var errStr *C.char
	ldbname := C.CString(path)
	defer C.leveldb_free(unsafe.Pointer(ldbname))

	C.leveldb_destroy_db(opts.Opt, ldbname, &errStr)
	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}

func (db *DB) Clear() error {
	bc := db.NewWriteBatch()
	defer bc.Close()

	var err error
	it := db.NewIterator()
	it.SeekToFirst()

	num := 0
	for ; it.Valid(); it.Next() {
		bc.Delete(it.Key())
		num++
		if num == 1000 {
			num = 0
			if err = bc.Commit(); err != nil {
				return err
			}
		}
	}

	err = bc.Commit()

	return err
}

func (db *DB) Put(key, value []byte) error {
	return db.put(db.writeOpts, key, value)
}

func (db *DB) SyncPut(key, value []byte) error {
	return db.put(db.syncWriteOpts, key, value)
}

func (db *DB) Get(key []byte) ([]byte, error) {
	return db.get(db.readOpts, key)
}

func (db *DB) Delete(key []byte) error {
	return db.delete(db.writeOpts, key)
}

func (db *DB) SyncDelete(key []byte) error {
	return db.delete(db.syncWriteOpts, key)
}

func (db *DB) NewWriteBatch() *WriteBatch {
	wb := &WriteBatch{
		db:     db,
		wbatch: C.leveldb_writebatch_create(),
	}
	return wb
}

func (db *DB) NewSnapshot() *Snapshot {
	s := &Snapshot{
		db:           db,
		snap:         C.leveldb_create_snapshot(db.db),
		readOpts:     NewReadOptions(),
		iteratorOpts: NewReadOptions(),
	}

	s.readOpts.SetSnapshot(s)
	s.iteratorOpts.SetSnapshot(s)
	s.iteratorOpts.SetFillCache(false)

	return s
}

func (db *DB) NewIterator() *Iterator {
	it := new(Iterator)

	it.it = C.leveldb_create_iterator(db.db, db.iteratorOpts.Opt)

	return it
}

func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, 0, -1, IteratorForward)
}

func (db *DB) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, 0, -1, IteratorBackward)
}

//limit < 0, unlimit
//offset must >= 0, if < 0, will get nothing
func (db *DB) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, offset, limit, IteratorForward)
}

//limit < 0, unlimit
//offset must >= 0, if < 0, will get nothing
func (db *DB) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *RangeLimitIterator {
	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, offset, limit, IteratorBackward)
}

func (db *DB) put(wo *WriteOptions, key, value []byte) error {
	var errStr *C.char
	var k, v *C.char
	if len(key) != 0 {
		k = (*C.char)(unsafe.Pointer(&key[0]))
	}
	if len(value) != 0 {
		v = (*C.char)(unsafe.Pointer(&value[0]))
	}

	lenk := len(key)
	lenv := len(value)
	C.leveldb_put(
		db.db, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr)

	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}

func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) {
	var errStr *C.char
	var vallen C.size_t
	var k *C.char
	if len(key) != 0 {
		k = (*C.char)(unsafe.Pointer(&key[0]))
	}

	value := C.leveldb_get(
		db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr)

	if errStr != nil {
		return nil, saveError(errStr)
	}

	if value == nil {
		return nil, nil
	}

	defer C.leveldb_free(unsafe.Pointer(value))
	return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil
}

func (db *DB) delete(wo *WriteOptions, key []byte) error {
	var errStr *C.char
	var k *C.char
	if len(key) != 0 {
		k = (*C.char)(unsafe.Pointer(&key[0]))
	}

	C.leveldb_delete(
		db.db, wo.Opt, k, C.size_t(len(key)), &errStr)

	if errStr != nil {
		return saveError(errStr)
	}
	return nil
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package irc

import (
	"strings"
)

type Sender struct {
	Name string
	Host string
}

type Message struct {
	Sender   Sender
	Receiver string
	Command  string
	Data     string
}

func parseMessage(line string) Message {
	msg := Message{}
	if strings.HasPrefix(line, ":") {
		idx := strings.Index(line, " ")
		msg.Sender = Sender{Name: line[1:idx]}

		if strings.Contains(msg.Sender.Name, "!") {
			s := strings.Split(msg.Sender.Name, "!")
			msg.Sender.Name = s[0]
			msg.Sender.Host = s[1]
		}

		line = line[idx+1:]
	}

	idx := strings.Index(line, " ")
	msg.Command = strings.ToLower(line[:idx])
	line = line[idx+1:]

	if strings.Contains(line, " ") {
		idx = strings.Index(line, ":")
		if idx >= 0 {
			msg.Receiver = strings.TrimSpace(line[0:idx])
			msg.Data = line[idx+1:]
		}
	} else {
		msg.Data = line[1:]
	}

	return msg
}
<commit_msg>Trim spaces from msg data<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package irc

import (
	"strings"
)

type Sender struct {
	Name string
	Host string
}

type Message struct {
	Sender   Sender
	Receiver string
	Command  string
	Data     string
}

func parseMessage(line string) Message {
	msg := Message{}
	if strings.HasPrefix(line, ":") {
		idx := strings.Index(line, " ")
		msg.Sender = Sender{Name: line[1:idx]}

		if strings.Contains(msg.Sender.Name, "!") {
			s := strings.Split(msg.Sender.Name, "!")
			msg.Sender.Name = s[0]
			msg.Sender.Host = s[1]
		}

		line = line[idx+1:]
	}

	idx := strings.Index(line, " ")
	msg.Command = strings.ToLower(line[:idx])
	line = line[idx+1:]

	if strings.Contains(line, " ") {
		idx = strings.Index(line, ":")
		if idx >= 0 {
			msg.Receiver = strings.TrimSpace(line[0:idx])
			msg.Data = strings.TrimSpace(line[idx+1:])
		}
	} else {
		msg.Data = line[1:]
	}

	return msg
}
<|endoftext|>


<commit_before>package models

import "time"

type FileStats struct {
	Name     string
	Size     int64
	ModTime  time.Time
	IsDir    bool
	SentName string
}
<commit_msg>add is compressed/encrypted to filestats<commit_after>package models

import "time"

// FileStats are the file stats transfered to the other
type FileStats struct {
	Name         string
	Size         int64
	ModTime      time.Time
	IsDir        bool
	SentName     string
	IsCompressed bool
	IsEncrypted  bool
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config_test

import (
	. "github.com/projectcalico/felix/config"

	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"net"
	"reflect"
)

var _ = DescribeTable("Config parsing",
	func(key, value string, expected interface{}) {
		config := New()
		oldVal := reflect.ValueOf(config).Elem().FieldByName(key).Interface()
		config.UpdateFrom(map[string]string{key: value},
			EnvironmentVariable)
		newVal := reflect.ValueOf(config).Elem().FieldByName(key).Interface()
		Expect(oldVal).NotTo(Equal(newVal))
		Expect(newVal).To(Equal(expected))
	},

	Entry("FelixHostname", "FelixHostname", "hostname", "hostname"),
	Entry("FelixHostname FQDN", "FelixHostname", "hostname.foo.bar.com", "hostname.foo.bar.com"),
	Entry("FelixHostname as IP", "FelixHostname", "1.2.3.4", "1.2.3.4"),

	Entry("EtcdAddr IP", "EtcdAddr", "10.0.0.1:1234", "10.0.0.1:1234"),
	Entry("EtcdAddr host", "EtcdAddr", "host:1234", "host:1234"),
	Entry("EtcdScheme", "EtcdScheme", "https", "https"),

	// Etcd key files will be tested for existence, skipping for now.

	Entry("EtcdEndpoints HTTP", "EtcdEndpoints",
		"http://127.0.0.1:1234, http://host:2345",
		[]string{"http://127.0.0.1:1234/", "http://host:2345/"}),
	Entry("EtcdEndpoints HTTPS", "EtcdEndpoints",
		"https://127.0.0.1:1234/, https://host:2345",
		[]string{"https://127.0.0.1:1234/", "https://host:2345/"}),

	Entry("StartupCleanupDelay 12", "StartupCleanupDelay", "12", int(12)),
	Entry("StartupCleanupDelay 0", "StartupCleanupDelay", "0", int(0)),
	Entry("PeriodicResyncInterval 1500", "PeriodicResyncInterval", "1500", int(1500)),
	Entry("PeriodicResyncInterval 0", "PeriodicResyncInterval", "0", int(0)),
	Entry("HostInterfacePollInterval", "HostInterfacePollInterval", "11", int(11)),
	Entry("HostInterfacePollInterval", "HostInterfacePollInterval", "0", int(0)),

	Entry("InterfacePrefix", "InterfacePrefix", "tap", "tap"),
	Entry("InterfacePrefix list", "InterfacePrefix", "tap,cali", "tap,cali"),

	Entry("ChainInsertMode append", "ChainInsertMode", "append", "append"),

	Entry("DefaultEndpointToHostAction", "DefaultEndpointToHostAction",
		"RETURN", "RETURN"),
	Entry("DefaultEndpointToHostAction", "DefaultEndpointToHostAction",
		"ACCEPT", "ACCEPT"),

	Entry("DropActionOverride", "DropActionOverride",
		"ACCEPT", "ACCEPT"),
	Entry("DropActionOverride norm", "DropActionOverride",
		"accept", "ACCEPT"),
	Entry("DropActionOverride LOG-and-ACCEPT", "DropActionOverride",
		"LOG-and-ACCEPT", "LOG-and-ACCEPT"),
	Entry("DropActionOverride log-and-accept", "DropActionOverride",
		"log-and-accept", "LOG-and-ACCEPT"),
	Entry("DropActionOverride log-and-drop", "DropActionOverride",
		"log-and-drop", "LOG-and-DROP"),

	Entry("LogFilePath", "LogFilePath", "/tmp/felix.log", "/tmp/felix.log"),

	Entry("LogSeverityFile", "LogSeverityFile", "debug", "DEBUG"),
	Entry("LogSeverityFile", "LogSeverityFile", "warning", "WARNING"),
	Entry("LogSeverityFile", "LogSeverityFile", "error", "ERROR"),
	Entry("LogSeverityFile", "LogSeverityFile", "critical", "CRITICAL"),

	Entry("LogSeverityScreen", "LogSeverityScreen", "debug", "DEBUG"),
	Entry("LogSeverityScreen", "LogSeverityScreen", "warning", "WARNING"),
	Entry("LogSeverityScreen", "LogSeverityScreen", "error", "ERROR"),
	Entry("LogSeverityScreen", "LogSeverityScreen", "critical", "CRITICAL"),

	Entry("LogSeveritySys", "LogSeveritySys", "debug", "DEBUG"),
	Entry("LogSeveritySys", "LogSeveritySys", "warning", "WARNING"),
	Entry("LogSeveritySys", "LogSeveritySys", "error", "ERROR"),
	Entry("LogSeveritySys", "LogSeveritySys", "critical", "CRITICAL"),

	Entry("IpInIpEnabled", "IpInIpEnabled", "true", true),
	Entry("IpInIpEnabled", "IpInIpEnabled", "y", true),
	Entry("IpInIpEnabled", "IpInIpEnabled", "True", true),

	Entry("IpInIpMtu", "IpInIpMtu", "1234", int(1234)),
	Entry("IpInIpTunnelAddr", "IpInIpTunnelAddr",
		"10.0.0.1", net.ParseIP("10.0.0.1")),

	Entry("ReportingIntervalSecs", "ReportingIntervalSecs", "31", int(31)),
	Entry("ReportingTTLSecs", "ReportingTTLSecs", "91", int(91)),

	Entry("EndpointReportingEnabled", "EndpointReportingEnabled",
		"true", true),
	Entry("EndpointReportingEnabled", "EndpointReportingEnabled",
		"yes", true),
	Entry("EndpointReportingDelaySecs", "EndpointReportingDelaySecs",
		"10", float64(10)),

	Entry("MaxIpsetSize", "MaxIpsetSize", "12345", int(12345)),
	Entry("IptablesMarkMask", "IptablesMarkMask", "0xf0f0", uint32(0xf0f0)),

	Entry("PrometheusMetricsEnabled", "PrometheusMetricsEnabled", "true", true),
	Entry("PrometheusMetricsPort", "PrometheusMetricsPort", "1234", int(1234)),

	Entry("FailsafeInboundHostPorts", "FailsafeInboundHostPorts", "1,2,3,4", []uint16{1, 2, 3, 4}),
	Entry("FailsafeOutboundHostPorts", "FailsafeOutboundHostPorts", "1,2,3,4", []uint16{1, 2, 3, 4}),
)

var _ = DescribeTable("Mark bit calculation tests",
	func(mask string, bitNum int, expected uint32) {
		config := New()
		config.UpdateFrom(map[string]string{"IptablesMarkMask": mask}, EnvironmentVariable)
		Expect(config.NthIPTablesMark(bitNum)).To(Equal(expected))
	},
	Entry("0th bit in 0xf", "0xf", 0, uint32(0x1)),
	Entry("1st bit in 0xf", "0xf", 1, uint32(0x2)),
	Entry("7th bit in 0xff", "0xff", 7, uint32(0x80)),
	Entry("0th bit of 0xff000000", "0xff000000", 0, uint32(0x01000000)),
)

var _ = DescribeTable("Next mark bit calculation tests",
	func(mask string, numCalls int, expected uint32) {
		config := New()
		config.UpdateFrom(map[string]string{"IptablesMarkMask": mask}, EnvironmentVariable)
		var mark uint32
		for i := 0; i < numCalls; i++ {
			mark = config.NextIptablesMark()
		}
		Expect(mark).To(Equal(expected))
	},
	Entry("0th bit in 0xf", "0xf", 1, uint32(0x1)),
	Entry("1st bit in 0xf", "0xf", 2, uint32(0x2)),
	Entry("7th bit in 0xff", "0xff", 8, uint32(0x80)),
	Entry("0th bit of 0xff000000", "0xff000000", 1, uint32(0x01000000)),
)
<commit_msg>Add more UT.<commit_after>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config_test

import (
	. "github.com/projectcalico/felix/config"

	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"net"
	"reflect"
)

var _ = DescribeTable("Config parsing",
	func(key, value string, expected interface{}) {
		config := New()
		oldVal := reflect.ValueOf(config).Elem().FieldByName(key).Interface()
		config.UpdateFrom(map[string]string{key: value},
			EnvironmentVariable)
		newVal := reflect.ValueOf(config).Elem().FieldByName(key).Interface()
		Expect(oldVal).NotTo(Equal(newVal))
		Expect(newVal).To(Equal(expected))
	},

	Entry("FelixHostname", "FelixHostname", "hostname", "hostname"),
	Entry("FelixHostname FQDN", "FelixHostname", "hostname.foo.bar.com", "hostname.foo.bar.com"),
	Entry("FelixHostname as IP", "FelixHostname", "1.2.3.4", "1.2.3.4"),

	Entry("EtcdAddr IP", "EtcdAddr", "10.0.0.1:1234", "10.0.0.1:1234"),
	Entry("EtcdAddr host", "EtcdAddr", "host:1234", "host:1234"),
	Entry("EtcdScheme", "EtcdScheme", "https", "https"),

	// Etcd key files will be tested for existence, skipping for now.

	Entry("EtcdEndpoints HTTP", "EtcdEndpoints",
		"http://127.0.0.1:1234, http://host:2345",
		[]string{"http://127.0.0.1:1234/", "http://host:2345/"}),
	Entry("EtcdEndpoints HTTPS", "EtcdEndpoints",
		"https://127.0.0.1:1234/, https://host:2345",
		[]string{"https://127.0.0.1:1234/", "https://host:2345/"}),

	Entry("StartupCleanupDelay 12", "StartupCleanupDelay", "12", int(12)),
	Entry("StartupCleanupDelay 0", "StartupCleanupDelay", "0", int(0)),
	Entry("PeriodicResyncInterval 1500", "PeriodicResyncInterval", "1500", int(1500)),
	Entry("PeriodicResyncInterval 0", "PeriodicResyncInterval", "0", int(0)),
	Entry("HostInterfacePollInterval", "HostInterfacePollInterval", "11", int(11)),
	Entry("HostInterfacePollInterval", "HostInterfacePollInterval", "0", int(0)),

	Entry("InterfacePrefix", "InterfacePrefix", "tap", "tap"),
	Entry("InterfacePrefix list", "InterfacePrefix", "tap,cali", "tap,cali"),

	Entry("ChainInsertMode append", "ChainInsertMode", "append", "append"),

	Entry("DefaultEndpointToHostAction", "DefaultEndpointToHostAction",
		"RETURN", "RETURN"),
	Entry("DefaultEndpointToHostAction", "DefaultEndpointToHostAction",
		"ACCEPT", "ACCEPT"),

	Entry("DropActionOverride", "DropActionOverride",
		"ACCEPT", "ACCEPT"),
	Entry("DropActionOverride norm", "DropActionOverride",
		"accept", "ACCEPT"),
	Entry("DropActionOverride LOG-and-ACCEPT", "DropActionOverride",
		"LOG-and-ACCEPT", "LOG-and-ACCEPT"),
	Entry("DropActionOverride log-and-accept", "DropActionOverride",
		"log-and-accept", "LOG-and-ACCEPT"),
	Entry("DropActionOverride log-and-drop", "DropActionOverride",
		"log-and-drop", "LOG-and-DROP"),

	Entry("LogFilePath", "LogFilePath", "/tmp/felix.log", "/tmp/felix.log"),

	Entry("LogSeverityFile", "LogSeverityFile", "debug", "DEBUG"),
	Entry("LogSeverityFile", "LogSeverityFile", "warning", "WARNING"),
	Entry("LogSeverityFile", "LogSeverityFile", "error", "ERROR"),
	Entry("LogSeverityFile", "LogSeverityFile", "critical", "CRITICAL"),

	Entry("LogSeverityScreen", "LogSeverityScreen", "debug", "DEBUG"),
	Entry("LogSeverityScreen", "LogSeverityScreen", "warning", "WARNING"),
	Entry("LogSeverityScreen", "LogSeverityScreen", "error", "ERROR"),
	Entry("LogSeverityScreen", "LogSeverityScreen", "critical", "CRITICAL"),

	Entry("LogSeveritySys", "LogSeveritySys", "debug", "DEBUG"),
	Entry("LogSeveritySys", "LogSeveritySys", "warning", "WARNING"),
	Entry("LogSeveritySys", "LogSeveritySys", "error", "ERROR"),
	Entry("LogSeveritySys", "LogSeveritySys", "critical", "CRITICAL"),

	Entry("IpInIpEnabled", "IpInIpEnabled", "true", true),
	Entry("IpInIpEnabled", "IpInIpEnabled", "y", true),
	Entry("IpInIpEnabled", "IpInIpEnabled", "True", true),

	Entry("IpInIpMtu", "IpInIpMtu", "1234", int(1234)),
	Entry("IpInIpTunnelAddr", "IpInIpTunnelAddr",
		"10.0.0.1", net.ParseIP("10.0.0.1")),

	Entry("ReportingIntervalSecs", "ReportingIntervalSecs", "31", int(31)),
	Entry("ReportingTTLSecs", "ReportingTTLSecs", "91", int(91)),

	Entry("EndpointReportingEnabled", "EndpointReportingEnabled",
		"true", true),
	Entry("EndpointReportingEnabled", "EndpointReportingEnabled",
		"yes", true),
	Entry("EndpointReportingDelaySecs", "EndpointReportingDelaySecs",
		"10", float64(10)),

	Entry("MaxIpsetSize", "MaxIpsetSize", "12345", int(12345)),
	Entry("IptablesMarkMask", "IptablesMarkMask", "0xf0f0", uint32(0xf0f0)),

	Entry("PrometheusMetricsEnabled", "PrometheusMetricsEnabled", "true", true),
	Entry("PrometheusMetricsPort", "PrometheusMetricsPort", "1234", int(1234)),

	Entry("FailsafeInboundHostPorts", "FailsafeInboundHostPorts", "1,2,3,4", []uint16{1, 2, 3, 4}),
	Entry("FailsafeOutboundHostPorts", "FailsafeOutboundHostPorts", "1,2,3,4", []uint16{1, 2, 3, 4}),
)

var _ = DescribeTable("Mark bit calculation tests",
	func(mask string, bitNum int, expected uint32) {
		config := New()
		config.UpdateFrom(map[string]string{"IptablesMarkMask": mask}, EnvironmentVariable)
		Expect(config.NthIPTablesMark(bitNum)).To(Equal(expected))
	},
	Entry("0th bit in 0xf", "0xf", 0, uint32(0x1)),
	Entry("1st bit in 0xf", "0xf", 1, uint32(0x2)),
	Entry("7th bit in 0xff", "0xff", 7, uint32(0x80)),
	Entry("4th bit in 0xf00f", "0xf00f", 4, uint32(0x1000)),
	Entry("3rd bit in 0xf00f", "0xf00f", 3, uint32(0x0008)),
	Entry("7th bit in 0xf00f", "0xf00f", 7, uint32(0x8000)),
	Entry("0th bit of 0xff000000", "0xff000000", 0, uint32(0x01000000)),
)

var _ = DescribeTable("Next mark bit calculation tests",
	func(mask string, numCalls int, expected uint32) {
		config := New()
		config.UpdateFrom(map[string]string{"IptablesMarkMask": mask}, EnvironmentVariable)
		var mark uint32
		for i := 0; i < numCalls; i++ {
			mark = config.NextIptablesMark()
		}
		Expect(mark).To(Equal(expected))
	},
	Entry("0th bit in 0xf", "0xf", 1, uint32(0x1)),
	Entry("1st bit in 0xf", "0xf", 2, uint32(0x2)),
	Entry("7th bit in 0xff", "0xff", 8, uint32(0x80)),
	Entry("7th bit in 0xf00f", "0xf00f", 8, uint32(0x8000)),
	Entry("0th bit of 0xff000000", "0xff000000", 1, uint32(0x01000000)),
)
<|endoftext|>


<commit_before>package scraper

// TODO: The date extracting does not quite work, off by a couple of hours

import (
	"hnews/services"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"hnews/Godeps/_workspace/src/github.com/yhat/scrape"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html/atom"
)

// Scraper ...
type Scraper struct {
}

// NewScraper returns a new Scraper which collects all static information from HN
func NewScraper() *Scraper {
	scraper := new(Scraper)
	go startScraping()
	return scraper
}

func startScraping() {
	newsCh := make(chan []services.News)
	commentsCh := make(chan []services.Comment)
	go scrapeFrontPage(newsCh)
	go scrapeComments(commentsCh)

	for {
		select {
		case newNews := <-newsCh:
			go services.SaveNews(newNews)
		case newComments := <-commentsCh:
			log.Println(len(newComments), "new comments.")
			go services.SaveComments(newComments)
		}
	}
}

/********************** News **********************/
// Parses the front page and sends a []News of the content
func scrapeFrontPage(newsCh chan []services.News) {
	for {
		for i := 1; i <= 16; i++ {
			var news []services.News

			url := "https://news.ycombinator.com/news?p=" + strconv.Itoa(i)
			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			points := parsePoints(root)
			ranks := parseRanks(root)
			titles, links := parseArticles(root)
			authors := parseAuthors(root)
			times := parseTimes(root)
			comments := parseNumComments(root)
			ids := parseIDs(root)

			for i := 0; i < len(ranks); i++ {
				rank := int32(ranks[i])
				title := titles[i]

				var time time.Time
				if i < len(times) {
					time = times[i]
				}

				var link string
				if i < len(links) {
					link = links[i]
				}

				var author string
				if i < len(authors) {
					author = authors[i]
				}

				var numPoints int32
				if i < len(points) {
					numPoints = int32(points[i])
				}

				var numComments int32
				if i < len(comments) {
					numComments = int32(comments[i])
				}

				var id int32
				if i < len(ids) {
					id = int32(ids[i])
				}

				news = append(news, services.News{id, rank, title, link, author, numPoints, time, numComments})
			}
			newsCh <- news
		}
		time.Sleep(1 * time.Second)
	}
}

// Parses out the rank of the articles.
func parseRanks(root *html.Node) []int {
	var ranks []int

	rankMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil && n.Parent.Parent != nil {
			grandparent := scrape.Attr(n.Parent.Parent, "class") == "athing"
			self := scrape.Attr(n, "class") == "rank"
			return grandparent && self
		}
		return false
	}

	rankNodes := scrape.FindAll(root, rankMatcher)
	for _, rankNode := range rankNodes {
		text := strings.Replace(scrape.Text(rankNode), ".", "", -1)
		rank, err := strconv.Atoi(text)
		if err != nil {
			rank = 0
		}
		ranks = append(ranks, rank)
	}
	return ranks
}

// Parses the authors of each Story on the frontpage
func parseAuthors(root *html.Node) []string {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "user" {
				return parent && true
			}
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	return authors
}

// Parses the number of comments on each Story on the frontpage
func parseNumComments(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "item" {
				return parent && true
			}
		}
		return false
	}

	var numComms []int // Number of comments for each Story
	commentNodes := scrape.FindAll(root, matcher)
	for _, commentNode := range commentNodes {
		text := scrape.Text(commentNode)
		words := strings.Fields(text)
		numComm, err := strconv.Atoi(words[0])
		if err != nil {
			numComm = 0
		}
		numComms = append(numComms, numComm)
	}
	return numComms
}

// Parses the timestamps from the HTML of the frontpage
func parseTimes(root *html.Node) []time.Time {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		text := scrape.Text(timeNode)
		date, err := parseTimeString(text)
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	return dates
}

// Parses the id of each Story on the page
func parseIDs(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		idTemp := scrape.Attr(idNode, "id")
		if len(idTemp) <= 6 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(idTemp[6:len(idTemp)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	return ids
}

// Quantity is of "hours"/"days"/"minutes"
// Text is of "4 hours ago", "41 days ago", etc
func parseTimeString(text string) (time.Time, error) {
	words := strings.Fields(text)

	timeAgo, err := strconv.Atoi(words[0])
	if err != nil {
		return time.Time{}, err
	}

	now := time.Now()
	var result time.Time
	switch words[1] {
	case "minutes":
		result = now.Add(time.Duration(-timeAgo) * time.Minute)
	case "hours", "hour":
		result = now.Add(time.Duration(-timeAgo) * time.Hour)
	case "days", "day":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	}
	return result, err
}

// Parses the title and the link of each Story on the page
func parseArticles(root *html.Node) ([]string, []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "class") == "athing"
		}
		return false
	}

	var titles []string
	var links []string

	articles := scrape.FindAll(root, matcher)
	for _, article := range articles {
		title := scrape.Text(article)
		link := scrape.Attr(article, "href")
		titles = append(titles, title)
		links = append(links, link)
	}
	return titles, links
}

func parsePoints(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		// must check for nil values
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var points []int

	pointsNodes := scrape.FindAll(root, matcher)
	for _, pointsNode := range pointsNodes {
		pointS := strings.Replace(scrape.Text(pointsNode), " points", "", -1)
		point, err := strconv.Atoi(pointS)
		if err != nil {
			point = 0
		}
		points = append(points, point)
	}
	return points
}

/********************** News **********************/

/******************** Comments ********************/
// Scrapes the Comments for every News item currently in the database.
func scrapeComments(commentsCh chan []services.Comment) {
	for {
		ids := services.ReadNewsIds()
		for _, id := range ids {
			url := "https://news.ycombinator.com/item?id=" + strconv.Itoa(int(id))

			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}
			go parseComments(root, id, commentsCh)
		}
		//time.Sleep(1 * time.Second)
	}
}

// Parses all the Comments for a particular News item.
func parseComments(root *html.Node, newsid int32, commentsCh chan []services.Comment) {
	offsets := parseOffsets(root)
	ids := parseCommentIDs(root)
	authors := parseCommentAuthors(root)
	times := parseCommentTimes(root)
	texts := parseCommentText(root)

	var comments []services.Comment
	for i := range authors {
		comment := services.Comment{int32(i + 1), newsid, int32(ids[i]), int32(offsets[i]),
			times[i], authors[i], texts[i]}
		comments = append(comments, comment)
	}
	commentsCh <- comments
}

// Parses the level for each Comment in the comment tree. Interval: 0-inf.
func parseOffsets(root *html.Node) []int {
	offsetMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Img && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "ind"
			return parent
		}
		return false
	}

	offsets := scrape.FindAll(root, offsetMatcher)

	var norm []int
	for _, offset := range offsets {
		lvl, _ := strconv.Atoi(scrape.Attr(offset, "width"))
		norm = append(norm, int(lvl/40))
	}
	return norm
}

// Parses all the comments authors
func parseCommentAuthors(root *html.Node) []string {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "comhead"
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	return authors
}

// Parses all the comments itemids
func parseCommentIDs(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		href := scrape.Attr(idNode, "href")
		if len(href) <= 8 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(href[8:len(href)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	return ids
}

// Parses all the comments timestamps
func parseCommentTimes(root *html.Node) []time.Time {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		date, err := parseTimeString(scrape.Text(timeNode))
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	return dates
}

// Parses the text of all Comments for a News
func parseCommentText(root *html.Node) []string {
	textMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "comment"
			return parent
		}
		return false
	}

	textNodes := scrape.FindAll(root, textMatcher)

	var texts []string
	for _, text := range textNodes {
		content := scrape.Text(text)
		// TODO: Remove trailing trash from the 'Reply' HTML node ...
		texts = append(texts, content) //[0:len(content)-5])
	}
	return texts
}

/******************** Comments ********************/
<commit_msg>Tried to increase performance with some more goroutines. Did not help much. Need profiling.<commit_after>package scraper

// TODO: The date extracting does not quite work, off by a couple of hours

import (
	"hnews/services"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"hnews/Godeps/_workspace/src/github.com/yhat/scrape"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html/atom"
)

// Scraper ...
type Scraper struct {
}

// NewScraper returns a new Scraper which collects all static information from HN
func NewScraper() *Scraper {
	scraper := new(Scraper)
	go startScraping()
	return scraper
}

func startScraping() {
	newsCh := make(chan []services.News)
	commentsCh := make(chan []services.Comment)
	go scrapeFrontPage(newsCh)
	go scrapeComments(commentsCh)

	for {
		select {
		case newNews := <-newsCh:
			log.Println(len(newNews), "new news.")
			go services.SaveNews(newNews)
		case newComments := <-commentsCh:
			log.Println(len(newComments), "new comments.")
			go services.SaveComments(newComments)
		}
	}
}

/********************** News **********************/
// Parses the front page and sends a []News of the content
func scrapeFrontPage(newsCh chan []services.News) {
	for {
		for i := 1; i <= 16; i++ {
			var news []services.News

			url := "https://news.ycombinator.com/news?p=" + strconv.Itoa(i)
			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			points := parsePoints(root)
			ranks := parseRanks(root)
			titles, links := parseArticles(root)
			authors := parseAuthors(root)
			times := parseTimes(root)
			comments := parseNumComments(root)
			ids := parseIDs(root)

			for i := 0; i < len(ranks); i++ {
				rank := int32(ranks[i])
				title := titles[i]

				var time time.Time
				if i < len(times) {
					time = times[i]
				}

				var link string
				if i < len(links) {
					link = links[i]
				}

				var author string
				if i < len(authors) {
					author = authors[i]
				}

				var numPoints int32
				if i < len(points) {
					numPoints = int32(points[i])
				}

				var numComments int32
				if i < len(comments) {
					numComments = int32(comments[i])
				}

				var id int32
				if i < len(ids) {
					id = int32(ids[i])
				}

				news = append(news, services.News{id, rank, title, link, author, numPoints, time, numComments})
			}
			newsCh <- news
		}
		time.Sleep(1 * time.Second)
	}
}

// Parses out the rank of the articles.
func parseRanks(root *html.Node) []int {
	var ranks []int

	rankMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil && n.Parent.Parent != nil {
			grandparent := scrape.Attr(n.Parent.Parent, "class") == "athing"
			self := scrape.Attr(n, "class") == "rank"
			return grandparent && self
		}
		return false
	}

	rankNodes := scrape.FindAll(root, rankMatcher)
	for _, rankNode := range rankNodes {
		text := strings.Replace(scrape.Text(rankNode), ".", "", -1)
		rank, err := strconv.Atoi(text)
		if err != nil {
			rank = 0
		}
		ranks = append(ranks, rank)
	}
	return ranks
}

// Parses the authors of each Story on the frontpage
func parseAuthors(root *html.Node) []string {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "user" {
				return parent && true
			}
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	return authors
}

// Parses the number of comments on each Story on the frontpage
func parseNumComments(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "item" {
				return parent && true
			}
		}
		return false
	}

	var numComms []int // Number of comments for each Story
	commentNodes := scrape.FindAll(root, matcher)
	for _, commentNode := range commentNodes {
		text := scrape.Text(commentNode)
		words := strings.Fields(text)
		numComm, err := strconv.Atoi(words[0])
		if err != nil {
			numComm = 0
		}
		numComms = append(numComms, numComm)
	}
	return numComms
}

// Parses the timestamps from the HTML of the frontpage
func parseTimes(root *html.Node) []time.Time {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		text := scrape.Text(timeNode)
		date, err := parseTimeString(text)
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	return dates
}

// Parses the id of each Story on the page
func parseIDs(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		idTemp := scrape.Attr(idNode, "id")
		if len(idTemp) <= 6 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(idTemp[6:len(idTemp)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	return ids
}

// Quantity is of "hours"/"days"/"minutes"
// Text is of "4 hours ago", "41 days ago", etc
func parseTimeString(text string) (time.Time, error) {
	words := strings.Fields(text)

	timeAgo, err := strconv.Atoi(words[0])
	if err != nil {
		return time.Time{}, err
	}

	now := time.Now()
	var result time.Time
	switch words[1] {
	case "minutes":
		result = now.Add(time.Duration(-timeAgo) * time.Minute)
	case "hours", "hour":
		result = now.Add(time.Duration(-timeAgo) * time.Hour)
	case "days", "day":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	}
	return result, err
}

// Parses the title and the link of each Story on the page
func parseArticles(root *html.Node) ([]string, []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "class") == "athing"
		}
		return false
	}

	var titles []string
	var links []string

	articles := scrape.FindAll(root, matcher)
	for _, article := range articles {
		title := scrape.Text(article)
		link := scrape.Attr(article, "href")
		titles = append(titles, title)
		links = append(links, link)
	}
	return titles, links
}

func parsePoints(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var points []int

	pointsNodes := scrape.FindAll(root, matcher)
	for _, pointsNode := range pointsNodes {
		pointS := strings.Replace(scrape.Text(pointsNode), " points", "", -1)
		point, err := strconv.Atoi(pointS)
		if err != nil {
			point = 0
		}
		points = append(points, point)
	}
	return points
}

/********************** News **********************/

/******************** Comments ********************/
// Scrapes the Comments for every News item currently in the database.
func scrapeComments(commentsCh chan []services.Comment) {
	for {
		ids := services.ReadNewsIds()
		for _, id := range ids {
			url := "https://news.ycombinator.com/item?id=" + strconv.Itoa(int(id))

			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}
			go parseComments(root, id, commentsCh)
		}
	}
}

// Parses all the Comments for a particular News item.
func parseComments(root *html.Node, newsid int32, commentsCh chan []services.Comment) {
	offsetsCh := make(chan []int)
	idsCh := make(chan []int)
	authorsCh := make(chan []string)
	timesCh := make(chan []time.Time)
	textsCh := make(chan []string)
	go parseOffsets(root, offsetsCh)
	go parseCommentIDs(root, idsCh)
	go parseCommentAuthors(root, authorsCh)
	go parseCommentTimes(root, timesCh)
	go parseCommentText(root, textsCh)
	offsets := <-offsetsCh
	ids := <-idsCh
	authors := <-authorsCh
	times := <-timesCh
	texts := <-textsCh

	var comments []services.Comment
	for i := range authors {
		comment := services.Comment{int32(i + 1), newsid, int32(ids[i]), int32(offsets[i]),
			times[i], authors[i], texts[i]}
		comments = append(comments, comment)
	}
	commentsCh <- comments
}

// Parses the level for each Comment in the comment tree. Interval: 0-inf.
func parseOffsets(root *html.Node, ch chan []int) {
	offsetMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Img && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "ind"
			return parent
		}
		return false
	}

	offsets := scrape.FindAll(root, offsetMatcher)

	var norm []int
	for _, offset := range offsets {
		lvl, _ := strconv.Atoi(scrape.Attr(offset, "width"))
		norm = append(norm, int(lvl/40))
	}
	ch <- norm
}

// Parses all the comments authors
func parseCommentAuthors(root *html.Node, ch chan []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "comhead"
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	ch <- authors
}

// Parses all the comments itemids
func parseCommentIDs(root *html.Node, ch chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		href := scrape.Attr(idNode, "href")
		if len(href) <= 8 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(href[8:len(href)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	ch <- ids
}

// Parses all the comments timestamps
func parseCommentTimes(root *html.Node, ch chan []time.Time) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		date, err := parseTimeString(scrape.Text(timeNode))
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	ch <- dates
}

// Parses the text of all Comments for a News
func parseCommentText(root *html.Node, ch chan []string) {
	textMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "comment"
			return parent
		}
		return false
	}

	textNodes := scrape.FindAll(root, textMatcher)

	var texts []string
	for _, text := range textNodes {
		content := scrape.Text(text)
		// TODO: Remove trailing trash from the 'Reply' HTML node ...
		texts = append(texts, content) //[0:len(content)-5])
	}
	ch <- texts
}

/******************** Comments ********************/
<|endoftext|>


<commit_before>package slack

import (
	"encoding/json"
	"fmt"
)

// AckMessage is used for messages received in reply to other messages
type AckMessage struct {
	ReplyTo   int    `json:"reply_to"`
	Timestamp string `json:"ts"`
	Text      string `json:"text"`
	RTMResponse
}

// RTMResponse encapsulates response details as returned by the Slack API
type RTMResponse struct {
	Ok    bool      `json:"ok"`
	Error *RTMError `json:"error"`
}

// RTMError encapsulates error information as returned by the Slack API
type RTMError struct {
	Code int
	Msg  string
}

func (s RTMError) Error() string {
	return fmt.Sprintf("Code %d - %s", s.Code, s.Msg)
}

// MessageEvent represents a Slack Message (used as the event type for an incoming message)
type MessageEvent Message

// RTMEvent is the main wrapper. You will find all the other messages attached
type RTMEvent struct {
	Type string
	Data interface{}
}

// HelloEvent represents the hello event
type HelloEvent struct{}

// PresenceChangeEvent represents the presence change event
type PresenceChangeEvent struct {
	Type     string `json:"type"`
	Presence string `json:"presence"`
	User     string `json:"user"`
}

// UserTypingEvent represents the user typing event
type UserTypingEvent struct {
	Type    string `json:"type"`
	User    string `json:"user"`
	Channel string `json:"channel"`
}

// PrefChangeEvent represents a user preferences change event
type PrefChangeEvent struct {
	Type  string          `json:"type"`
	Name  string          `json:"name"`
	Value json.RawMessage `json:"value"`
}

// ManualPresenceChangeEvent represents the manual presence change event
type ManualPresenceChangeEvent struct {
	Type     string `json:"type"`
	Presence string `json:"presence"`
}

// UserChangeEvent represents the user change event
type UserChangeEvent struct {
	Type string `json:"type"`
	User User   `json:"user"`
}

// EmojiChangedEvent represents the emoji changed event
type EmojiChangedEvent struct {
	Type           string `json:"type"`
	EventTimestamp string `json:"event_ts"`
}

// CommandsChangedEvent represents the commands changed event
type CommandsChangedEvent struct {
	Type           string `json:"type"`
	EventTimestamp string `json:"event_ts"`
}

// EmailDomainChangedEvent represents the email domain changed event
type EmailDomainChangedEvent struct {
	Type           string `json:"type"`
	EventTimestamp string `json:"event_ts"`
	EmailDomain    string `json:"email_domain"`
}

// BotAddedEvent represents the bot added event
type BotAddedEvent struct {
	Type string `json:"type"`
	Bot  Bot    `json:"bot"`
}

// BotChangedEvent represents the bot changed event
type BotChangedEvent struct {
	Type string `json:"type"`
	Bot  Bot    `json:"bot"`
}

// AccountsChangedEvent represents the accounts changed event
type AccountsChangedEvent struct {
	Type string `json:"type"`
}

// ReconnectUrlEvent represents the receiving reconnect url event
type ReconnectUrlEvent struct {
	Type string `json:"type"`
	URL  string `json:"url"`
}
<commit_msg>add fields to EmojiChangedEvent<commit_after>package slack

import (
	"encoding/json"
	"fmt"
)

// AckMessage is used for messages received in reply to other messages
type AckMessage struct {
	ReplyTo   int    `json:"reply_to"`
	Timestamp string `json:"ts"`
	Text      string `json:"text"`
	RTMResponse
}

// RTMResponse encapsulates response details as returned by the Slack API
type RTMResponse struct {
	Ok    bool      `json:"ok"`
	Error *RTMError `json:"error"`
}

// RTMError encapsulates error information as returned by the Slack API
type RTMError struct {
	Code int
	Msg  string
}

func (s RTMError) Error() string {
	return fmt.Sprintf("Code %d - %s", s.Code, s.Msg)
}

// MessageEvent represents a Slack Message (used as the event type for an incoming message)
type MessageEvent Message

// RTMEvent is the main wrapper. You will find all the other messages attached
type RTMEvent struct {
	Type string
	Data interface{}
}

// HelloEvent represents the hello event
type HelloEvent struct{}

// PresenceChangeEvent represents the presence change event
type PresenceChangeEvent struct {
	Type     string `json:"type"`
	Presence string `json:"presence"`
	User     string `json:"user"`
}

// UserTypingEvent represents the user typing event
type UserTypingEvent struct {
	Type    string `json:"type"`
	User    string `json:"user"`
	Channel string `json:"channel"`
}

// PrefChangeEvent represents a user preferences change event
type PrefChangeEvent struct {
	Type  string          `json:"type"`
	Name  string          `json:"name"`
	Value json.RawMessage `json:"value"`
}

// ManualPresenceChangeEvent represents the manual presence change event
type ManualPresenceChangeEvent struct {
	Type     string `json:"type"`
	Presence string `json:"presence"`
}

// UserChangeEvent represents the user change event
type UserChangeEvent struct {
	Type string `json:"type"`
	User User   `json:"user"`
}

// EmojiChangedEvent represents the emoji changed event
type EmojiChangedEvent struct {
	Type           string   `json:"type"`
	SubType        string   `json:"subtype"`
	Name           string   `json:"name"`
	Names          []string `json:"names"`
	EventTimestamp string   `json:"event_ts"`
}

// CommandsChangedEvent represents the commands changed event
type CommandsChangedEvent struct {
	Type           string `json:"type"`
	EventTimestamp string `json:"event_ts"`
}

// EmailDomainChangedEvent represents the email domain changed event
type EmailDomainChangedEvent struct {
	Type           string `json:"type"`
	EventTimestamp string `json:"event_ts"`
	EmailDomain    string `json:"email_domain"`
}

// BotAddedEvent represents the bot added event
type BotAddedEvent struct {
	Type string `json:"type"`
	Bot  Bot    `json:"bot"`
}

// BotChangedEvent represents the bot changed event
type BotChangedEvent struct {
	Type string `json:"type"`
	Bot  Bot    `json:"bot"`
}

// AccountsChangedEvent represents the accounts changed event
type AccountsChangedEvent struct {
	Type string `json:"type"`
}

// ReconnectUrlEvent represents the receiving reconnect url event
type ReconnectUrlEvent struct {
	Type string `json:"type"`
	URL  string `json:"url"`
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// Structs for XML building
type Package struct {
	Xmlns   string     `xml:"xmlns,attr"`
	Types   []MetaType `xml:"types"`
	Version string     `xml:"version"`
}

type MetaType struct {
	Members []string `xml:"members"`
	Name    string   `xml:"name"`
}

func createPackage() Package {
	return Package{
		Version: strings.TrimPrefix(apiVersion, "v"),
		Xmlns:   "http://soap.sforce.com/2006/04/metadata",
	}
}

type metapath struct {
	path string
	name string
	hasFolder bool
	onlyFolder bool
}

var metapaths = []metapath{
	metapath{path: "applications", name: "CustomApplication"},
	metapath{path: "assignmentRules", name: "AssignmentRules"},
	metapath{path: "aura", name: "AuraDefinitionBundle", hasFolder: true, onlyFolder: true},
	metapath{path: "autoResponseRules", name: "AutoResponseRules"},
	metapath{path: "classes", name: "ApexClass"},
	metapath{path: "communities", name: "Community"},
	metapath{path: "components", name: "ApexComponent"},
	metapath{path: "connectedApps", name: "ConnectedApp"},
	metapath{path: "dashboards", name: "Dashboard", hasFolder: true},
	metapath{path: "documents", name: "Document", hasFolder: true},
	metapath{path: "email", name: "EmailTemplate", hasFolder: true},
	metapath{path: "flexipages", name: "FlexiPage"},
	metapath{path: "groups", name: "Group"},
	metapath{path: "homePageLayouts", name: "HomePageLayout"},
	metapath{path: "labels", name: "CustomLabels"},
	metapath{path: "layouts", name: "Layout"},
	metapath{path: "objects", name: "CustomObject"},
	metapath{path: "objectTranslations", name: "CustomObjectTranslation"},
	metapath{path: "pages", name: "ApexPage"},
	metapath{path: "permissionsets", name: "PermissionSet"},
	metapath{path: "profiles", name: "Profile"},
	metapath{path: "quickActions", name: "QuickAction"},
	metapath{path: "remoteSiteSettings", name: "RemoteSiteSetting"},
	metapath{path: "reports", name: "Report", hasFolder: true},
	metapath{path: "roles", name: "Role"},
	metapath{path: "settings", name: "Settings"},
	metapath{path: "staticresources", name: "StaticResource"},
	metapath{path: "tabs", name: "CustomTab"},
	metapath{path: "triggers", name: "ApexTrigger"},
	metapath{path: "workflows", name: "Workflow"},
}

type PackageBuilder struct {
	IsPush   bool
	Metadata map[string]MetaType
	Files    ForceMetadataFiles
}

func NewPushBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: true}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

func NewFetchBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: false}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

// Build and return package.xml
func (pb PackageBuilder) PackageXml() []byte {
	p := createPackage()

	for _, metaType := range pb.Metadata {
		p.Types = append(p.Types, metaType)
	}

	byteXml, _ := xml.MarshalIndent(p, "", "    ")
	byteXml = append([]byte(xml.Header), byteXml...)
	//if err := ioutil.WriteFile("mypackage.xml", byteXml, 0644); err != nil {
	//ErrorAndExit(err.Error())
	//}
	return byteXml
}

// Returns the full ForceMetadataFiles container
func (pb *PackageBuilder) ForceMetadataFiles() ForceMetadataFiles {
	pb.Files["package.xml"] = pb.PackageXml()
	return pb.Files
}

// Add a file to the builder
func (pb *PackageBuilder) AddFile(fpath string) (fname string, err error) {
	fpath, err = filepath.Abs(fpath)
	if err != nil {
		return
	}
	_, err = os.Stat(fpath)
	if err != nil {
		return
	}

	isDestructiveChanges, err := regexp.MatchString("destructiveChanges(Pre|Post)?"+regexp.QuoteMeta(".")+"xml", fpath)
	if err != nil {
		return
	}

	metaName, fname := getMetaTypeFromPath(fpath)
	if !isDestructiveChanges && len(strings.Split(fname, ".")) == 1 {
		pb.AddMetaToPackage(metaName, fname)
	}

	// If it's a push, we want to actually add the files
	if pb.IsPush {
		if isDestructiveChanges {
			err = pb.addDestructiveChanges(fpath)
		} else {
			err = pb.addFileToWorkingDir(metaName, fpath)
		}
	}

	return
}

// Adds the file to a temp directory for deploy
func (pb *PackageBuilder) addFileToWorkingDir(metaName string, fpath string) (err error) {
	// Get relative dir from source
	srcDir := filepath.Dir(filepath.Dir(fpath))
	for _, mp := range metapaths {
		if metaName == mp.name && mp.hasFolder {
			srcDir = filepath.Dir(srcDir)
		}
	}
	frel, _ := filepath.Rel(srcDir, fpath)

	// Try to find meta file
	hasMeta := true
	fmeta := fpath + "-meta.xml"
	fmetarel := ""
	if _, err = os.Stat(fmeta); err != nil {
		if os.IsNotExist(err) {
			hasMeta = false
		} else {
			// Has error
			return
		}
	} else {
		// Should be present since we worked back to srcDir
		fmetarel, _ = filepath.Rel(srcDir, fmeta)
	}

	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	pb.Files[frel] = fdata
	if hasMeta {
		fdata, err = ioutil.ReadFile(fmeta)
		pb.Files[fmetarel] = fdata
		return
	}

	return
}

func (pb *PackageBuilder) addDestructiveChanges(fpath string) (err error) {
	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	frel, _ := filepath.Rel(filepath.Dir(fpath), fpath)
	pb.Files[frel] = fdata

	return
}

// Adds a metadata name to the pending package
func (pb *PackageBuilder) AddMetaToPackage(metaName string, name string) {
	mt := pb.Metadata[metaName]
	if mt.Name == "" {
		mt.Name = metaName
	}

	mt.Members = append(mt.Members, name)
	pb.Metadata[metaName] = mt
}

// Gets metadata type name and target name from a file path
func getMetaTypeFromPath(fpath string) (metaName string, name string) {
	fpath, err := filepath.Abs(fpath)
	if err != nil {
		ErrorAndExit("Cound not find " + fpath)
	}
	if _, err := os.Stat(fpath); err != nil {
		ErrorAndExit("Cound not open " + fpath)
	}

	// Get the metadata type and name for the file
	metaName, fileName := getMetaForPath(fpath)
	name = strings.TrimSuffix(fileName, filepath.Ext(fileName))
	return
}

// Gets partial path based on a meta type name
func getPathForMeta(metaname string) string {
	for _, mp := range metapaths {
		if strings.EqualFold(mp.name, metaname) {
			return mp.path
		}
	}

	// Unknown, so use metaname
	return metaname
}

// Gets meta type and name based on a path
func getMetaForPath(path string) (metaName string, objectName string) {
	parentDir := filepath.Dir(path)
	parentName := filepath.Base(parentDir)
	grandparentName := filepath.Base(filepath.Dir(parentDir))
	fileName := filepath.Base(path)

	for _, mp := range metapaths {
		if mp.hasFolder && grandparentName == mp.path {
			metaName = mp.name
			if mp.onlyFolder {
				objectName = parentName
			} else {
				objectName = parentName + "/" + fileName
			}
			return
		}
		if mp.path == parentName {
			metaName = mp.name
			objectName = fileName
			return
		}
	}

	// Unknown, so use path
	metaName = parentName
	objectName = fileName
	return
}
<commit_msg>Add Ability to Deploy Queues by Path<commit_after>package main

import (
	"encoding/xml"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// Structs for XML building
type Package struct {
	Xmlns   string     `xml:"xmlns,attr"`
	Types   []MetaType `xml:"types"`
	Version string     `xml:"version"`
}

type MetaType struct {
	Members []string `xml:"members"`
	Name    string   `xml:"name"`
}

func createPackage() Package {
	return Package{
		Version: strings.TrimPrefix(apiVersion, "v"),
		Xmlns:   "http://soap.sforce.com/2006/04/metadata",
	}
}

type metapath struct {
	path string
	name string
	hasFolder bool
	onlyFolder bool
}

var metapaths = []metapath{
	metapath{path: "applications", name: "CustomApplication"},
	metapath{path: "assignmentRules", name: "AssignmentRules"},
	metapath{path: "aura", name: "AuraDefinitionBundle", hasFolder: true, onlyFolder: true},
	metapath{path: "autoResponseRules", name: "AutoResponseRules"},
	metapath{path: "classes", name: "ApexClass"},
	metapath{path: "communities", name: "Community"},
	metapath{path: "components", name: "ApexComponent"},
	metapath{path: "connectedApps", name: "ConnectedApp"},
	metapath{path: "dashboards", name: "Dashboard", hasFolder: true},
	metapath{path: "documents", name: "Document", hasFolder: true},
	metapath{path: "email", name: "EmailTemplate", hasFolder: true},
	metapath{path: "flexipages", name: "FlexiPage"},
	metapath{path: "groups", name: "Group"},
	metapath{path: "homePageLayouts", name: "HomePageLayout"},
	metapath{path: "labels", name: "CustomLabels"},
	metapath{path: "layouts", name: "Layout"},
	metapath{path: "objects", name: "CustomObject"},
	metapath{path: "objectTranslations", name: "CustomObjectTranslation"},
	metapath{path: "pages", name: "ApexPage"},
	metapath{path: "permissionsets", name: "PermissionSet"},
	metapath{path: "profiles", name: "Profile"},
	metapath{path: "queues", name: "Queue"},
	metapath{path: "quickActions", name: "QuickAction"},
	metapath{path: "remoteSiteSettings", name: "RemoteSiteSetting"},
	metapath{path: "reports", name: "Report", hasFolder: true},
	metapath{path: "roles", name: "Role"},
	metapath{path: "settings", name: "Settings"},
	metapath{path: "staticresources", name: "StaticResource"},
	metapath{path: "tabs", name: "CustomTab"},
	metapath{path: "triggers", name: "ApexTrigger"},
	metapath{path: "workflows", name: "Workflow"},
}

type PackageBuilder struct {
	IsPush   bool
	Metadata map[string]MetaType
	Files    ForceMetadataFiles
}

func NewPushBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: true}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

func NewFetchBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: false}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

// Build and return package.xml
func (pb PackageBuilder) PackageXml() []byte {
	p := createPackage()

	for _, metaType := range pb.Metadata {
		p.Types = append(p.Types, metaType)
	}

	byteXml, _ := xml.MarshalIndent(p, "", "    ")
	byteXml = append([]byte(xml.Header), byteXml...)
	//if err := ioutil.WriteFile("mypackage.xml", byteXml, 0644); err != nil {
	//ErrorAndExit(err.Error())
	//}
	return byteXml
}

// Returns the full ForceMetadataFiles container
func (pb *PackageBuilder) ForceMetadataFiles() ForceMetadataFiles {
	pb.Files["package.xml"] = pb.PackageXml()
	return pb.Files
}

// Add a file to the builder
func (pb *PackageBuilder) AddFile(fpath string) (fname string, err error) {
	fpath, err = filepath.Abs(fpath)
	if err != nil {
		return
	}
	_, err = os.Stat(fpath)
	if err != nil {
		return
	}

	isDestructiveChanges, err := regexp.MatchString("destructiveChanges(Pre|Post)?"+regexp.QuoteMeta(".")+"xml", fpath)
	if err != nil {
		return
	}

	metaName, fname := getMetaTypeFromPath(fpath)
	if !isDestructiveChanges && len(strings.Split(fname, ".")) == 1 {
		pb.AddMetaToPackage(metaName, fname)
	}

	// If it's a push, we want to actually add the files
	if pb.IsPush {
		if isDestructiveChanges {
			err = pb.addDestructiveChanges(fpath)
		} else {
			err = pb.addFileToWorkingDir(metaName, fpath)
		}
	}

	return
}

// Adds the file to a temp directory for deploy
func (pb *PackageBuilder) addFileToWorkingDir(metaName string, fpath string) (err error) {
	// Get relative dir from source
	srcDir := filepath.Dir(filepath.Dir(fpath))
	for _, mp := range metapaths {
		if metaName == mp.name && mp.hasFolder {
			srcDir = filepath.Dir(srcDir)
		}
	}
	frel, _ := filepath.Rel(srcDir, fpath)

	// Try to find meta file
	hasMeta := true
	fmeta := fpath + "-meta.xml"
	fmetarel := ""
	if _, err = os.Stat(fmeta); err != nil {
		if os.IsNotExist(err) {
			hasMeta = false
		} else {
			// Has error
			return
		}
	} else {
		// Should be present since we worked back to srcDir
		fmetarel, _ = filepath.Rel(srcDir, fmeta)
	}

	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	pb.Files[frel] = fdata
	if hasMeta {
		fdata, err = ioutil.ReadFile(fmeta)
		pb.Files[fmetarel] = fdata
		return
	}

	return
}

func (pb *PackageBuilder) addDestructiveChanges(fpath string) (err error) {
	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	frel, _ := filepath.Rel(filepath.Dir(fpath), fpath)
	pb.Files[frel] = fdata

	return
}

// Adds a metadata name to the pending package
func (pb *PackageBuilder) AddMetaToPackage(metaName string, name string) {
	mt := pb.Metadata[metaName]
	if mt.Name == "" {
		mt.Name = metaName
	}

	mt.Members = append(mt.Members, name)
	pb.Metadata[metaName] = mt
}

// Gets metadata type name and target name from a file path
func getMetaTypeFromPath(fpath string) (metaName string, name string) {
	fpath, err := filepath.Abs(fpath)
	if err != nil {
		ErrorAndExit("Cound not find " + fpath)
	}
	if _, err := os.Stat(fpath); err != nil {
		ErrorAndExit("Cound not open " + fpath)
	}

	// Get the metadata type and name for the file
	metaName, fileName := getMetaForPath(fpath)
	name = strings.TrimSuffix(fileName, filepath.Ext(fileName))
	return
}

// Gets partial path based on a meta type name
func getPathForMeta(metaname string) string {
	for _, mp := range metapaths {
		if strings.EqualFold(mp.name, metaname) {
			return mp.path
		}
	}

	// Unknown, so use metaname
	return metaname
}

// Gets meta type and name based on a path
func getMetaForPath(path string) (metaName string, objectName string) {
	parentDir := filepath.Dir(path)
	parentName := filepath.Base(parentDir)
	grandparentName := filepath.Base(filepath.Dir(parentDir))
	fileName := filepath.Base(path)

	for _, mp := range metapaths {
		if mp.hasFolder && grandparentName == mp.path {
			metaName = mp.name
			if mp.onlyFolder {
				objectName = parentName
			} else {
				objectName = parentName + "/" + fileName
			}
			return
		}
		if mp.path == parentName {
			metaName = mp.name
			objectName = fileName
			return
		}
	}

	// Unknown, so use path
	metaName = parentName
	objectName = fileName
	return
}
<|endoftext|>


<commit_before>package packet

import (
	"errors"
	"fmt"
	"net"
)

// IPv4Header represents an IPv4 header
type IPv4Header struct {
	Version  int    // protocol version
	TOS      int    // type-of-service
	TotalLen int    // packet total length
	ID       int    // identification
	Flags    int    // flags
	FragOff  int    // fragment offset
	TTL      int    // time-to-live
	Protocol int    // next protocol
	Checksum int    // checksum
	Src      string // source address
	Dst      string // destination address
}

// IPv6Header represents an IPv6 header
type IPv6Header struct {
	Version      int    // protocol version
	TrafficClass int    // traffic class
	FlowLabel    int    // flow label
	PayloadLen   int    // payload length
	NextHeader   int    // next header
	HopLimit     int    // hop limit
	Src          string // source address
	Dst          string // destination address
}

type Datalink struct {
	SrcMAC    string
	DstMAC    string
	Vlan      int
	EtherType uint16
}

type Packet struct {
	L2 Datalink
	L3 interface{}
	L4 interface{}
}

const (
	IPv4HLen = 20
	IPv6HLen = 40

	EtherTypeIPv4      = 0x0800
	EtherTypeIPv6      = 0x86DD
	EtherTypeIEEE8021Q = 0x8100
)

var (
	errShortEthernetHeaderLength = errors.New("the ethernet header is too small")
	errShortIPv4HeaderLength     = errors.New("short ipv4 header length")
	errShortIPv6HeaderLength     = errors.New("short ipv6 header length")
	errShortEthernetLength       = errors.New("short ethernet header length")
	errUnknownEtherType          = errors.New("unknown ether type")
)

func decodeISO88023(b []byte) (Packet, error) {
	var (
		err    error
		packet Packet
	)

	if len(b) < 14 {
		return packet, errShortEthernetHeaderLength
	}

	packet.L2, err = decodeIEEE802(b)
	if err != nil {
		return packet, err
	}

	switch packet.L2.EtherType {
	case EtherTypeIPv4:
		b = b[14:]

		packet.L3, err = decodeIPv4Header(b)
		if err != nil {
			return packet, err
		}

		packet.L4, err = decodeTransportLayer(int(b[9]), b[IPv4HLen:])
		if err != nil {
			return packet, err
		}

		return packet, nil

	case EtherTypeIPv6:
		b = b[14:]

		packet.L3, err = decodeIPv6Header(b)
		if err != nil {
			return packet, err
		}

		packet.L4, err = decodeTransportLayer(int(b[6]), b[IPv6HLen:])
		if err != nil {
			return packet, err
		}

		return packet, nil

	case EtherTypeIEEE8021Q:
		vlan := int(b[14])<<8 | int(b[15])
		b[12], b[13] = b[16], b[17]
		b = append(b[:14], b[18:]...)
		packet, err = decodeISO88023(b)
		if err != nil {
			return packet, err
		}
		packet.L2.Vlan = vlan
		return packet, nil

	default:
		return packet, errUnknownEtherType
	}

	return packet, nil
}

func decodeIPv6Header(b []byte) (IPv6Header, error) {
	if len(b) < IPv6HLen {
		return IPv6Header{}, errShortIPv6HeaderLength
	}

	var (
		src net.IP = b[8:24]
		dst net.IP = b[24:40]
	)

	return IPv6Header{
		Version:      int(b[0]) >> 4,
		TrafficClass: int(b[0]&0x0f)<<4 | int(b[1])>>4,
		FlowLabel:    int(b[1]&0x0f)<<16 | int(b[2])<<8 | int(b[3]),
		PayloadLen:   int(uint16(b[4])<<8 | uint16(b[5])),
		NextHeader:   int(b[6]),
		HopLimit:     int(b[7]),
		Src:          src.String(),
		Dst:          dst.String(),
	}, nil
}

func decodeIPv4Header(b []byte) (IPv4Header, error) {
	if len(b) < IPv4HLen {
		return IPv4Header{}, errShortIPv4HeaderLength
	}

	return IPv4Header{
		Version:  int(b[0] & 0xf0 >> 4),
		TOS:      int(b[1]),
		TotalLen: int(b[2])<<8 | int(b[3]),
		ID:       int(b[4])<<8 | int(b[5]),
		Flags:    int(b[6] & 0x07),
		TTL:      int(b[8]),
		Protocol: int(b[9]),
		Checksum: int(b[10])<<8 | int(b[11]),
		Src:      fmt.Sprintf("%d.%d.%d.%d", b[12], b[13], b[14], b[15]),
		Dst:      fmt.Sprintf("%d.%d.%d.%d", b[16], b[17], b[18], b[19]),
	}, nil
}

func decodeIEEE802(b []byte) (Datalink, error) {
	var d Datalink

	if len(b) < 14 {
		return d, errShortEthernetLength
	}

	d.EtherType = uint16(b[13]) | uint16(b[12])<<8

	if d.EtherType != EtherTypeIEEE8021Q {
		d.SrcMAC = fmt.Sprintf("%0.2x:%0.2x:%0.2x:%0.2x:%0.2x:%0.2x", b[0], b[1], b[2], b[3], b[4], b[5])
		d.DstMAC = fmt.Sprintf("%0.2x:%0.2x:%0.2x:%0.2x:%0.2x:%0.2x", b[6], b[7], b[8], b[9], b[10], b[11])
	}

	return d, nil
}

func Decoder(b []byte) (Packet, error) {
	return decodeISO88023(b)
}
<commit_msg>add receiver, optimize<commit_after>package packet

import (
	"errors"
	"net"
)

// IPv4Header represents an IPv4 header
type IPv4Header struct {
	Version  int    // protocol version
	TOS      int    // type-of-service
	TotalLen int    // packet total length
	ID       int    // identification
	Flags    int    // flags
	FragOff  int    // fragment offset
	TTL      int    // time-to-live
	Protocol int    // next protocol
	Checksum int    // checksum
	Src      string // source address
	Dst      string // destination address
}

// IPv6Header represents an IPv6 header
type IPv6Header struct {
	Version      int    // protocol version
	TrafficClass int    // traffic class
	FlowLabel    int    // flow label
	PayloadLen   int    // payload length
	NextHeader   int    // next header
	HopLimit     int    // hop limit
	Src          string // source address
	Dst          string // destination address
}

const (
	IPv4HLen = 20
	IPv6HLen = 40

	EtherTypeIPv4      = 0x0800
	EtherTypeIPv6      = 0x86DD
	EtherTypeIEEE8021Q = 0x8100
)

var (
	errShortIPv4HeaderLength = errors.New("short ipv4 header length")
	errShortIPv6HeaderLength = errors.New("short ipv6 header length")
	errShortEthernetLength   = errors.New("short ethernet header length")
	errUnknownTransportLayer = errors.New("unknown transport layer")
	errUnknownL3Protocol     = errors.New("unknown network layer protocol")
)

func (p *Packet) decodeNextLayer() error {

	var (
		proto int
		len   int
	)

	switch p.L3.(type) {
	case IPv4Header:
		proto = p.L3.(IPv4Header).Protocol
	case IPv6Header:
		proto = p.L3.(IPv6Header).NextHeader
	default:
		return errUnknownL3Protocol
	}

	switch proto {
	case IANAProtoICMP:
		icmp, err := decodeICMP(p.data)
		if err != nil {
			return err
		}

		p.L4 = icmp
		len = 4
	case IANAProtoTCP:
		tcp, err := decoderTCP(p.data)
		if err != nil {
			return err
		}

		p.L4 = tcp
		len = 20
	case IANAProtoUDP:
		udp, err := decoderUDP(p.data)
		if err != nil {
			return err
		}

		p.L4 = udp
		len = 8
	default:
		return errUnknownTransportLayer
	}

	p.data = p.data[len:]

	return nil
}

func (p *Packet) decodeIPv6Header() error {
	if len(p.data) < IPv6HLen {
		return errShortIPv6HeaderLength
	}

	var (
		src net.IP = p.data[8:24]
		dst net.IP = p.data[24:40]
	)

	p.L3 = IPv6Header{
		Version:      int(p.data[0]) >> 4,
		TrafficClass: int(p.data[0]&0x0f)<<4 | int(p.data[1])>>4,
		FlowLabel:    int(p.data[1]&0x0f)<<16 | int(p.data[2])<<8 | int(p.data[3]),
		PayloadLen:   int(uint16(p.data[4])<<8 | uint16(p.data[5])),
		NextHeader:   int(p.data[6]),
		HopLimit:     int(p.data[7]),
		Src:          src.String(),
		Dst:          dst.String(),
	}

	p.data = p.data[IPv6HLen:]

	return nil
}

func (p *Packet) decodeIPv4Header() error {
	if len(p.data) < IPv4HLen {
		return errShortIPv4HeaderLength
	}

	var (
		src net.IP = p.data[12:16]
		dst net.IP = p.data[16:20]
	)

	p.L3 = IPv4Header{
		Version:  int(p.data[0] & 0xf0 >> 4),
		TOS:      int(p.data[1]),
		TotalLen: int(p.data[2])<<8 | int(p.data[3]),
		ID:       int(p.data[4])<<8 | int(p.data[5]),
		Flags:    int(p.data[6] & 0x07),
		TTL:      int(p.data[8]),
		Protocol: int(p.data[9]),
		Checksum: int(p.data[10])<<8 | int(p.data[11]),
		Src:      src.String(),
		Dst:      dst.String(),
	}

	p.data = p.data[IPv4HLen:]

	return nil
}
<|endoftext|>


<commit_before>package wechatApi

import "testing"
import "os"
import "fmt"
import "github.com/duck-lab/wechat-api/accessToken"
import "github.com/stretchr/testify/assert"

func TestInvalid(t *testing.T) {
	api := New("aa", "bb")
	_, err := api.GetAccessToken()
	if err != nil {
		t.Log("Failed as expected.")
	} else {
		t.Error("Success unexpectlly.")
	}
}

func TestValid(t *testing.T) {
	if os.Getenv("WECHAT_APPID") == "" {
		t.Error("You should set the WECHAT_APPID & WECHAT_APPSECRET in your env")
	}
	api := New(os.Getenv("WECHAT_APPID"), os.Getenv("WECHAT_APPSECRET"))
	token, err := api.GetAccessToken()
	fmt.Println(token.ExpiresTime)
	if err != nil {
		t.Error("Fetch token failed with reason: ", err.Error())
	} else {
		t.Log("Fetch token success.")
	}
}

func TestCallCount(t *testing.T) {
	if os.Getenv("WECHAT_APPID") == "" {
		t.Error("You should set the WECHAT_APPID & WECHAT_APPSECRET in your env")
	}
	api := New(os.Getenv("WECHAT_APPID"), os.Getenv("WECHAT_APPSECRET"))
	assert.Equal(t, 0, api.getCall(accessToken.APIName))
	token, err := api.GetAccessToken()
	fmt.Println(token, err)
	assert.Equal(t, 1, api.getCall(accessToken.APIName))
	token, err = api.GetAccessToken()
	fmt.Println(token, err)
	assert.Equal(t, 1, api.getCall(accessToken.APIName))
}
<commit_msg>fix name of API fetch<commit_after>package wechatApi

import "testing"
import "os"
import "fmt"
import "github.com/duck-lab/wechat-api/accessToken"
import "github.com/stretchr/testify/assert"

func TestInvalid(t *testing.T) {
	api := New("aa", "bb")
	_, err := api.GetAccessToken()
	if err != nil {
		t.Log("Failed as expected.")
	} else {
		t.Error("Success unexpectlly.")
	}
}

func TestValid(t *testing.T) {
	if os.Getenv("WECHAT_APPID") == "" {
		t.Error("You should set the WECHAT_APPID & WECHAT_APPSECRET in your env")
	}
	api := New(os.Getenv("WECHAT_APPID"), os.Getenv("WECHAT_APPSECRET"))
	token, err := api.GetAccessToken()
	fmt.Println(token.ExpiresTime)
	if err != nil {
		t.Error("Fetch token failed with reason: ", err.Error())
	} else {
		t.Log("Fetch token success.")
	}
}

func TestCallCount(t *testing.T) {
	if os.Getenv("WECHAT_APPID") == "" {
		t.Error("You should set the WECHAT_APPID & WECHAT_APPSECRET in your env")
	}
	api := New(os.Getenv("WECHAT_APPID"), os.Getenv("WECHAT_APPSECRET"))
	assert.Equal(t, 2, api.getCall(accessToken.APINameFetch))
	token, err := api.GetAccessToken()
	fmt.Println(token, err)
	assert.Equal(t, 3, api.getCall(accessToken.APINameFetch))
	token, err = api.GetAccessToken()
	fmt.Println(token, err)
	assert.Equal(t, 3, api.getCall(accessToken.APINameFetch))
}
<|endoftext|>