<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"neugram.io/eval"
	"neugram.io/eval/shell"
	"neugram.io/lang/tipe"
	"neugram.io/parser"

	"github.com/peterh/liner"
)

var (
	origMode liner.ModeApplier

	lineNg        *liner.State // ng-mode line reader
	historyNgFile = ""
	historyNg     = make(chan string, 1)
	lineSh        *liner.State // shell-mode line reader
	historyShFile = ""
	historySh     = make(chan string, 1)

	prg *eval.Program
)

func cleanup() {
	lineSh.Close()
	lineNg.Close()
}

func exit(code int) {
	cleanup()
	os.Exit(code)
}

func exitf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "ng: "+format, args...)
	exit(1)
}

func mode() liner.ModeApplier {
	m, err := liner.TerminalMode()
	if err != nil {
		exitf("terminal mode: %v", err)
	}
	return m
}

func main() {
	shell.Init()

	// TODO
	// This is getting a bit absurd. It's time to write our own liner
	// package, one that supports the two modes we need and meshes well
	// with our own signal handling.
	ch := make(chan os.Signal, 1)
	winch1 := make(chan os.Signal, 1)
	winch2 := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGWINCH)
	go func() {
		for {
			sig := <-ch
			fmt.Printf("got signal: %s\n", sig)
			switch sig {
			case syscall.SIGWINCH:
				// TODO: don't drop this signal.
				// Instead, rewrite liner and make sure we
				// are always processing this.
				select {
				case winch1 <- syscall.SIGWINCH:
				default:
				}
				select {
				case winch2 <- syscall.SIGWINCH:
				default:
				}
			}
		}
	}()

	origMode = mode()
	lineNg = liner.NewLiner(os.Stdin, winch1)
	lineSh = liner.NewLiner(os.Stdin, winch2)
	loop()
}

func setWindowSize(env map[interface{}]interface{}) {
	// TODO windowsize
	// TODO
	// TODO
	// TODO
	/*
		rows, cols, err := job.WindowSize(os.Stderr.Fd())
		if err != nil {
			fmt.Printf("ng: could not get window size: %v\n", err)
		} else {
			// TODO: these are meant to be shell variables, not
			// environment variables. But then, how do programs
			// like `ls` read them?
			env["LINES"] = strconv.Itoa(rows)
			env["COLUMNS"] = strconv.Itoa(cols)
		}
	*/
}

func loop() {
	p := parser.New()
	prg = eval.New()

	// TODO this env setup could be done in neugram code
	env := prg.Cur.Lookup("env").Value.(map[interface{}]interface{})
	for _, s := range os.Environ() {
		i := strings.Index(s, "=")
		env[s[:i]] = s[i+1:]
	}
	setWindowSize(env)

	lineNg.SetCompleter(completer)
	if f, err := os.Open(historyNgFile); err == nil {
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyNgFile, historyNg)

	lineSh.SetCompleter(completerSh)
	if f, err := os.Open(historyShFile); err == nil {
		lineSh.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyShFile, historySh)

	state := parser.StateStmt
	for {
		var (
			prompt  string
			line    *liner.State
			history chan string
		)
		switch state {
		case parser.StateUnknown:
			prompt, line, history = "??> ", lineNg, historyNg
		case parser.StateStmt:
			prompt, line, history = "ng> ", lineNg, historyNg
		case parser.StateStmtPartial:
			prompt, line, history = "..> ", lineNg, historyNg
		case parser.StateCmd:
			prompt, line, history = "$ ", lineSh, historySh
		case parser.StateCmdPartial:
			prompt, line, history = "..$ ", lineSh, historySh
		default:
			exitf("unkown parser state: %v", state)
		}
		data, err := line.Prompt(prompt)
		if err != nil {
			if err == io.EOF {
				exit(0)
			}
			exitf("error reading input: %v", err)
		}
		if data == "" {
			continue
		}
		line.AppendHistory(data)
		history <- data
		res := p.ParseLine([]byte(data))

		for _, s := range res.Stmts {
			v, t, err := prg.Eval(s)
			if err != nil {
				fmt.Printf("eval error: %v\n", err)
				continue
			}
			switch len(v) {
			case 0:
			case 1:
				printValue(t, v[0])
				fmt.Print("\n")
			default:
				fmt.Println(v)
			}
		}
		for _, err := range res.Errs {
			fmt.Println(err.Error())
		}
		//editMode := mode()
		//origMode.ApplyMode()
		if len(res.Cmds) > 0 {
			shell.Env = prg.Environ()
		}
		for _, cmd := range res.Cmds {
			j := &shell.Job{
				Cmd:    cmd,
				Params: prg,
				Stdin:  os.Stdin,
				Stdout: os.Stdout,
				Stderr: os.Stderr,
			}
			if err := j.Start(); err != nil {
				fmt.Println(err)
				continue
			}
			done, err := j.Wait()
			if err != nil {
				fmt.Println(err)
				continue
			}
			if !done {
				break // TODO not right, instead we should just have one cmd, not Cmds here.
			}
		}
		//editMode.ApplyMode()
		state = res.State
	}
}

func printValue(t tipe.Type, v interface{}) {
	// This is, effectively, a primitive type-aware printf implementation
	// that understands the neugram evaluator data layout. A far better
	// version of this would be an "ngfmt" package, that implemented the
	// printing command in neugram, using a "ngreflect" package. But it
	// will be a while until I build a reflect package, so this will have
	// to do.
	//
	// Still: avoid putting too much machinary in this. At some point soon
	// it's not worth the effort.
	switch t := tipe.Underlying(t).(type) {
	case *tipe.Struct:
		fmt.Print("{")
		for i, name := range t.FieldNames {
			fmt.Printf("%s: ", name)
			printValue(t.Fields[i], v.(*eval.StructVal).Fields[i].Value)
			if i < len(t.FieldNames)-1 {
				fmt.Print(", ")
			}
		}
		fmt.Print("}")
	default:
		fmt.Print(v)
	}
}

func init() {
	if home := os.Getenv("HOME"); home != "" {
		historyNgFile = filepath.Join(home, ".ng_history")
		historyShFile = filepath.Join(home, ".ng_sh_history")
	}
}

func historyWriter(dst string, src <-chan string) {
	var batch []string
	ticker := time.Tick(250 * time.Millisecond)
	for {
		select {
		case line := <-src:
			batch = append(batch, line)
		case <-ticker:
			if len(batch) > 0 && dst != "" {
				f, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err == nil {
					for _, line := range batch {
						fmt.Fprintf(f, "%s\n", line)
					}
					f.Close()
				}
			}
			batch = nil
		}
	}
}
<commit_msg>ng: use normal liner package<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"neugram.io/eval"
	"neugram.io/eval/shell"
	"neugram.io/lang/tipe"
	"neugram.io/parser"

	"github.com/peterh/liner"
)

var (
	origMode liner.ModeApplier

	lineNg        *liner.State // ng-mode line reader
	historyNgFile = ""
	historyNg     = make(chan string, 1)
	lineSh        *liner.State // shell-mode line reader
	historyShFile = ""
	historySh     = make(chan string, 1)

	prg *eval.Program
)

func cleanup() {
	lineSh.Close()
	lineNg.Close()
}

func exit(code int) {
	cleanup()
	os.Exit(code)
}

func exitf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "ng: "+format, args...)
	exit(1)
}

func mode() liner.ModeApplier {
	m, err := liner.TerminalMode()
	if err != nil {
		exitf("terminal mode: %v", err)
	}
	return m
}

func main() {
	shell.Init()

	origMode = mode()
	lineNg = liner.NewLiner()
	lineSh = liner.NewLiner()
	loop()
}

func setWindowSize(env map[interface{}]interface{}) {
	// TODO windowsize
	// TODO
	// TODO
	// TODO
	/*
		rows, cols, err := job.WindowSize(os.Stderr.Fd())
		if err != nil {
			fmt.Printf("ng: could not get window size: %v\n", err)
		} else {
			// TODO: these are meant to be shell variables, not
			// environment variables. But then, how do programs
			// like `ls` read them?
			env["LINES"] = strconv.Itoa(rows)
			env["COLUMNS"] = strconv.Itoa(cols)
		}
	*/
}

func loop() {
	p := parser.New()
	prg = eval.New()

	// TODO this env setup could be done in neugram code
	env := prg.Cur.Lookup("env").Value.(map[interface{}]interface{})
	for _, s := range os.Environ() {
		i := strings.Index(s, "=")
		env[s[:i]] = s[i+1:]
	}
	setWindowSize(env)

	lineNg.SetCompleter(completer)
	if f, err := os.Open(historyNgFile); err == nil {
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyNgFile, historyNg)

	lineSh.SetCompleter(completerSh)
	if f, err := os.Open(historyShFile); err == nil {
		lineSh.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyShFile, historySh)

	state := parser.StateStmt
	for {
		var (
			prompt  string
			line    *liner.State
			history chan string
		)
		switch state {
		case parser.StateUnknown:
			prompt, line, history = "??> ", lineNg, historyNg
		case parser.StateStmt:
			prompt, line, history = "ng> ", lineNg, historyNg
		case parser.StateStmtPartial:
			prompt, line, history = "..> ", lineNg, historyNg
		case parser.StateCmd:
			prompt, line, history = "$ ", lineSh, historySh
		case parser.StateCmdPartial:
			prompt, line, history = "..$ ", lineSh, historySh
		default:
			exitf("unkown parser state: %v", state)
		}
		data, err := line.Prompt(prompt)
		if err != nil {
			if err == io.EOF {
				exit(0)
			}
			exitf("error reading input: %v", err)
		}
		if data == "" {
			continue
		}
		line.AppendHistory(data)
		history <- data
		res := p.ParseLine([]byte(data))

		for _, s := range res.Stmts {
			v, t, err := prg.Eval(s)
			if err != nil {
				fmt.Printf("eval error: %v\n", err)
				continue
			}
			switch len(v) {
			case 0:
			case 1:
				printValue(t, v[0])
				fmt.Print("\n")
			default:
				fmt.Println(v)
			}
		}
		for _, err := range res.Errs {
			fmt.Println(err.Error())
		}
		//editMode := mode()
		//origMode.ApplyMode()
		if len(res.Cmds) > 0 {
			shell.Env = prg.Environ()
		}
		for _, cmd := range res.Cmds {
			j := &shell.Job{
				Cmd:    cmd,
				Params: prg,
				Stdin:  os.Stdin,
				Stdout: os.Stdout,
				Stderr: os.Stderr,
			}
			if err := j.Start(); err != nil {
				fmt.Println(err)
				continue
			}
			done, err := j.Wait()
			if err != nil {
				fmt.Println(err)
				continue
			}
			if !done {
				break // TODO not right, instead we should just have one cmd, not Cmds here.
			}
		}
		//editMode.ApplyMode()
		state = res.State
	}
}

func printValue(t tipe.Type, v interface{}) {
	// This is, effectively, a primitive type-aware printf implementation
	// that understands the neugram evaluator data layout. A far better
	// version of this would be an "ngfmt" package, that implemented the
	// printing command in neugram, using a "ngreflect" package. But it
	// will be a while until I build a reflect package, so this will have
	// to do.
	//
	// Still: avoid putting too much machinary in this. At some point soon
	// it's not worth the effort.
	switch t := tipe.Underlying(t).(type) {
	case *tipe.Struct:
		fmt.Print("{")
		for i, name := range t.FieldNames {
			fmt.Printf("%s: ", name)
			printValue(t.Fields[i], v.(*eval.StructVal).Fields[i].Value)
			if i < len(t.FieldNames)-1 {
				fmt.Print(", ")
			}
		}
		fmt.Print("}")
	default:
		fmt.Print(v)
	}
}

func init() {
	if home := os.Getenv("HOME"); home != "" {
		historyNgFile = filepath.Join(home, ".ng_history")
		historyShFile = filepath.Join(home, ".ng_sh_history")
	}
}

func historyWriter(dst string, src <-chan string) {
	var batch []string
	ticker := time.Tick(250 * time.Millisecond)
	for {
		select {
		case line := <-src:
			batch = append(batch, line)
		case <-ticker:
			if len(batch) > 0 && dst != "" {
				f, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err == nil {
					for _, line := range batch {
						fmt.Fprintf(f, "%s\n", line)
					}
					f.Close()
				}
			}
			batch = nil
		}
	}
}
<|endoftext|>


<commit_before>package host

import (
	"sync/atomic"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/modules"
)

// blockingPortForward is a dependency set that causes the host port forward
// call at startup to block for 10 seconds, simulating the amount of blocking
// that can occur in production.
//
// blockingPortForward will also cause managedClearPort to always return an
// error.
type blockingPortForward struct {
	productionDependencies
}

// disrupt will cause the port forward call to block for 10 seconds, but still
// complete normally. disrupt will also cause managedClearPort to return an
// error.
func (blockingPortForward) disrupt(s string) bool {
	// Return an error when clearing the port.
	if s == "managedClearPort return error" {
		return true
	}

	// Block during port forwarding.
	if s == "managedForwardPort" {
		time.Sleep(time.Second * 3)
	}
	return false
}

// TestPortFowardBlocking checks that the host does not accidentally call a
// write on a closed logger due to a long-running port forward call.
func TestPortForwardBlocking(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	t.Parallel()
	ht, err := newMockHostTester(blockingPortForward{}, "TestPortForwardBlocking")
	if err != nil {
		t.Fatal(err)
	}

	// The close operation would previously fail here because of improper
	// thread control regarding upnp and shutdown.
	err = ht.Close()
	if err != nil {
		t.Fatal(err)
	}

	// The trailing sleep is needed to catch the previously existing error
	// where the host was not shutting down correctly. Currently, the extra
	// sleep does nothing, but in the regression a logging panic would occur.
	time.Sleep(time.Second * 4)
}

// TestHostWorkingStatus checks that the host properly updates its working
// state
func TestHostWorkingStatus(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	t.Parallel()
	ht, err := newHostTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}

	if ht.host.WorkingStatus() != modules.HostWorkingStatusChecking {
		t.Fatal("expected working state to initially be modules.HostWorkingStatusChecking")
	}

	// Simulate some setting calls, and see if the host picks up on it.
	atomic.AddUint64(&ht.host.atomicSettingsCalls, workingStatusThreshold+1)
	time.Sleep(workingStatusFirstCheck + time.Second)
	if ht.host.WorkingStatus() != modules.HostWorkingStatusWorking {
		t.Fatal("expected host working status to be modules.HostWorkingStatusWorking after incrementing status calls")
	}

	// No more settings calls, host should believe it is not working now.
	time.Sleep(workingStatusFrequency + time.Second)
	if ht.host.WorkingStatus() != modules.HostWorkingStatusNotWorking {
		t.Fatal("expected host working status to be modules.HostWorkingStatusNotWorking after waiting workingStatusFrequency with no settings calls")
	}

	// Simulate some setting calls, and see if the host picks up on it.
	atomic.AddUint64(&ht.host.atomicSettingsCalls, workingStatusThreshold+1)
	time.Sleep(workingStatusFirstCheck + time.Second)
	if ht.host.WorkingStatus() != modules.HostWorkingStatusNotWorking {
		t.Fatal("expected host working status to be modules.HostWorkingStatusWorking after incrementing status calls")
	}
	time.Sleep(workingStatusFrequency - workingStatusFirstCheck)
	if ht.host.WorkingStatus() != modules.HostWorkingStatusWorking {
		t.Fatal("expected host working status to be modules.HostWorkingStatusWorking after incrementing status calls")
	}
}

// TestHostConnectabilityStatus checks that the host properly updates its connectable
// state
func TestHostConnectabilityStatus(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	t.Parallel()
	ht, err := newHostTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}

	if ht.host.ConnectabilityStatus() != modules.HostConnectabilityStatusChecking {
		t.Fatal("expected connectability state to initially be ConnectablityStateChecking")
	}
	time.Sleep(connectabilityCheckFirstWait + time.Second)
	if ht.host.ConnectabilityStatus() != modules.HostConnectabilityStatusConnectable {
		t.Fatal("expected connectability state to be modules.HostConnectabilityStatusConnectable")
	}
}
<commit_msg>fix NDFs in host status tests, close host testers<commit_after>package host

import (
	"sync/atomic"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/modules"
)

// blockingPortForward is a dependency set that causes the host port forward
// call at startup to block for 10 seconds, simulating the amount of blocking
// that can occur in production.
//
// blockingPortForward will also cause managedClearPort to always return an
// error.
type blockingPortForward struct {
	productionDependencies
}

// disrupt will cause the port forward call to block for 10 seconds, but still
// complete normally. disrupt will also cause managedClearPort to return an
// error.
func (blockingPortForward) disrupt(s string) bool {
	// Return an error when clearing the port.
	if s == "managedClearPort return error" {
		return true
	}

	// Block during port forwarding.
	if s == "managedForwardPort" {
		time.Sleep(time.Second * 3)
	}
	return false
}

// TestPortFowardBlocking checks that the host does not accidentally call a
// write on a closed logger due to a long-running port forward call.
func TestPortForwardBlocking(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	t.Parallel()
	ht, err := newMockHostTester(blockingPortForward{}, "TestPortForwardBlocking")
	if err != nil {
		t.Fatal(err)
	}

	// The close operation would previously fail here because of improper
	// thread control regarding upnp and shutdown.
	err = ht.Close()
	if err != nil {
		t.Fatal(err)
	}

	// The trailing sleep is needed to catch the previously existing error
	// where the host was not shutting down correctly. Currently, the extra
	// sleep does nothing, but in the regression a logging panic would occur.
	time.Sleep(time.Second * 4)
}

// TestHostWorkingStatus checks that the host properly updates its working
// state
func TestHostWorkingStatus(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	t.Parallel()

	ht, err := newHostTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer ht.Close()

	if ht.host.WorkingStatus() != modules.HostWorkingStatusChecking {
		t.Fatal("expected working state to initially be modules.HostWorkingStatusChecking")
	}

	for i := 0; i < 5; i++ {
		// Simulate some setting calls, and see if the host picks up on it.
		atomic.AddUint64(&ht.host.atomicSettingsCalls, workingStatusThreshold+1)

		success := false
		for start := time.Now(); time.Since(start) < 30*time.Second; time.Sleep(time.Millisecond * 10) {
			if ht.host.WorkingStatus() == modules.HostWorkingStatusWorking {
				success = true
				break
			}
		}
		if !success {
			t.Fatal("expected working state to flip to HostWorkingStatusWorking after incrementing settings calls")
		}

		// make no settins calls, host should flip back to NotWorking
		success = false
		for start := time.Now(); time.Since(start) < 30*time.Second; time.Sleep(time.Millisecond * 10) {
			if ht.host.WorkingStatus() == modules.HostWorkingStatusNotWorking {
				success = true
				break
			}
		}
		if !success {
			t.Fatal("expected working state to flip to HostStatusNotWorking if no settings calls occur")
		}
	}
}

// TestHostConnectabilityStatus checks that the host properly updates its connectable
// state
func TestHostConnectabilityStatus(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	t.Parallel()
	ht, err := newHostTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer ht.Close()

	if ht.host.ConnectabilityStatus() != modules.HostConnectabilityStatusChecking {
		t.Fatal("expected connectability state to initially be ConnectablityStateChecking")
	}

	success := false
	for start := time.Now(); time.Since(start) < 30*time.Second; time.Sleep(time.Millisecond * 10) {
		if ht.host.ConnectabilityStatus() == modules.HostConnectabilityStatusConnectable {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("expected connectability state to flip to HostConnectabilityStatusConnectable")
	}
}
<|endoftext|>


<commit_before>package main

import "os/exec"
import "strings"
import "runtime"
import "github.com/fatih/color"
import "os"

func main() {

  Green := color.New(color.FgGreen)
  BoldGreen := Green.Add(color.Bold)
  Yellow := color.New(color.FgYellow)
  BoldYellow := Yellow.Add(color.Bold)
  Red := color.New(color.FgRed)
  BoldRed := Red.Add(color.Bold)
  White := color.New(color.FgWhite)
  BoldWhite := White.Add(color.Bold)

  color.Red(" ██░ ██ ▓█████  ██▀███   ▄████▄   █    ██  ██▓    ▓█████   ██████ ")
  color.Red("▓██░ ██▒▓█   ▀ ▓██ ▒ ██▒▒██▀ ▀█   ██  ▓██▒▓██▒    ▓█   ▀ ▒██    ▒ ")
  color.Red("▒██▀▀██░▒███   ▓██ ░▄█ ▒▒▓█    ▄ ▓██  ▒██░▒██░    ▒███   ░ ▓██▄   ")
  color.Red("░▓█ ░██ ▒▓█  ▄ ▒██▀▀█▄  ▒▓▓▄ ▄██▒▓▓█  ░██░▒██░    ▒▓█  ▄   ▒   ██▒")
  color.Red("░▓█▒░██▓░▒████▒░██▓ ▒██▒▒ ▓███▀ ░▒▒█████▓ ░██████▒░▒████▒▒██████▒▒")
  color.Red(" ▒ ░░▒░▒░░ ▒░ ░░ ▒▓ ░▒▓░░ ░▒ ▒  ░░▒▓▒ ▒ ▒ ░ ▒░▓  ░░░ ▒░ ░▒ ▒▓▒ ▒ ░")
  color.Red(" ▒ ░▒░ ░ ░ ░  ░  ░▒ ░ ▒░  ░  ▒   ░░▒░ ░ ░ ░ ░ ▒  ░ ░ ░  ░░ ░▒  ░ ░")
  color.Red(" ░  ░░ ░   ░     ░░   ░ ░         ░░░ ░ ░   ░ ░      ░   ░  ░  ░  ")
  color.Red(" ░  ░  ░   ░  ░   ░     ░ ░         ░         ░  ░   ░  ░      ░  ")
  color.Red("                        ░                                         ")

  color.Green("\n+ -- --=[      HERCULES FRAMEWORK           ]")
  BoldGreen.Println("+ -- --=[            Ege Balcı              ]")

  Priv := CheckSUDO()

  if Priv == false {
  	BoldRed.Println("[!] ERROR : Setup needs root privileges")
  	os.Exit(1)
  }

  BoldWhite.Println("\n\n[*] STARTING HERCULES SETUP \n")

  BoldYellow.Println("[*] Detecting OS...")

  if runtime.GOOS == "linux" {
    OsVersion, _ := exec.Command("sh", "-c", "uname -a").Output()
    BoldYellow.Println("[*] OS Detected : " + string(OsVersion))
    BoldYellow.Println("[*] Installing golang...")
    Go := exec.Command("sh", "-c", "sudo apt-get install golang")
    Go.Stdout = os.Stdout
    Go.Stdin = os.Stdin
    Go.Run()
    BoldYellow.Println("[*] Installing upx...")
    UPX := exec.Command("sh", "-c", "sudo apt-get install upx")
    UPX.Stdout = os.Stdout
    UPX.Stdin = os.Stdin
    UPX.Run()
    BoldYellow.Println("[*] Installing openssl...")
    OSSL := exec.Command("sh", "-c", "sudo apt-get install openssl")
    OSSL.Stdout = os.Stdout
    OSSL.Stdin = os.Stdin
    OSSL.Run()
    BoldYellow.Println("[*] Installing git...")
    Git := exec.Command("sh", "-c", "sudo apt-get install git")
    Git.Stdout = os.Stdout
    Git.Stdin = os.Stdin
    Git.Run()

    BoldYellow.Println("[*] Cloning EGESPLOIT Library...")
    exec.Command("sh", "-c", "sudo git clone https://github.com/EgeBalci/EGESPLOIT.git").Run()
    exec.Command("sh", "-c", "sudo cp EGESPLOIT /usr/lib/go-1.6/src/").Run()
    exec.Command("sh", "-c", "sudo mv EGESPLOIT /usr/lib/go/src/").Run()

    Stat, Err := CheckValid()

    if Stat == false {
      BoldYellow.Println("\n")
      BoldRed.Println(Err)
    }else{
      BoldGreen.Println("\n\n[+] Setup completed successfully")
    }

  }else if runtime.GOOS != "linux" {
    BoldRed.Println("[!] ERROR : HERCULES+ only supports linux distributions")
  }

}

func CheckValid()  (bool, string){
  OutESP, _ := exec.Command("sh", "-c", "cd /usr/lib/go/src/ && ls").Output()
  if (!strings.Contains(string(OutESP), "EGESPLOIT")) {
    return false, "[!] ERROR : EGESPLOIT library is not installed"
  }

  OutESP2, _ := exec.Command("sh", "-c", "cd /usr/lib/go-1.6/src/ && ls").Output()
  if (!strings.Contains(string(OutESP2), "EGESPLOIT")) {
    return false, "[!] ERROR : EGESPLOIT library is not installed"
  }

  OutUPX, _ := exec.Command("sh", "-c", "upx").Output()
  if (!strings.Contains(string(OutUPX), "Copyright")) {
    return false, "[!] ERROR : upx is not installed"
  }

  OutGO, _ := exec.Command("sh", "-c", "go version").Output()
  if (!strings.Contains(string(OutGO), "version")) {
    return false, "[!] ERROR : golang is not installed"
  }

  return true, ""

}

func CheckSUDO() (bool){
	User, _ := exec.Command("sh", "-c", "whoami").Output()
	if string(User) == "root" {
		return true
	}else {
		return false
	}

}
<commit_msg>Update Setup.go<commit_after>package main

import "os/exec"
import "strings"
import "runtime"
import "github.com/fatih/color"
import "os"

func main() {

  Green := color.New(color.FgGreen)
  BoldGreen := Green.Add(color.Bold)
  Yellow := color.New(color.FgYellow)
  BoldYellow := Yellow.Add(color.Bold)
  Red := color.New(color.FgRed)
  BoldRed := Red.Add(color.Bold)
  White := color.New(color.FgWhite)
  BoldWhite := White.Add(color.Bold)

  color.Red(" ██░ ██ ▓█████  ██▀███   ▄████▄   █    ██  ██▓    ▓█████   ██████ ")
  color.Red("▓██░ ██▒▓█   ▀ ▓██ ▒ ██▒▒██▀ ▀█   ██  ▓██▒▓██▒    ▓█   ▀ ▒██    ▒ ")
  color.Red("▒██▀▀██░▒███   ▓██ ░▄█ ▒▒▓█    ▄ ▓██  ▒██░▒██░    ▒███   ░ ▓██▄   ")
  color.Red("░▓█ ░██ ▒▓█  ▄ ▒██▀▀█▄  ▒▓▓▄ ▄██▒▓▓█  ░██░▒██░    ▒▓█  ▄   ▒   ██▒")
  color.Red("░▓█▒░██▓░▒████▒░██▓ ▒██▒▒ ▓███▀ ░▒▒█████▓ ░██████▒░▒████▒▒██████▒▒")
  color.Red(" ▒ ░░▒░▒░░ ▒░ ░░ ▒▓ ░▒▓░░ ░▒ ▒  ░░▒▓▒ ▒ ▒ ░ ▒░▓  ░░░ ▒░ ░▒ ▒▓▒ ▒ ░")
  color.Red(" ▒ ░▒░ ░ ░ ░  ░  ░▒ ░ ▒░  ░  ▒   ░░▒░ ░ ░ ░ ░ ▒  ░ ░ ░  ░░ ░▒  ░ ░")
  color.Red(" ░  ░░ ░   ░     ░░   ░ ░         ░░░ ░ ░   ░ ░      ░   ░  ░  ░  ")
  color.Red(" ░  ░  ░   ░  ░   ░     ░ ░         ░         ░  ░   ░  ░      ░  ")
  color.Red("                        ░                                         ")

  color.Green("\n+ -- --=[      HERCULES FRAMEWORK           ]")
  BoldGreen.Println("+ -- --=[            Ege Balcı              ]")

  Priv := CheckSUDO()

  if Priv == false {
  	BoldRed.Println("[!] ERROR : Setup needs root privileges")
  	os.Exit(1)
  }

  BoldWhite.Println("\n\n[*] STARTING HERCULES SETUP \n")

  BoldYellow.Println("[*] Detecting OS...")

  if runtime.GOOS == "linux" {
    OsVersion, _ := exec.Command("sh", "-c", "uname -a").Output()
    BoldYellow.Println("[*] OS Detected : " + string(OsVersion))
    BoldYellow.Println("[*] Installing golang...")
    Go := exec.Command("sh", "-c", "sudo apt-get install golang")
    Go.Stdout = os.Stdout
    Go.Stdin = os.Stdin
    Go.Run()
    BoldYellow.Println("[*] Installing upx...")
    UPX := exec.Command("sh", "-c", "sudo apt-get install upx")
    UPX.Stdout = os.Stdout
    UPX.Stdin = os.Stdin
    UPX.Run()
    BoldYellow.Println("[*] Installing openssl...")
    OSSL := exec.Command("sh", "-c", "sudo apt-get install openssl")
    OSSL.Stdout = os.Stdout
    OSSL.Stdin = os.Stdin
    OSSL.Run()
    BoldYellow.Println("[*] Installing git...")
    Git := exec.Command("sh", "-c", "sudo apt-get install git")
    Git.Stdout = os.Stdout
    Git.Stdin = os.Stdin
    Git.Run()

    BoldYellow.Println("[*] Cloning EGESPLOIT Library...")
    exec.Command("sh", "-c", "sudo git clone https://github.com/EgeBalci/EGESPLOIT.git").Run()
    exec.Command("sh", "-c", "sudo cp EGESPLOIT /usr/lib/go-1.6/src/").Run()
    exec.Command("sh", "-c", "sudo mv EGESPLOIT /usr/lib/go/src/").Run()

    BoldYellow.Println("[*] Moving binaries...")
    exec.Command("sh", "-c", "sudo chmod 777 *").Run()
    exec.Command("sh", "-c", "sudo mv HERCULES /bin/").Run()
    exec.Command("sh", "-c", "sudo mv HERCULES_x64 /bin/").Run()
    exec.Command("sh", "-c", "sudo mv Update /bin/").Run()

    Stat, Err := CheckValid()

    if Stat == false {
      BoldYellow.Println("\n")
      BoldRed.Println(Err)
    }else{
      BoldGreen.Println("\n\n[+] Setup completed successfully")
    }

  }else if runtime.GOOS != "linux" {
    BoldRed.Println("[!] ERROR : HERCULES+ only supports linux distributions")
  }

}

func CheckValid()  (bool, string){
  OutESP, _ := exec.Command("sh", "-c", "cd /usr/lib/go/src/ && ls").Output()
  if (!strings.Contains(string(OutESP), "EGESPLOIT")) {
    return false, "[!] ERROR : EGESPLOIT library is not installed"
  }

  OutESP2, _ := exec.Command("sh", "-c", "cd /usr/lib/go-1.6/src/ && ls").Output()
  if (!strings.Contains(string(OutESP2), "EGESPLOIT")) {
    return false, "[!] ERROR : EGESPLOIT library is not installed"
  }

  OutUPX, _ := exec.Command("sh", "-c", "upx").Output()
  if (!strings.Contains(string(OutUPX), "Copyright")) {
    return false, "[!] ERROR : upx is not installed"
  }

  OutGO, _ := exec.Command("sh", "-c", "go version").Output()
  if (!strings.Contains(string(OutGO), "version")) {
    return false, "[!] ERROR : golang is not installed"
  }

  OutBin, _ := exec.Command("sh", "-c", "cd /bin/ && ls").Output()
  if (!strings.Contains(string(OutBin), "HERCULES")) || (!strings.Contains(string(OutBin), "HERCULES_x64")) || (!strings.Contains(string(OutBin), "Update")) {
    return false, "[!] ERROR : Unable to move HERCULES binaries "
  }

  return true, ""

}

func CheckSUDO() (bool){
	User, _ := exec.Command("sh", "-c", "whoami").Output()
	if strings.Contains(string(User), "root") {
		return true
	}else {
		return false
	}

}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/neurosnap/sentences"
)

type Tokenizer struct {
	sentences       []*sentences.Sentence
	currentSentence []string
	sentenceId      int
	wordId          int
	finished        bool
}

func NewTokenizer() *Tokenizer {
	return &Tokenizer{}
}

func (t *Tokenizer) Process(fileName string) error {
	t.sentenceId = 0
	t.wordId = 0
	t.finished = false
	file, err := os.Open(fileName)
	if err != nil {
		return err
	}
	return t.process(file)
}

func (t *Tokenizer) process(reader io.Reader) error {
	// TODO: Use proper training data
	training := sentences.NewStorage()
	tokenizer := sentences.NewSentenceTokenizer(training)
	text, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}
	t.sentences = tokenizer.Tokenize(string(text))
	if len(t.sentences) > 0 {
		t.currentSentence = t.processSentence(t.sentences[t.sentenceId].String())
	}
	return nil
}

func (t *Tokenizer) processSentence(text string) []string {
	return strings.Fields(text)
}

func (t *Tokenizer) getNextWord() (string, bool) {
	if t.finished {
		return "", false
	}
	t.wordId++
	if len(t.currentSentence) == t.wordId {
		t.sentenceId++
		if len(t.sentences) == t.sentenceId {
			return "", false
		}
		sentence := strings.TrimSpace(t.sentences[t.sentenceId].Text)
		if sentence == "" {
			t.finished = true
			return "", false
		}
		t.currentSentence = t.processSentence(t.sentences[t.sentenceId].Text)
		t.wordId = 0
	}
	return t.currentSentence[t.wordId], true
}
<commit_msg>Updated names<commit_after>package main

import (
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/neurosnap/sentences"
)

type Tokenizer struct {
	sentences       []*sentences.Sentence
	currentSentence []string
	sentenceID      int
	wordID          int
	finished        bool
}

func NewTokenizer() *Tokenizer {
	return &Tokenizer{}
}

func (t *Tokenizer) Process(fileName string) error {
	t.sentenceID = 0
	t.wordID = 0
	t.finished = false
	file, err := os.Open(fileName)
	if err != nil {
		return err
	}
	return t.process(file)
}

func (t *Tokenizer) process(reader io.Reader) error {
	// TODO: Use proper training data
	training := sentences.NewStorage()
	tokenizer := sentences.NewSentenceTokenizer(training)
	text, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}
	t.sentences = tokenizer.Tokenize(string(text))
	if len(t.sentences) > 0 {
		t.currentSentence = t.processSentence(t.sentences[t.sentenceID].String())
	}
	return nil
}

func (t *Tokenizer) processSentence(text string) []string {
	return strings.Fields(text)
}

func (t *Tokenizer) getNextWord() (string, bool) {
	if t.finished {
		return "", false
	}
	t.wordID++
	if len(t.currentSentence) == t.wordID {
		t.sentenceID++
		if len(t.sentences) == t.sentenceID {
			return "", false
		}
		sentence := strings.TrimSpace(t.sentences[t.sentenceID].Text)
		if sentence == "" {
			t.finished = true
			return "", false
		}
		t.currentSentence = t.processSentence(t.sentences[t.sentenceID].Text)
		t.wordID = 0
	}
	return t.currentSentence[t.wordID], true
}
<|endoftext|>


<commit_before>package LevenshteinTrie

import (
	"bufio"
	"os"
	"testing"
)

var tree *TrieNode

func TestInsert(t *testing.T) {
	tree = NewTrie()
	file, err := os.Open("/usr/share/dict/words")
	if err != nil {
		t.Fatal(err)
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if err != nil {
			break
		}

		tree.InsertText(line)
	}
}

func TestPrefixSearch(t *testing.T) {
	expected := []string{
		"zygosaccharomyces",
		"zygose",
		"zygosis",
		"zygosperm",
		"zygosphenal",
		"zygosphene",
		"zygosphere",
		"zygosporange",
		"zygosporangium",
		"zygospore",
		"zygosporic",
		"zygosporophore",
		"zygostyle",
	}
	words := tree.Suffix("zygos")
	for _, e := range expected {
		if !contains(words, e) {
			t.Errorf("Missing word: %s", e)
		}
	}
}

func contains(words []string, word string) bool {
	for _, w := range words {
		if w == word {
			return true
		}
	}
	return false
}
func TestLevenshteinSearch(t *testing.T) {
	expected := []struct {
		query  string
		result []QueryResult
	}{
		{"accidia", []QueryResult{
			{"accidia", 0},
			{"accidie", 1},
		}},
	}
	for _, e := range expected {
		results := tree.Levenshtein(e.query, 1)
		if !containsq(results, e.query) {
			t.Errorf("Missing term: %s", e.query)
		}
	}
}

func containsq(results []QueryResult, word string) bool {
	for _, r := range results {
		if r.Val == word {
			return true
		}
	}
	return false
}
<commit_msg>add distance to test table<commit_after>package LevenshteinTrie

import (
	"bufio"
	"os"
	"testing"
)

var tree *TrieNode

func TestInsert(t *testing.T) {
	tree = NewTrie()
	file, err := os.Open("/usr/share/dict/words")
	if err != nil {
		t.Fatal(err)
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if err != nil {
			break
		}

		tree.InsertText(line)
	}
}

func TestPrefixSearch(t *testing.T) {
	expected := []string{
		"zygosaccharomyces",
		"zygose",
		"zygosis",
		"zygosperm",
		"zygosphenal",
		"zygosphene",
		"zygosphere",
		"zygosporange",
		"zygosporangium",
		"zygospore",
		"zygosporic",
		"zygosporophore",
		"zygostyle",
	}
	words := tree.Suffix("zygos")
	for _, e := range expected {
		if !contains(words, e) {
			t.Errorf("Missing word: %s", e)
		}
	}
}

func contains(words []string, word string) bool {
	for _, w := range words {
		if w == word {
			return true
		}
	}
	return false
}
func TestLevenshteinSearch(t *testing.T) {
	expected := []struct {
		query    string
		result   []QueryResult
		distance int
	}{
		{"accidia", []QueryResult{
			{"accidia", 0},
			{"accidie", 1},
		}, 1},
	}
	for _, e := range expected {
		results := tree.Levenshtein(e.query, e.distance)
		if !containsq(results, e.query) {
			t.Errorf("Missing term: %s", e.query)
		}
	}
}

func containsq(results []QueryResult, word string) bool {
	for _, r := range results {
		if r.Val == word {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	grn "github.com/hnakamur/cgoroonga"
)

type Site struct {
	URL   string
	Title string
}

func createSiteTable(ctx *grn.Ctx) (err error) {
	keyType := ctx.At(grn.DB_SHORT_TEXT)
	table, err := ctx.TableOpenOrCreate("Site", "",
		grn.OBJ_TABLE_HASH_KEY|grn.OBJ_PERSISTENT, keyType, nil)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, table)

	columnType := ctx.At(grn.DB_TEXT)
	column, err := ctx.ColumnOpenOrCreate(table, "title", "",
		grn.OBJ_PERSISTENT|grn.OBJ_COLUMN_SCALAR, columnType)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, column)

	return
}

func insertSites(ctx *grn.Ctx, sites ...Site) (err error) {
	table := ctx.Get("Site")
	defer ctx.ObjUnlinkDefer(&err, table)

	var column *grn.Obj
	column, err = ctx.ObjColumn(table, "title")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, column)

	var value grn.Obj
	defer ctx.ObjUnlinkDefer(&err, &value)

	for _, site := range sites {
		var recordID grn.ID
		recordID, _, err = ctx.TableAdd(table, site.URL)
		if err != nil {
			return
		}

		grn.TextInit(&value, 0)
		ctx.TextPut(&value, site.Title)
		err = ctx.ObjSetValue(column, recordID, &value, grn.OBJ_SET)
		if err != nil {
			return
		}
	}
	return
}

func selectSites(ctx *grn.Ctx) (err error) {
	table := ctx.Get("Site")
	defer ctx.ObjUnlinkDefer(&err, table)

	var keyColumn *grn.Obj
	keyColumn, err = ctx.ObjColumn(table, "_key")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, keyColumn)

	var titleColumn *grn.Obj
	titleColumn, err = ctx.ObjColumn(table, "title")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, titleColumn)

	res := table

	count, err := ctx.TableSize(res)
	if err != nil {
		return
	}
	fmt.Printf("record count=%d\n", count)

	tc, err := ctx.TableCursorOpen(res, "", "", 0, -1, grn.CURSOR_ASCENDING)
	if err != nil {
		return
	}

	var buf grn.Obj
	defer ctx.ObjUnlinkDefer(&err, &buf)
	for {
		id := ctx.TableCursorNext(tc)
		if id == grn.ID_NIL {
			break
		}
		grn.TextInit(&buf, 0)
		grn.BulkRewind(&buf)
		ctx.ObjGetValue(keyColumn, id, &buf)
		key := grn.BulkHead(&buf)

		grn.TextInit(&buf, 0)
		grn.BulkRewind(&buf)
		ctx.ObjGetValue(titleColumn, id, &buf)
		title := grn.BulkHead(&buf)

		fmt.Printf("key=%s, title=%s\n", key, title)
	}
	ctx.TableCursorClose(tc)

	return
}

func run() (err error) {
	err = grn.Init()
	if err != nil {
		return
	}
	defer grn.FinDefer(&err)

	ctx, err := grn.CtxOpen(0)
	if err != nil {
		return
	}
	defer ctx.CloseDefer(&err)

	var db *grn.Obj
	db, err = ctx.DBOpenOrCreate("site.db", nil)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, db)

	err = createSiteTable(ctx)
	if err != nil {
		return
	}

	err = insertSites(ctx,
		Site{URL: "http://example.org/", Title: "This is test record 1!"},
		Site{URL: "http://example.net/", Title: "test record 2."},
		Site{URL: "http://example.com/", Title: "test test record three."},
		Site{URL: "http://example.net/afr", Title: "test record four."},
		Site{URL: "http://example.org/aba", Title: "test test test record five."},
		Site{URL: "http://example.com/rab", Title: "test test test test record six."},
		Site{URL: "http://example.net/atv", Title: "test test test record seven."},
		Site{URL: "http://example.org/gat", Title: "test test record eight."},
		Site{URL: "http://example.com/vdw", Title: "test test record nine."},
	)
	if err != nil {
		return
	}

	err = selectSites(ctx)
	return
}

func main() {
	err := run()
	if err != nil {
		panic(err)
	}
}
<commit_msg>Add search example function to list_records<commit_after>package main

import (
	"fmt"

	grn "github.com/hnakamur/cgoroonga"
)

type Site struct {
	URL   string
	Title string
}

func createSiteTable(ctx *grn.Ctx) (err error) {
	keyType := ctx.At(grn.DB_SHORT_TEXT)
	table, err := ctx.TableOpenOrCreate("Site", "",
		grn.OBJ_TABLE_HASH_KEY|grn.OBJ_PERSISTENT, keyType, nil)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, table)

	columnType := ctx.At(grn.DB_TEXT)
	column, err := ctx.ColumnOpenOrCreate(table, "title", "",
		grn.OBJ_PERSISTENT|grn.OBJ_COLUMN_SCALAR, columnType)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, column)

	return
}

func insertSites(ctx *grn.Ctx, sites ...Site) (err error) {
	table := ctx.Get("Site")
	defer ctx.ObjUnlinkDefer(&err, table)

	var column *grn.Obj
	column, err = ctx.ObjColumn(table, "title")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, column)

	var value grn.Obj
	defer ctx.ObjUnlinkDefer(&err, &value)

	for _, site := range sites {
		var recordID grn.ID
		recordID, _, err = ctx.TableAdd(table, site.URL)
		if err != nil {
			return
		}

		grn.TextInit(&value, 0)
		ctx.TextPut(&value, site.Title)
		err = ctx.ObjSetValue(column, recordID, &value, grn.OBJ_SET)
		if err != nil {
			return
		}
	}
	return
}

func selectSites(ctx *grn.Ctx) (err error) {
	table := ctx.Get("Site")
	defer ctx.ObjUnlinkDefer(&err, table)

	var keyColumn *grn.Obj
	keyColumn, err = ctx.ObjColumn(table, "_key")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, keyColumn)

	var titleColumn *grn.Obj
	titleColumn, err = ctx.ObjColumn(table, "title")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, titleColumn)

	res := table

	count, err := ctx.TableSize(res)
	if err != nil {
		return
	}
	fmt.Printf("record count=%d\n", count)

	tc, err := ctx.TableCursorOpen(res, "", "", 0, -1, grn.CURSOR_ASCENDING)
	if err != nil {
		return
	}

	var buf grn.Obj
	defer ctx.ObjUnlinkDefer(&err, &buf)
	for {
		id := ctx.TableCursorNext(tc)
		if id == grn.ID_NIL {
			break
		}
		grn.TextInit(&buf, 0)
		grn.BulkRewind(&buf)
		ctx.ObjGetValue(keyColumn, id, &buf)
		key := grn.BulkHead(&buf)

		grn.TextInit(&buf, 0)
		grn.BulkRewind(&buf)
		ctx.ObjGetValue(titleColumn, id, &buf)
		title := grn.BulkHead(&buf)

		fmt.Printf("id=%d, key=%s, title=%s\n", id, key, title)
	}
	ctx.TableCursorClose(tc)

	return
}

func selectComSites(ctx *grn.Ctx) (err error) {
	table := ctx.Get("Site")
	defer ctx.ObjUnlinkDefer(&err, table)

	cond, v, err := ctx.ExprCreateForQuery(table)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, cond)
	defer ctx.ObjUnlinkDefer(&err, v)

	flags := grn.EXPR_SYNTAX_QUERY | grn.EXPR_ALLOW_PRAGMA | grn.EXPR_ALLOW_COLUMN
	err = ctx.ExprParse(cond, "_key:@.com", nil, grn.OP_MATCH, grn.OP_AND, flags)
	if err != nil {
		return
	}

	res, err := ctx.TableSelect(table, cond, nil, grn.OP_OR)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, res)

	count, err := ctx.TableSize(res)
	if err != nil {
		return
	}
	fmt.Printf("record count=%d\n", count)

	var keyColumn *grn.Obj
	keyColumn, err = ctx.ObjColumn(res, "_key")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, keyColumn)

	var titleColumn *grn.Obj
	titleColumn, err = ctx.ObjColumn(res, "title")
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, titleColumn)

	tc, err := ctx.TableCursorOpen(res, "", "", 0, -1, grn.CURSOR_ASCENDING)
	if err != nil {
		return
	}

	var buf grn.Obj
	defer ctx.ObjUnlinkDefer(&err, &buf)
	for {
		id := ctx.TableCursorNext(tc)
		if id == grn.ID_NIL {
			break
		}
		grn.TextInit(&buf, 0)
		grn.BulkRewind(&buf)
		ctx.ObjGetValue(keyColumn, id, &buf)
		key := grn.BulkHead(&buf)

		grn.TextInit(&buf, 0)
		grn.BulkRewind(&buf)
		ctx.ObjGetValue(titleColumn, id, &buf)
		title := grn.BulkHead(&buf)

		fmt.Printf("id=%d, key=%s, title=%s\n", id, key, title)
	}
	ctx.TableCursorClose(tc)

	return
}

func run() (err error) {
	err = grn.Init()
	if err != nil {
		return
	}
	defer grn.FinDefer(&err)

	ctx, err := grn.CtxOpen(0)
	if err != nil {
		return
	}
	defer ctx.CloseDefer(&err)

	var db *grn.Obj
	db, err = ctx.DBOpenOrCreate("site.db", nil)
	if err != nil {
		return
	}
	defer ctx.ObjUnlinkDefer(&err, db)

	err = createSiteTable(ctx)
	if err != nil {
		return
	}

	err = insertSites(ctx,
		Site{URL: "http://example.org/", Title: "This is test record 1!"},
		Site{URL: "http://example.net/", Title: "test record 2."},
		Site{URL: "http://example.com/", Title: "test test record three."},
		Site{URL: "http://example.net/afr", Title: "test record four."},
		Site{URL: "http://example.org/aba", Title: "test test test record five."},
		Site{URL: "http://example.com/rab", Title: "test test test test record six."},
		Site{URL: "http://example.net/atv", Title: "test test test record seven."},
		Site{URL: "http://example.org/gat", Title: "test test record eight."},
		Site{URL: "http://example.com/vdw", Title: "test test record nine."},
	)
	if err != nil {
		return
	}

	err = selectSites(ctx)
	if err != nil {
		return
	}

	err = selectComSites(ctx)
	return
}

func main() {
	err := run()
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"

	"github.com/asaskevich/govalidator"
	"github.com/gin-gonic/gin"
	"github.com/luizbafilho/fusis/api/types"
)

func (as ApiService) serviceList(c *gin.Context) {
	services := as.balancer.GetServices()
	if len(services) == 0 {
		c.Status(http.StatusNoContent)
		return
	}
	c.JSON(http.StatusOK, services)
}

func (as ApiService) serviceGet(c *gin.Context) {
	serviceId := c.Param("service_name")
	service, err := as.balancer.GetService(serviceId)
	if err != nil {
		if err == types.ErrServiceNotFound {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("GetService() failed: %v", err)})
		}
		return
	}
	c.JSON(http.StatusOK, service)
}

func (as ApiService) serviceCreate(c *gin.Context) {
	var newService types.Service
	if err := c.BindJSON(&newService); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	//Guarantees that no one tries to create a destination together with a service
	newService.Destinations = []types.Destination{}

	if _, errs := govalidator.ValidateStruct(newService); errs != nil {
		c.JSON(http.StatusBadRequest, gin.H{"errors": govalidator.ErrorsByField(errs)})
		return
	}

	// If everthing is ok send it to Raft
	err := as.balancer.AddService(&newService)
	if err != nil {
		if err == types.ErrServiceAlreadyExists {
			c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("UpsertService() failed: %v", err)})
		}
		return
	}

	c.Header("Location", fmt.Sprintf("/services/%s", newService.Name))
	c.JSON(http.StatusCreated, newService)
}

func (as ApiService) serviceDelete(c *gin.Context) {
	serviceId := c.Param("service_name")
	_, err := as.balancer.GetService(serviceId)
	if err != nil {
		if err == types.ErrServiceNotFound {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("GetService() failed: %v", err)})
		}
		return
	}

	err = as.balancer.DeleteService(serviceId)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("DeleteService() failed: %v\n", err)})
		return
	}

	c.Status(http.StatusNoContent)
}

func (as ApiService) destinationCreate(c *gin.Context) {
	serviceName := c.Param("service_name")
	service, err := as.balancer.GetService(serviceName)
	if err != nil {
		if err == types.ErrServiceNotFound {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("GetService() failed: %v", err)})
		}
		return
	}

	destination := &types.Destination{Weight: 1, Mode: "route", ServiceId: serviceName}
	if err := c.BindJSON(destination); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, errs := govalidator.ValidateStruct(destination); errs != nil {
		c.JSON(http.StatusBadRequest, gin.H{"errors": govalidator.ErrorsByField(errs)})
		return
	}

	err = as.balancer.AddDestination(service, destination)
	if err != nil {
		if err == types.ErrDestinationAlreadyExists {
			c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("UpsertDestination() failed: %v\n", err)})
		}
		return
	}

	c.Header("Location", fmt.Sprintf("/services/%s/destinations/%s", serviceName, destination.Name))
	c.JSON(http.StatusCreated, destination)
}

func (as ApiService) destinationDelete(c *gin.Context) {
	destinationId := c.Param("destination_name")
	dst, err := as.balancer.GetDestination(destinationId)
	if err != nil {
		if _, ok := err.(types.ErrNotFound); ok {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("GetDestination() failed: %v", err)})
		}
		return
	}

	err = as.balancer.DeleteDestination(dst)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("DeleteDestination() failed: %v\n", err)})
	}

	c.Status(http.StatusNoContent)
}

func (as ApiService) flush(c *gin.Context) {
	// err := as.types.Flush()
	// if err != nil {
	// 	c.JSON(400, gin.H{"error": err.Error()})
	// 	return
	// }
	//
	// err = types.Flush()
	// if err != nil {
	// 	c.JSON(400, gin.H{"error": err.Error()})
	// 	return
	// }
}
<commit_msg>api: add errors to gin context for logging<commit_after>package api

import (
	"fmt"
	"net/http"

	"github.com/asaskevich/govalidator"
	"github.com/gin-gonic/gin"
	"github.com/luizbafilho/fusis/api/types"
)

func (as ApiService) serviceList(c *gin.Context) {
	services := as.balancer.GetServices()
	if len(services) == 0 {
		c.Status(http.StatusNoContent)
		return
	}
	c.JSON(http.StatusOK, services)
}

func (as ApiService) serviceGet(c *gin.Context) {
	serviceId := c.Param("service_name")
	service, err := as.balancer.GetService(serviceId)
	if err != nil {
		c.Error(err)
		if err == types.ErrServiceNotFound {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("GetService() failed: %v", err)})
		}
		return
	}
	c.JSON(http.StatusOK, service)
}

func (as ApiService) serviceCreate(c *gin.Context) {
	var newService types.Service
	if err := c.BindJSON(&newService); err != nil {
		c.Error(err)
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	//Guarantees that no one tries to create a destination together with a service
	newService.Destinations = []types.Destination{}

	if _, errs := govalidator.ValidateStruct(newService); errs != nil {
		c.Error(errs)
		c.JSON(http.StatusBadRequest, gin.H{"errors": govalidator.ErrorsByField(errs)})
		return
	}

	// If everthing is ok send it to Raft
	err := as.balancer.AddService(&newService)
	if err != nil {
		c.Error(err)
		if err == types.ErrServiceAlreadyExists {
			c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("UpsertService() failed: %v", err)})
		}
		return
	}

	c.Header("Location", fmt.Sprintf("/services/%s", newService.Name))
	c.JSON(http.StatusCreated, newService)
}

func (as ApiService) serviceDelete(c *gin.Context) {
	serviceId := c.Param("service_name")
	_, err := as.balancer.GetService(serviceId)
	if err != nil {
		c.Error(err)
		if err == types.ErrServiceNotFound {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("GetService() failed: %v", err)})
		}
		return
	}

	err = as.balancer.DeleteService(serviceId)
	if err != nil {
		c.Error(err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("DeleteService() failed: %v\n", err)})
		return
	}

	c.Status(http.StatusNoContent)
}

func (as ApiService) destinationCreate(c *gin.Context) {
	serviceName := c.Param("service_name")
	service, err := as.balancer.GetService(serviceName)
	if err != nil {
		c.Error(err)
		if err == types.ErrServiceNotFound {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("GetService() failed: %v", err)})
		}
		return
	}

	destination := &types.Destination{Weight: 1, Mode: "route", ServiceId: serviceName}
	if err := c.BindJSON(destination); err != nil {
		c.Error(err)
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, errs := govalidator.ValidateStruct(destination); errs != nil {
		c.Error(errs)
		c.JSON(http.StatusBadRequest, gin.H{"errors": govalidator.ErrorsByField(errs)})
		return
	}

	err = as.balancer.AddDestination(service, destination)
	if err != nil {
		c.Error(err)
		if err == types.ErrDestinationAlreadyExists {
			c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("UpsertDestination() failed: %v\n", err)})
		}
		return
	}

	c.Header("Location", fmt.Sprintf("/services/%s/destinations/%s", serviceName, destination.Name))
	c.JSON(http.StatusCreated, destination)
}

func (as ApiService) destinationDelete(c *gin.Context) {
	destinationId := c.Param("destination_name")
	dst, err := as.balancer.GetDestination(destinationId)
	if err != nil {
		c.Error(err)
		if _, ok := err.(types.ErrNotFound); ok {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("GetDestination() failed: %v", err)})
		}
		return
	}

	err = as.balancer.DeleteDestination(dst)
	if err != nil {
		c.Error(err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("DeleteDestination() failed: %v\n", err)})
	}

	c.Status(http.StatusNoContent)
}

func (as ApiService) flush(c *gin.Context) {
	// err := as.types.Flush()
	// if err != nil {
	// 	c.JSON(400, gin.H{"error": err.Error()})
	// 	return
	// }
	//
	// err = types.Flush()
	// if err != nil {
	// 	c.JSON(400, gin.H{"error": err.Error()})
	// 	return
	// }
}
<|endoftext|>


<commit_before>package app

import (
	"crypto/ecdsa"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/state"
	ethTypes "github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethdb"

	"github.com/tendermint/abci/types"
	"github.com/tendermint/tmlibs/log"
)

func setupTestCase(t *testing.T) (app *TendereumApplication, tearDown func(t *testing.T)) {
	t.Log("Setting up one test case.")

	logger := log.NewTMLogger(os.Stdout)
	tmpDir, err := ioutil.TempDir("", "tendereum-apptest")
	require.NoError(t, err)

	app = NewTendereumApplication(tmpDir, logger)

	tearDown = func(t *testing.T) {
		t.Log("Tearing down one test case.")
		os.RemoveAll(tmpDir)
	}

	return app, tearDown
}

func TestInfo(t *testing.T) {
	assert := assert.New(t)
	_ = require.New(t)

	app, tearDown := setupTestCase(t)
	defer tearDown(t)

	req := types.RequestInfo{Version: fmt.Sprintf("0.11.0")}
	res := app.Info(req)

	assert.Equal("Tendereum", res.Data)
	assert.Equal("0.1.0", res.Version)
	hash := common.BytesToHash(res.LastBlockAppHash)
	assert.False(common.EmptyHash(hash))
}

func setupDB() (ethdb.Database, *state.StateDB, error) {
	tmpDir, err := ioutil.TempDir("", "tendereum-apptest")
	if err != nil {
		return nil, nil, err
	}
	return loadDB(tmpDir)
}

func TestStore(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	eth, db, err := setupDB()
	require.NoError(err)
	// note that original hash is not 0 bytes (as per state.New above),
	// but some actual value
	ogHash := db.IntermediateRoot(true)

	// account is non-existent
	addr := common.StringToAddress("hello")
	assert.True(db.Empty(addr))

	// until we set a value
	val := big.NewInt(123456789)
	db.SetBalance(addr, val)
	assert.False(db.Empty(addr))
	assert.Equal(val, db.GetBalance(addr))

	// perform hash, see it change
	hash := db.IntermediateRoot(true)
	assert.NotEqual(ogHash, hash)

	// write the db, commit doesn't change hash
	commit, err := db.CommitTo(eth, true)
	require.NoError(err)
	assert.Equal(hash, commit)
}

func balanceQuery(addr common.Address) types.RequestQuery {
	return types.RequestQuery{
		Path: QueryBalance,
		Data: addr.Bytes(),
	}
}

func nonceQuery(addr common.Address) types.RequestQuery {
	return types.RequestQuery{
		Path: QueryNonce,
		Data: addr.Bytes(),
	}
}

func checkBalance(t *testing.T, app *TendereumApplication, addr common.Address, expected *big.Int) {
	qres := app.Query(balanceQuery(addr))
	require.True(t, qres.GetCode().IsOK())
	assert.Equal(t, expected.String(), qres.GetLog())
}

func checkNonce(t *testing.T, app *TendereumApplication, addr common.Address, expected uint64) {
	qres := app.Query(nonceQuery(addr))
	require.True(t, qres.GetCode().IsOK())
	assert.Equal(t, fmt.Sprintf("%d", expected), qres.GetLog())
}

func TestGenesisQuery(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	app, tearDown := setupTestCase(t)
	defer tearDown(t)
	og := app.was.state.IntermediateRoot(true)

	// create three addresses
	addr1 := common.StringToAddress("genesis")
	addr2 := common.StringToAddress("block")
	addr3 := common.StringToAddress("fomo")

	// two are granted balance in genesis
	bal1 := big.NewInt(1234567890)
	bal2 := big.NewInt(9988776655443322)
	zero := big.NewInt(0)

	// grant them
	res := app.SetOption(string(addr1[:]), bal1.String())
	require.Equal("Success", res)
	res = app.SetOption(string(addr2[:]), bal2.String())
	require.Equal("Success", res)

	// query values
	checkBalance(t, app, addr1, bal1)
	checkBalance(t, app, addr2, bal2)
	checkBalance(t, app, addr3, zero)
	checkNonce(t, app, addr1, 0)

	qbad := app.Query(types.RequestQuery{Path: "/bad path"})
	assert.False(qbad.GetCode().IsOK())

	// committing this data should update hash
	cres := app.Commit()
	require.True(cres.IsOK())
	hash := common.BytesToHash(cres.Data)
	assert.NotEqual(og, hash)

	// queries should still work
	checkBalance(t, app, addr2, bal2)

	// commit again doesn't change anything
	cres = app.Commit()
	require.True(cres.IsOK())
	hash2 := common.BytesToHash(cres.Data)
	assert.Equal(hash, hash2)
}

func TestSendTx(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	app, tearDown := setupTestCase(t)
	defer tearDown(t)

	// some constants
	recv := common.StringToAddress("receive")
	bal := big.NewInt(9988776655443322)
	amount := big.NewInt(98765432100)

	// make new key and address
	priv, err := crypto.GenerateKey()
	require.NoError(err)
	pub := priv.Public()
	ecPub := pub.(*ecdsa.PublicKey)
	sender := crypto.PubkeyToAddress(*ecPub)

	// grant initial tokens
	res := app.SetOption(string(sender[:]), bal.String())
	require.Equal("Success", res)

	// commit genesis, and get initial root hash
	cres := app.Commit()
	require.True(cres.IsOK())
	hash := common.BytesToHash(cres.Data)

	// create tx
	gasLimit := big.NewInt(1000000)
	gasPrice := big.NewInt(100)
	tx := ethTypes.NewTransaction(0, recv, amount, gasLimit, gasPrice, nil)

	// sign it
	tx, err = ethTypes.SignTx(tx, app.signer, priv)
	require.NoError(err)
	from, err := ethTypes.Sender(app.signer, tx)
	require.NoError(err)
	assert.Equal(sender, from)

	// debug
	fmt.Printf("Tx: %s\n", tx)

	// generate bytes
	txBytes, err := encodeTx(tx)
	require.NoError(err)

	// make sure checktx works
	check := app.CheckTx(txBytes)
	assert.True(check.IsOK(), check.Log)

	// check the delivery
	checkNonce(t, app, sender, 0)
	del := app.DeliverTx(txBytes)
	assert.True(del.IsOK(), del.Log)

	// commit again, and very the hash changed
	cres = app.Commit()
	require.True(cres.IsOK())
	hash2 := common.BytesToHash(cres.Data)
	assert.NotEqual(hash, hash2)

	// verify state is good
	checkNonce(t, app, sender, 1)
	after := bal.Sub(bal, amount)
	// where does this come from???
	after.Sub(after, big.NewInt(2100000))
	checkBalance(t, app, sender, after)

}
<commit_msg>Assert gas handling for sendtx<commit_after>package app

import (
	"crypto/ecdsa"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/state"
	ethTypes "github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethdb"

	"github.com/tendermint/abci/types"
	"github.com/tendermint/tmlibs/log"
)

func setupTestCase(t *testing.T) (app *TendereumApplication, tearDown func(t *testing.T)) {
	t.Log("Setting up one test case.")

	logger := log.NewTMLogger(os.Stdout)
	tmpDir, err := ioutil.TempDir("", "tendereum-apptest")
	require.NoError(t, err)

	app = NewTendereumApplication(tmpDir, logger)

	tearDown = func(t *testing.T) {
		t.Log("Tearing down one test case.")
		os.RemoveAll(tmpDir)
	}

	return app, tearDown
}

func TestInfo(t *testing.T) {
	assert := assert.New(t)
	_ = require.New(t)

	app, tearDown := setupTestCase(t)
	defer tearDown(t)

	req := types.RequestInfo{Version: fmt.Sprintf("0.11.0")}
	res := app.Info(req)

	assert.Equal("Tendereum", res.Data)
	assert.Equal("0.1.0", res.Version)
	hash := common.BytesToHash(res.LastBlockAppHash)
	assert.False(common.EmptyHash(hash))
}

func setupDB() (ethdb.Database, *state.StateDB, error) {
	tmpDir, err := ioutil.TempDir("", "tendereum-apptest")
	if err != nil {
		return nil, nil, err
	}
	return loadDB(tmpDir)
}

func TestStore(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	eth, db, err := setupDB()
	require.NoError(err)
	// note that original hash is not 0 bytes (as per state.New above),
	// but some actual value
	ogHash := db.IntermediateRoot(true)

	// account is non-existent
	addr := common.StringToAddress("hello")
	assert.True(db.Empty(addr))

	// until we set a value
	val := big.NewInt(123456789)
	db.SetBalance(addr, val)
	assert.False(db.Empty(addr))
	assert.Equal(val, db.GetBalance(addr))

	// perform hash, see it change
	hash := db.IntermediateRoot(true)
	assert.NotEqual(ogHash, hash)

	// write the db, commit doesn't change hash
	commit, err := db.CommitTo(eth, true)
	require.NoError(err)
	assert.Equal(hash, commit)
}

func balanceQuery(addr common.Address) types.RequestQuery {
	return types.RequestQuery{
		Path: QueryBalance,
		Data: addr.Bytes(),
	}
}

func nonceQuery(addr common.Address) types.RequestQuery {
	return types.RequestQuery{
		Path: QueryNonce,
		Data: addr.Bytes(),
	}
}

func checkBalance(t *testing.T, app *TendereumApplication, addr common.Address, expected *big.Int) {
	qres := app.Query(balanceQuery(addr))
	require.True(t, qres.GetCode().IsOK())
	assert.Equal(t, expected.String(), qres.GetLog())
}

func checkNonce(t *testing.T, app *TendereumApplication, addr common.Address, expected uint64) {
	qres := app.Query(nonceQuery(addr))
	require.True(t, qres.GetCode().IsOK())
	assert.Equal(t, fmt.Sprintf("%d", expected), qres.GetLog())
}

func TestGenesisQuery(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	app, tearDown := setupTestCase(t)
	defer tearDown(t)
	og := app.was.state.IntermediateRoot(true)

	// create three addresses
	addr1 := common.StringToAddress("genesis")
	addr2 := common.StringToAddress("block")
	addr3 := common.StringToAddress("fomo")

	// two are granted balance in genesis
	bal1 := big.NewInt(1234567890)
	bal2 := big.NewInt(9988776655443322)
	zero := big.NewInt(0)

	// grant them
	res := app.SetOption(string(addr1[:]), bal1.String())
	require.Equal("Success", res)
	res = app.SetOption(string(addr2[:]), bal2.String())
	require.Equal("Success", res)

	// query values
	checkBalance(t, app, addr1, bal1)
	checkBalance(t, app, addr2, bal2)
	checkBalance(t, app, addr3, zero)
	checkNonce(t, app, addr1, 0)

	qbad := app.Query(types.RequestQuery{Path: "/bad path"})
	assert.False(qbad.GetCode().IsOK())

	// committing this data should update hash
	cres := app.Commit()
	require.True(cres.IsOK())
	hash := common.BytesToHash(cres.Data)
	assert.NotEqual(og, hash)

	// queries should still work
	checkBalance(t, app, addr2, bal2)

	// commit again doesn't change anything
	cres = app.Commit()
	require.True(cres.IsOK())
	hash2 := common.BytesToHash(cres.Data)
	assert.Equal(hash, hash2)
}

func TestSendTx(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	app, tearDown := setupTestCase(t)
	defer tearDown(t)

	// some constants
	recv := common.StringToAddress("receive")
	bal := big.NewInt(9988776655443322)
	amount := big.NewInt(98765432100)

	// make new key and address
	priv, err := crypto.GenerateKey()
	require.NoError(err)
	pub := priv.Public()
	ecPub := pub.(*ecdsa.PublicKey)
	sender := crypto.PubkeyToAddress(*ecPub)

	// grant initial tokens
	res := app.SetOption(string(sender[:]), bal.String())
	require.Equal("Success", res)

	// commit genesis, and get initial root hash
	cres := app.Commit()
	require.True(cres.IsOK())
	hash := common.BytesToHash(cres.Data)

	// create tx
	// cutoff here...
	gasLimit := big.NewInt(21000)
	gasPrice := big.NewInt(50)
	tx := ethTypes.NewTransaction(0, recv, amount, gasLimit, gasPrice, nil)

	// sign it
	tx, err = ethTypes.SignTx(tx, app.signer, priv)
	require.NoError(err)
	from, err := ethTypes.Sender(app.signer, tx)
	require.NoError(err)
	assert.Equal(sender, from)

	// debug
	fmt.Printf("Tx: %s\n", tx)

	// generate bytes
	txBytes, err := encodeTx(tx)
	require.NoError(err)

	// make sure checktx works
	check := app.CheckTx(txBytes)
	assert.True(check.IsOK(), check.Log)

	// check the delivery
	checkNonce(t, app, sender, 0)
	del := app.DeliverTx(txBytes)
	assert.True(del.IsOK(), del.Log)

	// commit again, and very the hash changed
	cres = app.Commit()
	require.True(cres.IsOK())
	hash2 := common.BytesToHash(cres.Data)
	assert.NotEqual(hash, hash2)

	// verify state is good
	checkNonce(t, app, sender, 1)
	after := bal.Sub(bal, amount)
	// where does this come from???
	gasPaid := big.NewInt(0).Mul(big.NewInt(21000), gasPrice)
	after.Sub(after, gasPaid)
	checkBalance(t, app, sender, after)

}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/gofxh/blog/app"
	"github.com/gofxh/blog/app/core"
	"github.com/gofxh/blog/app/log"
	"path/filepath"
	"time"
)

var Serv = cli.Command{
	Name:   "serv",
	Usage:  "start web server",
	Action: ServAction,
}

func ServAction(ctx *cli.Context) {
	t := time.Now()
	log.Info("Serv|Begin")

	// init global vars
	app.Db = core.NewDatabase(filepath.Join(app.Config.UserDirectory, app.Config.UserDataFile))
	app.Server = core.NewServer(fmt.Sprintf("%s:%s", app.Config.HttpHost, app.Config.HttpAddress))

	// start server
	core.Start(app.Server)

	log.Info("Serv|Close|%.1fms", time.Since(t).Seconds()*1000)
}
<commit_msg>add server-init action<commit_after>package cmd

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/gofxh/blog/app"
	"github.com/gofxh/blog/app/action"
	"github.com/gofxh/blog/app/core"
	"github.com/gofxh/blog/app/log"
	"github.com/lunny/tango"
	"github.com/tango-contrib/binding"
	"github.com/tango-contrib/renders"
	"path/filepath"
	"time"
)

var Serv = cli.Command{
	Name:   "serv",
	Usage:  "start web server",
	Action: ServAction,
}

func ServAction(ctx *cli.Context) {
	t := time.Now()
	log.Info("Serv|Begin")

	// init global vars
	app.Db = core.NewDatabase(filepath.Join(app.Config.UserDirectory, app.Config.UserDataFile))
	app.Server = core.NewServer(fmt.Sprintf("%s:%s", app.Config.HttpHost, app.Config.HttpAddress))

	// init server
	action.Call(InitServer, nil)
	// init router
	action.Call(InitRoute, nil)

	// start server
	core.Start(app.Server)

	log.Info("Serv|Close|%.1fms", time.Since(t).Seconds()*1000)
}

// init server,
// set static files, theme and middleware
func InitServer(_ interface{}) *action.Result {

	// set static directory
	app.Server.Use(tango.Static(tango.StaticOptions{
		RootPath: filepath.Join(app.Config.UserDirectory, app.Config.UserThemeDirectory),
		Prefix:   "theme",
	}))
	app.Server.Use(tango.Static(tango.StaticOptions{
		RootPath: filepath.Join(app.Config.UserDirectory, app.Config.UserUploadDirectory),
		Prefix:   "upload",
	}))

	// set theme directory
	app.Server.Use(renders.New(renders.Options{
		Reload:     true,
		Directory:  filepath.Join(app.Config.UserDirectory, app.Config.UserThemeDirectory),
		Extensions: []string{".tmpl"},
	}))

	// binding middleware
	app.Server.Use(binding.Bind())

	return action.OkResult(nil)
}

// init router,
// set route rules
func InitRoute(_ interface{}) *action.Result {
	return action.OkResult(nil)
}
<|endoftext|>


<commit_before>package main

import (
    "os"
    "log"
    "fmt"
    "flag"
    "time"
    "sort"
    "bufio"
    "regexp"
    "strings"
    "strconv"
    "io/ioutil"
    "path/filepath"
)

const (
    HISTORY_SIZE = 600
    AGING_CONSTANT = 86400
    FIELD_SEP = "\x00"
    SORT_BY_FRECENCY = "frecency"
    SORT_BY_HITS = "hits"
    SORT_BY_ATIME = "atime"
)

var (
    now = time.Now().Unix()
    historySize int64
    agingConstant int64
)

type Data struct {
    path string
    hits, atime int64
}

type Datae []Data

type ByFrecency struct {
    Datae
}

type ByAtime struct {
    Datae
}

type ByHits struct {
    Datae
}

func (d Datae) Len() int {
    return len(d)
}

func (d Datae) Swap(i, j int) {
    d[i], d[j] = d[j], d[i]
}

func(b ByFrecency) Less(i, j int) bool {
    return Score(b.Datae[i].hits, now - b.Datae[i].atime) < Score(b.Datae[j].hits, now - b.Datae[j].atime)
}

func (b ByHits) Less(i, j int) bool {
    return b.Datae[i].hits < b.Datae[j].hits
}

func (b ByAtime) Less(i, j int) bool {
    return b.Datae[i].atime < b.Datae[j].atime
}

func Score(hits int64, age int64) (float64) {
    return float64(hits) * float64(agingConstant) / float64(agingConstant + age)
}

func (d Datae) Sort(method string) {
    if method == SORT_BY_FRECENCY {
        sort.Sort(sort.Reverse(ByFrecency{d}))
    } else if method == SORT_BY_HITS {
        sort.Sort(sort.Reverse(ByHits{d}))
    } else if method == SORT_BY_ATIME {
        sort.Sort(sort.Reverse(ByAtime{d}))
    } else {
        log.Fatalf("Unknown sort method: '%v'.", method)
    }
}

func check(e error) {
    if (e != nil) {
        panic(e)
    }
}

func ReadData(r *bufio.Reader) (Data, error) {
    line, err := r.ReadString('\n')
    if (err != nil) {
        return Data{}, err
    }
    line = line[:len(line) - 1]
    tok := strings.Split(line, FIELD_SEP)
    atime, err := strconv.ParseInt(tok[0], 10, 64)
    check(err)
    hits, err := strconv.ParseInt(tok[1], 10, 64)
    check(err)
    path := tok[2]
    return Data{path, hits, atime}, nil
}

func main() {
    var dataFile string
    var pathFlag string
    if dataFile = os.Getenv("Z_DATA_FILE"); len(dataFile) == 0 {
        dataFile = os.Getenv("HOME") + string(os.PathSeparator) + ".z"
    }
    if historySize, _ = strconv.ParseInt(os.Getenv("Z_HISTORY_SIZE"), 10, 64); historySize < 1 {
        historySize = HISTORY_SIZE
    }
    if agingConstant, _ = strconv.ParseInt(os.Getenv("Z_AGING_CONSTANT"), 10, 64); agingConstant < 1 {
        agingConstant = AGING_CONSTANT
    }
    results := make(Datae, 0, historySize)
    addFlag := flag.String("a", "", "Add the given item to the data file")
    deleteFlag := flag.String("d", "", "Delete the given item from the data file")
    sortFlag := flag.String("s", SORT_BY_FRECENCY, "Use the given sort method")
    inputFlag := flag.String("i", dataFile, "Use the given file as data file")
    flag.Parse()
    dataFile = *inputFlag
    if len(*addFlag) != 0 {
        pathFlag = *addFlag
    } else if len(*deleteFlag) != 0 {
        pathFlag = *deleteFlag
    }
    fobj, err := os.Open(dataFile)
    check(err)
    defer fobj.Close()
    var bf = bufio.NewReader(fobj)
    if len(pathFlag) == 0 {
        var sPattern string
        if sPattern = strings.Join(flag.Args(), ".*"); len(sPattern) == 0 {
            sPattern = ".*"
        }
        pattern, err := regexp.Compile("(?i)" + sPattern)
        check(err)
        for d, err := ReadData(bf); err == nil; d, err = ReadData(bf) {
            if pattern.MatchString(d.path) {
                results = append(results, d)
            }
        }
        results.Sort(*sortFlag)
        for _, d := range results {
            fmt.Printf("%v\n", d.path)
        }
    } else {
        var index int64 = -1
        var cur int64
        for d, err := ReadData(bf); err == nil; d, err = ReadData(bf) {
            if (index < 0 && pathFlag == d.path) {
                index = cur
            }
            results = append(results, d)
            cur++
        }
        if len(*addFlag) != 0 {
            if (index < 0) {
                results = append(results, Data{pathFlag, 1, now})
            } else {
                results[index] = Data{pathFlag, results[index].hits + 1, now}
            }
            results.Sort(*sortFlag)
        } else if len(*deleteFlag) != 0 {
            if (index < 0) {
                log.Printf("Item is missing: '%v'.", pathFlag)
            } else {
                results = append(results[:index], results[index + 1:]...)
            }
        }
        fobj, err := ioutil.TempFile(filepath.Dir(dataFile), filepath.Base(dataFile))
        check(err)
        defer fobj.Close()
        bf := bufio.NewWriter(fobj)
        cur = 0
        for _, d := range results {
            if cur >= historySize {
                break
            }
            _, err := bf.WriteString(fmt.Sprintf("%v%v%v%v%v\n", d.atime, FIELD_SEP, d.hits, FIELD_SEP, d.path))
            check(err)
            cur++
        }
        err = bf.Flush()
        check(err)
        err = os.Rename(fobj.Name(), dataFile)
        check(err)
    }
}
<commit_msg>Reformat via gofmt<commit_after>package main

import (
    "bufio"
    "flag"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "path/filepath"
    "regexp"
    "sort"
    "strconv"
    "strings"
    "time"
)

const (
    HISTORY_SIZE     = 600
    AGING_CONSTANT   = 86400
    FIELD_SEP        = "\x00"
    SORT_BY_FRECENCY = "frecency"
    SORT_BY_HITS     = "hits"
    SORT_BY_ATIME    = "atime"
)

var (
    now           = time.Now().Unix()
    historySize   int64
    agingConstant int64
)

type Data struct {
    path        string
    hits, atime int64
}

type Datae []Data

type ByFrecency struct {
    Datae
}

type ByAtime struct {
    Datae
}

type ByHits struct {
    Datae
}

func (d Datae) Len() int {
    return len(d)
}

func (d Datae) Swap(i, j int) {
    d[i], d[j] = d[j], d[i]
}

func (b ByFrecency) Less(i, j int) bool {
    return Score(b.Datae[i].hits, now-b.Datae[i].atime) < Score(b.Datae[j].hits, now-b.Datae[j].atime)
}

func (b ByHits) Less(i, j int) bool {
    return b.Datae[i].hits < b.Datae[j].hits
}

func (b ByAtime) Less(i, j int) bool {
    return b.Datae[i].atime < b.Datae[j].atime
}

func Score(hits int64, age int64) float64 {
    return float64(hits) * float64(agingConstant) / float64(agingConstant+age)
}

func (d Datae) Sort(method string) {
    if method == SORT_BY_FRECENCY {
        sort.Sort(sort.Reverse(ByFrecency{d}))
    } else if method == SORT_BY_HITS {
        sort.Sort(sort.Reverse(ByHits{d}))
    } else if method == SORT_BY_ATIME {
        sort.Sort(sort.Reverse(ByAtime{d}))
    } else {
        log.Fatalf("Unknown sort method: '%v'.", method)
    }
}

func check(e error) {
    if e != nil {
        panic(e)
    }
}

func ReadData(r *bufio.Reader) (Data, error) {
    line, err := r.ReadString('\n')
    if err != nil {
        return Data{}, err
    }
    line = line[:len(line)-1]
    tok := strings.Split(line, FIELD_SEP)
    atime, err := strconv.ParseInt(tok[0], 10, 64)
    check(err)
    hits, err := strconv.ParseInt(tok[1], 10, 64)
    check(err)
    path := tok[2]
    return Data{path, hits, atime}, nil
}

func main() {
    var dataFile string
    var pathFlag string
    if dataFile = os.Getenv("Z_DATA_FILE"); len(dataFile) == 0 {
        dataFile = os.Getenv("HOME") + string(os.PathSeparator) + ".z"
    }
    if historySize, _ = strconv.ParseInt(os.Getenv("Z_HISTORY_SIZE"), 10, 64); historySize < 1 {
        historySize = HISTORY_SIZE
    }
    if agingConstant, _ = strconv.ParseInt(os.Getenv("Z_AGING_CONSTANT"), 10, 64); agingConstant < 1 {
        agingConstant = AGING_CONSTANT
    }
    results := make(Datae, 0, historySize)
    addFlag := flag.String("a", "", "Add the given item to the data file")
    deleteFlag := flag.String("d", "", "Delete the given item from the data file")
    sortFlag := flag.String("s", SORT_BY_FRECENCY, "Use the given sort method")
    inputFlag := flag.String("i", dataFile, "Use the given file as data file")
    flag.Parse()
    dataFile = *inputFlag
    if len(*addFlag) != 0 {
        pathFlag = *addFlag
    } else if len(*deleteFlag) != 0 {
        pathFlag = *deleteFlag
    }
    fobj, err := os.Open(dataFile)
    check(err)
    defer fobj.Close()
    var bf = bufio.NewReader(fobj)
    if len(pathFlag) == 0 {
        var sPattern string
        if sPattern = strings.Join(flag.Args(), ".*"); len(sPattern) == 0 {
            sPattern = ".*"
        }
        pattern, err := regexp.Compile("(?i)" + sPattern)
        check(err)
        for d, err := ReadData(bf); err == nil; d, err = ReadData(bf) {
            if pattern.MatchString(d.path) {
                results = append(results, d)
            }
        }
        results.Sort(*sortFlag)
        for _, d := range results {
            fmt.Printf("%v\n", d.path)
        }
    } else {
        var index int64 = -1
        var cur int64
        for d, err := ReadData(bf); err == nil; d, err = ReadData(bf) {
            if index < 0 && pathFlag == d.path {
                index = cur
            }
            results = append(results, d)
            cur++
        }
        if len(*addFlag) != 0 {
            if index < 0 {
                results = append(results, Data{pathFlag, 1, now})
            } else {
                results[index] = Data{pathFlag, results[index].hits + 1, now}
            }
            results.Sort(*sortFlag)
        } else if len(*deleteFlag) != 0 {
            if index < 0 {
                log.Printf("Item is missing: '%v'.", pathFlag)
            } else {
                results = append(results[:index], results[index+1:]...)
            }
        }
        fobj, err := ioutil.TempFile(filepath.Dir(dataFile), filepath.Base(dataFile))
        check(err)
        defer fobj.Close()
        bf := bufio.NewWriter(fobj)
        cur = 0
        for _, d := range results {
            if cur >= historySize {
                break
            }
            _, err := bf.WriteString(fmt.Sprintf("%v%v%v%v%v\n", d.atime, FIELD_SEP, d.hits, FIELD_SEP, d.path))
            check(err)
            cur++
        }
        err = bf.Flush()
        check(err)
        err = os.Rename(fobj.Name(), dataFile)
        check(err)
    }
}
<|endoftext|>


<commit_before>package tika

import (
    "context"
    "fmt"
    "net/http"
    "net/url"
    "testing"

    "github.com/dankinder/httpmock"
    "github.com/stretchr/testify/mock"
    "github.com/stretchr/testify/suite"

    "github.com/ipfs-search/ipfs-search/components/extractor"
    indexTypes "github.com/ipfs-search/ipfs-search/components/index/types"
    "github.com/ipfs-search/ipfs-search/components/protocol"

    "github.com/ipfs-search/ipfs-search/instr"
    t "github.com/ipfs-search/ipfs-search/types"
)

const testCID = "QmehHHRh1a7u66r7fugebp6f6wGNMGCa7eho9cgjwhAcm2"

type TikaTestSuite struct {
    suite.Suite

    ctx context.Context
    e   extractor.Extractor

    cfg *Config
    // httpClient *http.Client
    protocol *protocol.Mock
    // instr      *instr.Instrumentation

    mockAPIHandler *httpmock.MockHandler
    mockAPIServer  *httpmock.Server
    responseHeader http.Header
}

func (s *TikaTestSuite) SetupTest() {
    s.ctx = context.Background()

    s.mockAPIHandler = &httpmock.MockHandler{}
    s.mockAPIServer = httpmock.NewServer(s.mockAPIHandler)
    s.responseHeader = http.Header{
        "Content-Type": []string{"application/json"},
    }

    s.cfg = DefaultConfig()
    s.cfg.TikaServerURL = s.mockAPIServer.URL()

    s.protocol = &protocol.Mock{}

    s.e = New(s.cfg, http.DefaultClient, s.protocol, instr.New())
}

func (s *TikaTestSuite) TearDownTest() {
    s.mockAPIServer.Close()
}

func (s TikaTestSuite) TestExtract() {
    testJSON := []byte(`
        {
          "metadata": {
            "title": [
              "How Filecoin Supports Video Storage"
            ],
            "Content-Type": [
              "text/html; charset=UTF-8"
            ]
          },
          "content": "The Filecoin Space Race is now live! Learn More\n\t\t Thank you!",
          "language": {
            "language": "en",
            "confidence": "HIGH",
            "rawScore": 0.99999505
          },
          "urls": [
            "https://filecoin.io/uploads/video-storage-social.png",
            "https://proto.school/#/tutorials?course=filecoin"
          ],
          "ipfs_tika_version": "dev-build"
        }
    `)

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: 400,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Body: testJSON,
        }).
        Once()

    f := &indexTypes.File{
        Document: indexTypes.Document{
            Size: r.Size,
        },
    }

    err := s.e.Extract(s.ctx, r, &f)

    s.NoError(err)
    s.mockAPIHandler.AssertExpectations(s.T())

    s.Equal(uint64(400), f.Size)
    s.Equal([]interface{}{"How Filecoin Supports Video Storage"}, f.Metadata["title"])
    s.Equal("en", f.Language.Language)
    s.Equal(0.99999505, f.Language.RawScore)
    s.Contains(f.URLs, "https://proto.school/#/tutorials?course=filecoin")
}

func (s TikaTestSuite) TestExtractMaxFileSize() {
    s.cfg.MaxFileSize = 100
    s.e = New(s.cfg, http.DefaultClient, s.protocol, instr.New())

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: uint64(s.cfg.MaxFileSize + 1),
        },
    }

    f := &indexTypes.File{}
    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrFileTooLarge)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func (s TikaTestSuite) TestExtractRequestError() {
    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: 400,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    f := &indexTypes.File{
        Document: indexTypes.Document{
            Size: r.Size,
        },
    }

    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrRequest)
}

func (s TikaTestSuite) TestURLEscape() {
    // Regression test:
    // http://ipfs-tika:8081/ipfs/QmehSxmTPRCr85Xjgzjut6uWQihoTfqg9VVihJ892bmZCp/Killing_Yourself_to_Live:_85%_of_a_True_Story.html
    // panic: creating request: parse http://ipfs-tika:8081/ipfs/QmehSxmTPRCr85Xjgzjut6uWQihoTfqg9VVihJ892bmZCp/Killing_Yourself_to_Live:_85%_of_a_True_Story.html: invalid URL escape "%_o"

    tikaURL := "/ipfs/QmehSxmTPRCr85Xjgzjut6uWQihoTfqg9VVihJ892bmZCp/" + url.PathEscape("Killing_Yourself_to_Live:_85%_of_a_True_Story.html")
    gwURL := "http://localhost:8080" + tikaURL

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
    }

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Body: []byte("{}"),
        }).
        Once()

    f := &indexTypes.File{}

    err := s.e.Extract(s.ctx, r, &f)

    s.NoError(err)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func (s TikaTestSuite) TestTika500() {
    // 500 will just propagate whatever error we're getting from a lower level
    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Status: 500,
            Body:   []byte("{}"),
        }).
        Once()

    f := &indexTypes.File{}

    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrUnexpectedResponse)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func (s TikaTestSuite) TestExtractInvalidJSON() {
    testJSON := []byte(`invalid JSON`)

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: 400,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Body: testJSON,
        }).
        Once()

    f := &indexTypes.File{
        Document: indexTypes.Document{
            Size: r.Size,
        },
    }

    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrUnexpectedResponse)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func TestTikaTestSuite(t *testing.T) {
    suite.Run(t, new(TikaTestSuite))
}
<commit_msg>Remove unused commented code.<commit_after>package tika

import (
    "context"
    "fmt"
    "net/http"
    "net/url"
    "testing"

    "github.com/dankinder/httpmock"
    "github.com/stretchr/testify/mock"
    "github.com/stretchr/testify/suite"

    "github.com/ipfs-search/ipfs-search/components/extractor"
    indexTypes "github.com/ipfs-search/ipfs-search/components/index/types"
    "github.com/ipfs-search/ipfs-search/components/protocol"

    "github.com/ipfs-search/ipfs-search/instr"
    t "github.com/ipfs-search/ipfs-search/types"
)

const testCID = "QmehHHRh1a7u66r7fugebp6f6wGNMGCa7eho9cgjwhAcm2"

type TikaTestSuite struct {
    suite.Suite

    ctx context.Context
    e   extractor.Extractor

    cfg      *Config
    protocol *protocol.Mock

    mockAPIHandler *httpmock.MockHandler
    mockAPIServer  *httpmock.Server
    responseHeader http.Header
}

func (s *TikaTestSuite) SetupTest() {
    s.ctx = context.Background()

    s.mockAPIHandler = &httpmock.MockHandler{}
    s.mockAPIServer = httpmock.NewServer(s.mockAPIHandler)
    s.responseHeader = http.Header{
        "Content-Type": []string{"application/json"},
    }

    s.cfg = DefaultConfig()
    s.cfg.TikaServerURL = s.mockAPIServer.URL()

    s.protocol = &protocol.Mock{}

    s.e = New(s.cfg, http.DefaultClient, s.protocol, instr.New())
}

func (s *TikaTestSuite) TearDownTest() {
    s.mockAPIServer.Close()
}

func (s TikaTestSuite) TestExtract() {
    testJSON := []byte(`
        {
          "metadata": {
            "title": [
              "How Filecoin Supports Video Storage"
            ],
            "Content-Type": [
              "text/html; charset=UTF-8"
            ]
          },
          "content": "The Filecoin Space Race is now live! Learn More\n\t\t Thank you!",
          "language": {
            "language": "en",
            "confidence": "HIGH",
            "rawScore": 0.99999505
          },
          "urls": [
            "https://filecoin.io/uploads/video-storage-social.png",
            "https://proto.school/#/tutorials?course=filecoin"
          ],
          "ipfs_tika_version": "dev-build"
        }
    `)

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: 400,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Body: testJSON,
        }).
        Once()

    f := &indexTypes.File{
        Document: indexTypes.Document{
            Size: r.Size,
        },
    }

    err := s.e.Extract(s.ctx, r, &f)

    s.NoError(err)
    s.mockAPIHandler.AssertExpectations(s.T())

    s.Equal(uint64(400), f.Size)
    s.Equal([]interface{}{"How Filecoin Supports Video Storage"}, f.Metadata["title"])
    s.Equal("en", f.Language.Language)
    s.Equal(0.99999505, f.Language.RawScore)
    s.Contains(f.URLs, "https://proto.school/#/tutorials?course=filecoin")
}

func (s TikaTestSuite) TestExtractMaxFileSize() {
    s.cfg.MaxFileSize = 100
    s.e = New(s.cfg, http.DefaultClient, s.protocol, instr.New())

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: uint64(s.cfg.MaxFileSize + 1),
        },
    }

    f := &indexTypes.File{}
    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrFileTooLarge)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func (s TikaTestSuite) TestExtractRequestError() {
    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: 400,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    f := &indexTypes.File{
        Document: indexTypes.Document{
            Size: r.Size,
        },
    }

    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrRequest)
}

func (s TikaTestSuite) TestURLEscape() {
    // Regression test:
    // http://ipfs-tika:8081/ipfs/QmehSxmTPRCr85Xjgzjut6uWQihoTfqg9VVihJ892bmZCp/Killing_Yourself_to_Live:_85%_of_a_True_Story.html
    // panic: creating request: parse http://ipfs-tika:8081/ipfs/QmehSxmTPRCr85Xjgzjut6uWQihoTfqg9VVihJ892bmZCp/Killing_Yourself_to_Live:_85%_of_a_True_Story.html: invalid URL escape "%_o"

    tikaURL := "/ipfs/QmehSxmTPRCr85Xjgzjut6uWQihoTfqg9VVihJ892bmZCp/" + url.PathEscape("Killing_Yourself_to_Live:_85%_of_a_True_Story.html")
    gwURL := "http://localhost:8080" + tikaURL

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
    }

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Body: []byte("{}"),
        }).
        Once()

    f := &indexTypes.File{}

    err := s.e.Extract(s.ctx, r, &f)

    s.NoError(err)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func (s TikaTestSuite) TestTika500() {
    // 500 will just propagate whatever error we're getting from a lower level
    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Status: 500,
            Body:   []byte("{}"),
        }).
        Once()

    f := &indexTypes.File{}

    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrUnexpectedResponse)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func (s TikaTestSuite) TestExtractInvalidJSON() {
    testJSON := []byte(`invalid JSON`)

    r := &t.AnnotatedResource{
        Resource: &t.Resource{
            Protocol: t.IPFSProtocol,
            ID:       testCID,
        },
        Stat: t.Stat{
            Size: 400,
        },
    }

    tikaURL := fmt.Sprintf("/ipfs/%s", testCID)
    gwURL := "http://localhost:8080" + tikaURL

    s.protocol.
        On("GatewayURL", r).
        Return(gwURL).
        Once()

    s.mockAPIHandler.
        On("Handle", "GET", tikaURL, mock.Anything).
        Return(httpmock.Response{
            Body: testJSON,
        }).
        Once()

    f := &indexTypes.File{
        Document: indexTypes.Document{
            Size: r.Size,
        },
    }

    err := s.e.Extract(s.ctx, r, &f)

    s.Error(err, extractor.ErrUnexpectedResponse)
    s.mockAPIHandler.AssertExpectations(s.T())
}

func TestTikaTestSuite(t *testing.T) {
    suite.Run(t, new(TikaTestSuite))
}
<|endoftext|>