<commit_before>// Package plugin enables plugins to register with Abot and connect to the
// database.
package plugin

import (
	"bufio"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"

	"github.com/dchest/stemmer/porter2"
	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/language"
	_ "github.com/lib/pq" // Import the pq PostgreSQL driver
)

// ErrMissingPluginName is returned when a plugin name is expected, but
// but a blank name is provided.
var ErrMissingPluginName = errors.New("missing plugin name")

// ErrMissingTrigger is returned when a trigger is expected but none
// were found.
var ErrMissingTrigger = errors.New("missing plugin trigger")

// New builds a Plugin with its trigger, RPC, and configuration settings from
// its plugin.json.
func New(url string) (*dt.Plugin, error) {
	if err := core.LoadEnvVars(); err != nil {
		log.Fatal(err)
	}

	// Read plugin.json data from within plugins.go, unmarshal into struct
	p := filepath.Join("plugins.go")
	fi, err := os.OpenFile(p, os.O_RDONLY|os.O_CREATE, 0666)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err = fi.Close(); err != nil {
			log.Info("failed to close file", fi.Name())
			return
		}
	}()
	var found bool
	var data string
	scn := bufio.NewScanner(fi)
	for scn.Scan() {
		t := scn.Text()
		if !found && t != url {
			continue
		} else if t == url {
			found = true
			continue
		} else if len(t) >= 1 && t[0] == '}' {
			data += t
			break
		}
		data += t
	}
	if err = scn.Err(); err != nil {
		return nil, err
	}
	db, err := core.ConnectDB()
	if err != nil {
		return nil, err
	}
	c := dt.PluginConfig{}
	if len(data) > 0 {
		if err = json.Unmarshal([]byte(data), &c); err != nil {
			return nil, err
		}
		if len(c.Name) == 0 {
			return nil, ErrMissingPluginName
		}
	}
	l := log.New(c.Name)
	l.SetDebug(os.Getenv("ABOT_DEBUG") == "true")
	plg := &dt.Plugin{
		Trigger:     &dt.StructuredInput{},
		SetBranches: func(in *dt.Msg) [][]dt.State { return nil },
		Events: &dt.PluginEvents{
			PostReceive:    func(cmd *string) {},
			PreProcessing:  func(cmd *string, u *dt.User) {},
			PostProcessing: func(in *dt.Msg) {},
			PreResponse:    func(in *dt.Msg, resp *string) {},
		},
		Config: c,
		DB:     db,
		Log:    l,
	}
	plg.SM = dt.NewStateMachine(plg)
	return plg, nil
}

// Register enables Abot to notify plugins when specific StructuredInput is
// encountered matching triggers set in the plugins themselves. Note that
// plugins will only listen when (Command and Object) or (Intent) criteria are
// met. There's no support currently for duplicate routes, e.g.
// "find_restaurant" leading to either one of two plugins.
func Register(p *dt.Plugin) error {
	p.Log.Debug("registering", p.Config.Name)
	for _, i := range p.Trigger.Intents {
		s := "__intent_" + strings.ToLower(i)
		oldPlg := core.RegPlugins.Get(s)
		if oldPlg != nil && oldPlg.Config.Name != p.Config.Name {
			p.Log.Infof("found duplicate plugin or trigger %s on %s",
				p.Config.Name, s)
		}
		core.RegPlugins.Set(s, p)
	}
	for _, c := range p.Trigger.Commands {
		for _, o := range p.Trigger.Objects {
			s := strings.ToLower(c + "_" + o)
			oldPlg := core.RegPlugins.Get(s)
			if oldPlg != nil && oldPlg.Config.Name != p.Config.Name {
				p.Log.Info("found duplicate plugin or trigger",
					p.Config.Name, "on", s)
			}
			core.RegPlugins.Set(s, p)
		}
	}

	// registerPlugin is called whenever Keywords or Triggers are changed,
	// but we don't want to append duplicate entries to our
	// core.AllPlugins.
	for _, plg := range core.AllPlugins {
		if plg.Config.Name == p.Config.Name {
			return nil
		}
	}
	core.AllPlugins = append(core.AllPlugins, p)
	p.SM.SetStates(p.States)
	return nil
}

// SetKeywords processes and registers keywords with Abot's core for routing.
func SetKeywords(p *dt.Plugin, khs ...dt.KeywordHandler) {
	k := dt.Keywords{
		Commands: map[string]struct{}{},
		Objects:  map[string]struct{}{},
		Intents:  map[string]struct{}{},
		Dict:     map[string]*dt.KeywordFn{},
	}
	eng := porter2.Stemmer
	for _, kh := range khs {
		for _, intent := range kh.Trigger.Intents {
			k.Dict["I_"+intent] = &kh.Fn
			k.Intents[intent] = struct{}{}
			if !language.Contains(p.Trigger.Intents, intent) {
				p.Trigger.Intents = append(p.Trigger.Intents, intent)
			}
		}
		for _, cmd := range kh.Trigger.Commands {
			k.Dict["C_"+cmd] = &kh.Fn
			cmd = eng.Stem(cmd)
			k.Commands[cmd] = struct{}{}
			if !language.Contains(p.Trigger.Commands, cmd) {
				p.Trigger.Commands = append(p.Trigger.Commands, cmd)
			}
		}
		for _, obj := range kh.Trigger.Objects {
			k.Dict["O_"+obj] = &kh.Fn
			obj = eng.Stem(obj)
			k.Objects[obj] = struct{}{}
			if !language.Contains(p.Trigger.Objects, obj) {
				p.Trigger.Objects = append(p.Trigger.Objects, obj)
			}
		}
	}
	p.Keywords = &k
}

// SetStates is a convenience function provided to match the API of NewKeywords
// and AppendTrigger.
func SetStates(p *dt.Plugin, states [][]dt.State) {
	p.States = states
}

// AppendTrigger appends the StructuredInput's modified contents to a plugin.
// All Commands and Objects stemmed using the Porter2 Snowball algorithm.
func AppendTrigger(p *dt.Plugin, t *dt.StructuredInput) {
	eng := porter2.Stemmer
	for _, cmd := range t.Commands {
		cmd = eng.Stem(cmd)
		if !language.Contains(p.Trigger.Commands, cmd) {
			p.Trigger.Commands = append(p.Trigger.Commands, cmd)
		}
	}
	for _, obj := range t.Objects {
		obj = eng.Stem(obj)
		if !language.Contains(p.Trigger.Objects, obj) {
			p.Trigger.Objects = append(p.Trigger.Objects, obj)
		}
	}
}
<commit_msg>Prevent creation of plugin.go in current dir (#124)<commit_after>// Package plugin enables plugins to register with Abot and connect to the
// database.
package plugin

import (
	"bufio"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"

	"github.com/dchest/stemmer/porter2"
	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/language"
	_ "github.com/lib/pq" // Import the pq PostgreSQL driver
)

// ErrMissingPluginName is returned when a plugin name is expected, but
// but a blank name is provided.
var ErrMissingPluginName = errors.New("missing plugin name")

// ErrMissingTrigger is returned when a trigger is expected but none
// were found.
var ErrMissingTrigger = errors.New("missing plugin trigger")

// New builds a Plugin with its trigger, RPC, and configuration settings from
// its plugin.json.
func New(url string) (*dt.Plugin, error) {
	if err := core.LoadEnvVars(); err != nil {
		log.Fatal(err)
	}

	// Read plugin.json data from within plugins.go, unmarshal into struct
	p := filepath.Join(os.Getenv("ABOT_PATH"), "plugins.go")
	fi, err := os.OpenFile(p, os.O_RDONLY|os.O_CREATE, 0666)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err = fi.Close(); err != nil {
			log.Info("failed to close file", fi.Name())
			return
		}
	}()
	var found bool
	var data string
	scn := bufio.NewScanner(fi)
	for scn.Scan() {
		t := scn.Text()
		if !found && t != url {
			continue
		} else if t == url {
			found = true
			continue
		} else if len(t) >= 1 && t[0] == '}' {
			data += t
			break
		}
		data += t
	}
	if err = scn.Err(); err != nil {
		return nil, err
	}
	db, err := core.ConnectDB()
	if err != nil {
		return nil, err
	}
	c := dt.PluginConfig{}
	if len(data) > 0 {
		if err = json.Unmarshal([]byte(data), &c); err != nil {
			return nil, err
		}
		if len(c.Name) == 0 {
			return nil, ErrMissingPluginName
		}
	}
	l := log.New(c.Name)
	l.SetDebug(os.Getenv("ABOT_DEBUG") == "true")
	plg := &dt.Plugin{
		Trigger:     &dt.StructuredInput{},
		SetBranches: func(in *dt.Msg) [][]dt.State { return nil },
		Events: &dt.PluginEvents{
			PostReceive:    func(cmd *string) {},
			PreProcessing:  func(cmd *string, u *dt.User) {},
			PostProcessing: func(in *dt.Msg) {},
			PreResponse:    func(in *dt.Msg, resp *string) {},
		},
		Config: c,
		DB:     db,
		Log:    l,
	}
	plg.SM = dt.NewStateMachine(plg)
	return plg, nil
}

// Register enables Abot to notify plugins when specific StructuredInput is
// encountered matching triggers set in the plugins themselves. Note that
// plugins will only listen when (Command and Object) or (Intent) criteria are
// met. There's no support currently for duplicate routes, e.g.
// "find_restaurant" leading to either one of two plugins.
func Register(p *dt.Plugin) error {
	p.Log.Debug("registering", p.Config.Name)
	for _, i := range p.Trigger.Intents {
		s := "__intent_" + strings.ToLower(i)
		oldPlg := core.RegPlugins.Get(s)
		if oldPlg != nil && oldPlg.Config.Name != p.Config.Name {
			p.Log.Infof("found duplicate plugin or trigger %s on %s",
				p.Config.Name, s)
		}
		core.RegPlugins.Set(s, p)
	}
	for _, c := range p.Trigger.Commands {
		for _, o := range p.Trigger.Objects {
			s := strings.ToLower(c + "_" + o)
			oldPlg := core.RegPlugins.Get(s)
			if oldPlg != nil && oldPlg.Config.Name != p.Config.Name {
				p.Log.Info("found duplicate plugin or trigger",
					p.Config.Name, "on", s)
			}
			core.RegPlugins.Set(s, p)
		}
	}

	// registerPlugin is called whenever Keywords or Triggers are changed,
	// but we don't want to append duplicate entries to our
	// core.AllPlugins.
	for _, plg := range core.AllPlugins {
		if plg.Config.Name == p.Config.Name {
			return nil
		}
	}
	core.AllPlugins = append(core.AllPlugins, p)
	p.SM.SetStates(p.States)
	return nil
}

// SetKeywords processes and registers keywords with Abot's core for routing.
func SetKeywords(p *dt.Plugin, khs ...dt.KeywordHandler) {
	k := dt.Keywords{
		Commands: map[string]struct{}{},
		Objects:  map[string]struct{}{},
		Intents:  map[string]struct{}{},
		Dict:     map[string]*dt.KeywordFn{},
	}
	eng := porter2.Stemmer
	for _, kh := range khs {
		for _, intent := range kh.Trigger.Intents {
			k.Dict["I_"+intent] = &kh.Fn
			k.Intents[intent] = struct{}{}
			if !language.Contains(p.Trigger.Intents, intent) {
				p.Trigger.Intents = append(p.Trigger.Intents, intent)
			}
		}
		for _, cmd := range kh.Trigger.Commands {
			k.Dict["C_"+cmd] = &kh.Fn
			cmd = eng.Stem(cmd)
			k.Commands[cmd] = struct{}{}
			if !language.Contains(p.Trigger.Commands, cmd) {
				p.Trigger.Commands = append(p.Trigger.Commands, cmd)
			}
		}
		for _, obj := range kh.Trigger.Objects {
			k.Dict["O_"+obj] = &kh.Fn
			obj = eng.Stem(obj)
			k.Objects[obj] = struct{}{}
			if !language.Contains(p.Trigger.Objects, obj) {
				p.Trigger.Objects = append(p.Trigger.Objects, obj)
			}
		}
	}
	p.Keywords = &k
}

// SetStates is a convenience function provided to match the API of NewKeywords
// and AppendTrigger.
func SetStates(p *dt.Plugin, states [][]dt.State) {
	p.States = states
}

// AppendTrigger appends the StructuredInput's modified contents to a plugin.
// All Commands and Objects stemmed using the Porter2 Snowball algorithm.
func AppendTrigger(p *dt.Plugin, t *dt.StructuredInput) {
	eng := porter2.Stemmer
	for _, cmd := range t.Commands {
		cmd = eng.Stem(cmd)
		if !language.Contains(p.Trigger.Commands, cmd) {
			p.Trigger.Commands = append(p.Trigger.Commands, cmd)
		}
	}
	for _, obj := range t.Objects {
		obj = eng.Stem(obj)
		if !language.Contains(p.Trigger.Objects, obj) {
			p.Trigger.Objects = append(p.Trigger.Objects, obj)
		}
	}
}
<|endoftext|>


<commit_before>package goshare

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"time"

	golerror "github.com/abhishekkr/gol/golerror"
	golkeyval "github.com/abhishekkr/gol/golkeyval"
	gollist "github.com/abhishekkr/gol/gollist"
	gollog "github.com/abhishekkr/gol/gollog"
)

var (
	db      golkeyval.DBEngine
	logInfo Log
)

/* banner just brand print */
func banner() {
	fmt.Println("**************************************************")
	fmt.Println("  ___  ____      ___        __   _   __")
	fmt.Println("  |    |  |      |    |  | /  \\ | ) |")
	fmt.Println("  | =| |  |  =~  |==| |==| |==| |=  |=")
	fmt.Println("  |__| |__|      ___| |  | |  | | \\ |__")
	fmt.Println("")
	fmt.Println("**************************************************")
}

/*
DoYouWannaContinue checking if you still wanna keep the goshare up.
*/
func DoYouWannaContinue() {
	var input string
	for {
		fmt.Println("Do you wanna exit. (yes|no):\n\n")

		fmt.Scanf("%s", &input)

		if input == "yes" || input == "y" {
			break
		}
	}
}

/*
goshareDB returns golkeyval DBEngine for it.
*/
func goshareDB(config Config) golkeyval.DBEngine {
	db := golkeyval.GetDBEngine(config["DBEngine"])
	db.Configure(config)
	db.CreateDB()
	return db
}

/*
GoShareEngine putting together base engine for GoShare as per config.
dbpath, server_uri, httpport, rep_port, *string
*/
func GoShareEngine(config Config) {
	runtime.GOMAXPROCS(runtime.NumCPU())

	// remember it will be same DB instance shared across goshare package
	db = goshareDB(config)
	logInfo = gollog.Log{Level: "info", Thread: make(chan string)}
	go logInfo.LogIt()

	if config["cpuprofile"] != "" {
		f, err := os.Create(config["cpuprofile"])
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		go func() {
			time.Sleep(100 * time.Second)
			pprof.StopCPUProfile()
		}()
	}

	_httpPort, errHTTPPort := strconv.Atoi(config["http-port"])
	_replyPorts, errReplyPorts := gollist.CSVToNumbers(config["rep-ports"])
	if errHTTPPort == nil && errReplyPorts == nil {
		go GoShareHTTP(config["server-uri"], _httpPort)
		go GoShareZMQ(config["server-uri"], _replyPorts)
	} else {
		golerror.Boohoo("Port parameters to bind, error-ed while conversion to number.", true)
	}
}

/*
GoShare is daddy-o of goshare instance.
*/
func GoShare() {
	banner()
	runtime.GOMAXPROCS(runtime.NumCPU())
	GoShareEngine(ConfigFromFlags())
	DoYouWannaContinue()
}
<commit_msg>[bug] Log is shifted to gollog.Log<commit_after>package goshare

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"time"

	golerror "github.com/abhishekkr/gol/golerror"
	golkeyval "github.com/abhishekkr/gol/golkeyval"
	gollist "github.com/abhishekkr/gol/gollist"
	gollog "github.com/abhishekkr/gol/gollog"
)

var (
	db      golkeyval.DBEngine
	logInfo gollog.Log
)

/* banner just brand print */
func banner() {
	fmt.Println("**************************************************")
	fmt.Println("  ___  ____      ___        __   _   __")
	fmt.Println("  |    |  |      |    |  | /  \\ | ) |")
	fmt.Println("  | =| |  |  =~  |==| |==| |==| |=  |=")
	fmt.Println("  |__| |__|      ___| |  | |  | | \\ |__")
	fmt.Println("")
	fmt.Println("**************************************************")
}

/*
DoYouWannaContinue checking if you still wanna keep the goshare up.
*/
func DoYouWannaContinue() {
	var input string
	for {
		fmt.Println("Do you wanna exit. (yes|no):\n\n")

		fmt.Scanf("%s", &input)

		if input == "yes" || input == "y" {
			break
		}
	}
}

/*
goshareDB returns golkeyval DBEngine for it.
*/
func goshareDB(config Config) golkeyval.DBEngine {
	db := golkeyval.GetDBEngine(config["DBEngine"])
	db.Configure(config)
	db.CreateDB()
	return db
}

/*
GoShareEngine putting together base engine for GoShare as per config.
dbpath, server_uri, httpport, rep_port, *string
*/
func GoShareEngine(config Config) {
	runtime.GOMAXPROCS(runtime.NumCPU())

	// remember it will be same DB instance shared across goshare package
	db = goshareDB(config)
	logInfo = gollog.Log{Level: "info", Thread: make(chan string)}
	go logInfo.LogIt()

	if config["cpuprofile"] != "" {
		f, err := os.Create(config["cpuprofile"])
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		go func() {
			time.Sleep(100 * time.Second)
			pprof.StopCPUProfile()
		}()
	}

	_httpPort, errHTTPPort := strconv.Atoi(config["http-port"])
	_replyPorts, errReplyPorts := gollist.CSVToNumbers(config["rep-ports"])
	if errHTTPPort == nil && errReplyPorts == nil {
		go GoShareHTTP(config["server-uri"], _httpPort)
		go GoShareZMQ(config["server-uri"], _replyPorts)
	} else {
		golerror.Boohoo("Port parameters to bind, error-ed while conversion to number.", true)
	}
}

/*
GoShare is daddy-o of goshare instance.
*/
func GoShare() {
	banner()
	runtime.GOMAXPROCS(runtime.NumCPU())
	GoShareEngine(ConfigFromFlags())
	DoYouWannaContinue()
}
<|endoftext|>


<commit_before>package server

import (
	_ "dockerstack/dockerstack-agent/config"
	"encoding/json"
	"fmt"
	"github.com/shirou/gopsutil/mem"
	_ "github.com/shirou/gopsutil/process"
	_ "dockerstack/dockerstack-agent/logging"
	_ "os"
	"os/exec"
)

type ServerData struct {
	Apikey string `json:"apikey"`
	Data   struct {
		Available int `json:"available"`
		Total     int `json:"total"`
		Used      int `json:"used"`
	} `json:"data"`
	Serverip string `json:"serverip"`
}

type ProcessData struct {
	Apikey string `json:"apikey"`
	Data   struct {
		Process []struct {
			Name string `json:"name"`
			Path string `json:"path"`
			Size string `json:"size"`
		} `json:"process"`
	} `json:"data"`
	Serverip string `json:"serverip"`
}

type ProcesPids struct {
	Process []struct {
		Pid int `json:"pid"`
	} `json:"process"`
}

func ServerMemory() (*ServerData, error) {

	data := &ServerData{}
	v, _ := mem.VirtualMemory()

	data.Data.Total = int(v.Total / 1024)
	data.Data.Available = int(v.Available / 1024)
	data.Data.Used = int(v.Used / 1024)
	m, _ := json.Marshal(data)

	fmt.Println(string(m))

	return data, nil
}

func ProcessList() ([]string, error) {

	pids,_ := getPids()

	fmt.Print(pids)

	// var proc []string

	// for _, pid := range pids {

		// fmt.Println(string(pid))
		// 	p, _ := process.NewProcess(pid)

		// 	// data := &ProcessData{}

		// 	fmt.Println(p.Name())
		// 	fmt.Println(p.Exe())
		// 	fmt.Println(p.MemoryInfo())

		// 	// proc[0] = p.Name()
		// 	// proc[1] = p.Exe()
		// 	// proc[2] = p.MemoryInfo()

	// }

	return nil, nil

}

func DiskSpace() {

}

//Internal Functions

//Gets Top 5 Process Id's
func getPids() (*ProcesPids, error) {

	pids := &ProcesPids{}
	cmd := "ps aux | awk '{print $2, $4, $11}' | sort -k2rn | head -n 5|awk '{print $1}'"

	out, err := exec.Command("bash", "-c", cmd).Output()
	if err != nil {
		fmt.Println(err.Error())
	}

// 	for _, dat := range pids.Process {
//
// 		for _, proc := range out {
//
// 			dat.Pid = string(proc)
// 		}
// 	}
//
	fmt.Print(string(out))

	return out

}
<commit_msg>Getting top 5 process running with Server Memory<commit_after>package server

import (
	"bufio"
	"dockerstack/dockerstack-agent/config"
	_ "dockerstack/dockerstack-agent/logging"
	"fmt"
	ps "github.com/mitchellh/go-ps"
	"github.com/shirou/gopsutil/disk"
	"github.com/shirou/gopsutil/mem"
	_ "github.com/shirou/gopsutil/process"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	_ "reflect"
	_ "regexp"
	"strconv"
)

type ServerData struct {
	Apikey string `json:"apikey"`
	Data   struct {
		Available int `json:"available"`
		Total     int `json:"total"`
		Used      int `json:"used"`
	} `json:"data"`
	Serverip string `json:"serverip"`
}

/*
Sample output
{
  "apikey": 123456,
  "serverip" : "172.27.9.56",
  "data":[{
    "name":"docker",
    "pid": 925
  },
  {
    "name": "apache2",
    "pid":1024
  }]
}
*/

type ProcessData struct {
	Apikey   string `json:"apikey"`
	Data     []Process
	Serverip string `json:"serverip"`
}

type Process struct {
	Name string `json:"name"`
	Pid  int    `json:"pid"`
}

func ServerMemory() (*ServerData, error) {

	data := &ServerData{}
	v, _ := mem.VirtualMemory()

	ief, _ := net.InterfaceByName(config.Main("interface"))

	addrs, err := ief.Addrs()
	if err != nil {
		err.Error()
	}

	data.Data.Total = int(v.Total / 1024)
	data.Data.Available = int(v.Available / 1024)
	data.Data.Used = int(v.Used / 1024)
	data.Serverip = addrs[0].String()
	data.Apikey = config.ApiKey()

	fmt.Println(ProcessList())
	return data, nil
}

//Lists Top 5 ProcessPids with Names running on the Host
func ProcessList() (*ProcessData, error) {

	k := getPids()

	fmt.Println(k)

	pids, _ := ps.Processes()

	data := new(ProcessData)

	ief, _ := net.InterfaceByName(config.Main("interface"))

	addrs, err := ief.Addrs()
	if err != nil {
		err.Error()
	}

	data.Apikey = config.ApiKey()
	data.Serverip = addrs[0].String()

	file, _ := os.Open("/tmp/process.txt")

	reader := bufio.NewReader(file)

	scanner := bufio.NewScanner(reader)

	for scanner.Scan() {

		for _, pid := range pids {

			if scanner.Text() == strconv.Itoa(pid.Pid()) {

				data.Data = append(data.Data, Process{Name: pid.Executable(), Pid: pid.Pid()})

			}
		}

	}

	return data, nil

}

func DiskSpace() (*disk.DiskUsageStat, error) {

	data, _ := disk.DiskUsage("/")

	return data, nil

}

//Internal Functions

//Gets Top 5 Process Id's
func getPids() string {

	//pids := &ProcesPids{}
	cmd := "ps aux | awk '{print $2, $4, $11}' | sort -k2rn | head -n 5|awk '{print $1}'"

	out, err := exec.Command("bash", "-c", cmd).Output()
	if err != nil {
		fmt.Println(err.Error())
	}

	er := ioutil.WriteFile("/tmp/process.txt", out, 0644)
	if er != nil {
		fmt.Print(er.Error())
	}

	return string(out)

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/manifest/schema1"
	humanize "github.com/dustin/go-humanize"
	"github.com/jessfraz/reg/clair"
	"github.com/jessfraz/reg/registry"
	"github.com/jessfraz/reg/utils"
	"github.com/urfave/cli"
)

const (
	// VERSION is the binary version.
	VERSION = "v0.1.0"

	dockerConfigPath = ".docker/config.json"
)

var (
	updating = false
	wg       sync.WaitGroup
)

// preload initializes any global options and configuration
// before the main or sub commands are run.
func preload(c *cli.Context) (err error) {
	if c.GlobalBool("debug") {
		logrus.SetLevel(logrus.DebugLevel)
	}

	return nil
}

func moveFile(src, dest string) (err error) {
	srcfile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcfile.Close()
	defer os.Remove(src)

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer destfile.Close()

	if _, err := io.Copy(destfile, srcfile); err != nil {
		return err
	}

	srcinfo, err := os.Stat(src)
	if err != nil {
		return err
	}

	return os.Chmod(dest, srcinfo.Mode())
}

func main() {
	app := cli.NewApp()
	app.Name = "reg-server"
	app.Version = VERSION
	app.Author = "@jessfraz"
	app.Email = "no-reply@butts.com"
	app.Usage = "Docker registry v2 static UI server."
	app.Before = preload
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "run in debug mode",
		},
		cli.StringFlag{
			Name:  "username, u",
			Usage: "username for the registry",
		},
		cli.StringFlag{
			Name:  "password, p",
			Usage: "password for the registry",
		},
		cli.StringFlag{
			Name:  "registry, r",
			Usage: "URL to the private registry (ex. r.j3ss.co)",
		},
		cli.StringFlag{
			Name:  "port",
			Value: "8080",
			Usage: "port for server to run on",
		},
		cli.StringFlag{
			Name:  "cert",
			Usage: "path to ssl cert",
		},
		cli.StringFlag{
			Name:  "key",
			Usage: "path to ssl key",
		},
		cli.StringFlag{
			Name:  "interval",
			Value: "5m",
			Usage: "interval to generate new index.html's at",
		},
		cli.StringFlag{
			Name:  "clair",
			Usage: "url to clair instance",
		},
	}
	app.Action = func(c *cli.Context) error {
		auth, err := utils.GetAuthConfig(c)
		if err != nil {
			return err
		}

		// create the registry client
		r, err := registry.New(auth, c.GlobalBool("debug"))
		if err != nil {
			return err
		}

		// get the path to the static directory
		wd, err := os.Getwd()
		if err != nil {
			return err
		}
		staticDir := filepath.Join(wd, "static")

		// create the initial index
		if err := createStaticIndex(r, staticDir, c.GlobalString("clair")); err != nil {
			return cli.NewExitError(fmt.Sprintf("Error creating index: %s", err.Error()), 1)
		}

		// parse the duration
		dur, err := time.ParseDuration(c.String("interval"))
		if err != nil {
			logrus.Fatalf("parsing %s as duration failed: %v", c.String("interval"), err)
		}
		ticker := time.NewTicker(dur)

		go func() {
			// create more indexes every X minutes based off interval
			for range ticker.C {
				if !updating {
					if err := createStaticIndex(r, staticDir, c.GlobalString("clair")); err != nil {
						logrus.Warnf("creating static index failed: %v", err)
						wg.Wait()
						updating = false
					}
					if c.GlobalString("clair") != "" {
						wg.Wait()
					}
				}
			}
		}()

		// create mux server
		mux := http.NewServeMux()

		// static files handler
		staticHandler := http.FileServer(http.Dir(staticDir))
		mux.Handle("/", staticHandler)

		// set up the server
		port := c.String("port")
		server := &http.Server{
			Addr:    ":" + port,
			Handler: mux,
		}
		logrus.Infof("Starting server on port %q", port)
		if c.String("cert") != "" && c.String("key") != "" {
			logrus.Fatal(server.ListenAndServeTLS(c.String("cert"), c.String("key")))
		} else {
			logrus.Fatal(server.ListenAndServe())
		}

		return nil
	}

	app.Run(os.Args)
}

type data struct {
	RegistryURL string
	LastUpdated string
	Repos       []repository
}

type repository struct {
	Name        string
	Tag         string
	RepoURI     string
	CreatedDate string
	VulnURI     string
}

type v1Compatibility struct {
	ID      string    `json:"id"`
	Created time.Time `json:"created"`
}

func createStaticIndex(r *registry.Registry, staticDir, clairURI string) error {
	updating = true
	logrus.Info("fetching catalog")
	repoList, err := r.Catalog("")
	if err != nil {
		return fmt.Errorf("getting catalog failed: %v", err)
	}

	logrus.Info("fetching tags")
	var repos []repository
	for i, repo := range repoList {
		// get the tags
		tags, err := r.Tags(repo)
		if err != nil {
			return fmt.Errorf("getting tags for %s failed: %v", repo, err)
		}
		for j, tag := range tags {
			// get the manifest

			m1, err := r.ManifestV1(repo, tag)
			if err != nil {
				logrus.Warnf("getting v1 manifest for %s:%s failed: %v", repo, tag, err)
			}

			var createdDate string
			for _, h := range m1.History {
				var comp v1Compatibility
				if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil {
					return fmt.Errorf("unmarshal v1compatibility failed: %v", err)
				}
				createdDate = humanize.Time(comp.Created)
				break
			}

			repoURI := fmt.Sprintf("%s/%s", r.Domain, repo)
			if tag != "latest" {
				repoURI += ":" + tag
			}

			newrepo := repository{
				Name:        repo,
				Tag:         tag,
				RepoURI:     repoURI,
				CreatedDate: createdDate,
			}

			if clairURI != "" {
				wg.Add(1)

				go func(repo, tag string, i, j int) {
					defer wg.Done()

					throttle := time.Tick(time.Duration(time.Duration((i+1)*(j+1)*3) * time.Second))
					<-throttle

					logrus.Infof("creating vulns.txt for %s:%s", repo, tag)

					if err := createVulnStaticPage(r, staticDir, clairURI, repo, tag, m1); err != nil {
						// return fmt.Errorf("creating vuln static page for %s:%s failed: %v", repo, tag, err)
						logrus.Warnf("creating vuln static page for %s:%s failed: %v", repo, tag, err)
					}
				}(repo, tag, i, j)

				newrepo.VulnURI = filepath.Join(repo, tag, "vulns.txt")
			}
			repos = append(repos, newrepo)
		}
	}

	// create temporoary file to save template to
	logrus.Info("creating temporary file for template")
	f, err := ioutil.TempFile("", "reg-server")
	if err != nil {
		return fmt.Errorf("creating temp file failed: %v", err)
	}
	defer f.Close()
	defer os.Remove(f.Name())

	// parse & execute the template
	logrus.Info("parsing and executing the template")
	templateDir := filepath.Join(staticDir, "../templates")
	lp := filepath.Join(templateDir, "layout.html")

	d := data{
		RegistryURL: r.Domain,
		Repos:       repos,
		LastUpdated: time.Now().Local().Format(time.RFC1123),
	}
	tmpl := template.Must(template.New("").ParseFiles(lp))
	if err := tmpl.ExecuteTemplate(f, "layout", d); err != nil {
		return fmt.Errorf("execute template failed: %v", err)
	}
	f.Close()

	index := filepath.Join(staticDir, "index.html")
	logrus.Infof("renaming the temporary file %s to %s", f.Name(), index)
	if err := moveFile(f.Name(), index); err != nil {
		return fmt.Errorf("renaming result from %s to %s failed: %v", f.Name(), index, err)
	}
	updating = false
	return nil
}

func createVulnStaticPage(r *registry.Registry, staticDir, clairURI, repo, tag string, m schema1.SignedManifest) error {
	// filter out the empty layers
	var filteredLayers []schema1.FSLayer
	for _, layer := range m.FSLayers {
		if layer.BlobSum != clair.EmptyLayerBlobSum {
			filteredLayers = append(filteredLayers, layer)
		}
	}
	m.FSLayers = filteredLayers
	if len(m.FSLayers) == 0 {
		fmt.Printf("No need to analyse image %s:%s as there is no non-emtpy layer", repo, tag)
		return nil
	}

	// initialize clair
	cr, err := clair.New(clairURI, false)
	if err != nil {
		return err
	}

	for i := len(m.FSLayers) - 1; i >= 0; i-- {
		// form the clair layer
		l, err := utils.NewClairLayer(r, repo, m.FSLayers, i)
		if err != nil {
			return err
		}

		// post the layer
		if _, err := cr.PostLayer(l); err != nil {
			return err
		}
	}

	vl, err := cr.GetLayer(m.FSLayers[0].BlobSum.String(), false, true)
	if err != nil {
		return err
	}

	// get the vulns
	var vulns []clair.Vulnerability
	for _, f := range vl.Features {
		for _, v := range f.Vulnerabilities {
			vulns = append(vulns, v)
		}
	}

	path := filepath.Join(staticDir, repo, tag, "vulns.txt")
	if err := os.MkdirAll(filepath.Dir(path), 0644); err != nil {
		return err
	}
	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer file.Close()

	fmt.Fprintf(file, "Found %d vulnerabilities \n", len(vulns))
	fmt.Fprintln(file, "")

	vulnsBy := func(sev string, store map[string][]clair.Vulnerability) []clair.Vulnerability {
		items, found := store[sev]
		if !found {
			items = make([]clair.Vulnerability, 0)
			store[sev] = items
		}
		return items
	}

	// group by severity
	store := make(map[string][]clair.Vulnerability)
	for _, v := range vulns {
		sevRow := vulnsBy(v.Severity, store)
		store[v.Severity] = append(sevRow, v)
	}

	// iterate over the priorities list
	iteratePriorities := func(f func(sev string)) {
		for _, sev := range clair.Priorities {
			if len(store[sev]) != 0 {
				f(sev)
			}
		}
	}

	iteratePriorities(func(sev string) {
		fmt.Fprintf(file, "%s: %d\n", sev, len(store[sev]))
	})
	fmt.Fprintln(file, "")

	// return an error if there are more than 10 bad vulns
	lenBadVulns := len(store["High"]) + len(store["Critical"]) + len(store["Defcon1"])
	if lenBadVulns > 10 {
		fmt.Fprintln(file, "--------------- ALERT ---------------")
		fmt.Fprintf(file, "%d bad vunerabilities found", lenBadVulns)
	}
	fmt.Fprintln(file, "")

	iteratePriorities(func(sev string) {
		for _, v := range store[sev] {
			fmt.Fprintf(file, "%s: [%s] \n%s\n%s\n", v.Name, v.Severity, v.Description, v.Link)
			fmt.Fprintln(file, "-----------------------------------------")
		}
	})

	return nil
}
<commit_msg>cleanup<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/manifest/schema1"
	humanize "github.com/dustin/go-humanize"
	"github.com/jessfraz/reg/clair"
	"github.com/jessfraz/reg/registry"
	"github.com/jessfraz/reg/utils"
	"github.com/urfave/cli"
)

const (
	// VERSION is the binary version.
	VERSION = "v0.1.0"

	dockerConfigPath = ".docker/config.json"
)

var (
	updating = false
	wg       sync.WaitGroup
)

// preload initializes any global options and configuration
// before the main or sub commands are run.
func preload(c *cli.Context) (err error) {
	if c.GlobalBool("debug") {
		logrus.SetLevel(logrus.DebugLevel)
	}

	return nil
}

func moveFile(src, dest string) (err error) {
	srcfile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcfile.Close()
	defer os.Remove(src)

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer destfile.Close()

	if _, err := io.Copy(destfile, srcfile); err != nil {
		return err
	}

	srcinfo, err := os.Stat(src)
	if err != nil {
		return err
	}

	return os.Chmod(dest, srcinfo.Mode())
}

func main() {
	app := cli.NewApp()
	app.Name = "reg-server"
	app.Version = VERSION
	app.Author = "@jessfraz"
	app.Email = "no-reply@butts.com"
	app.Usage = "Docker registry v2 static UI server."
	app.Before = preload
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "run in debug mode",
		},
		cli.StringFlag{
			Name:  "username, u",
			Usage: "username for the registry",
		},
		cli.StringFlag{
			Name:  "password, p",
			Usage: "password for the registry",
		},
		cli.StringFlag{
			Name:  "registry, r",
			Usage: "URL to the private registry (ex. r.j3ss.co)",
		},
		cli.StringFlag{
			Name:  "port",
			Value: "8080",
			Usage: "port for server to run on",
		},
		cli.StringFlag{
			Name:  "cert",
			Usage: "path to ssl cert",
		},
		cli.StringFlag{
			Name:  "key",
			Usage: "path to ssl key",
		},
		cli.StringFlag{
			Name:  "interval",
			Value: "5m",
			Usage: "interval to generate new index.html's at",
		},
		cli.StringFlag{
			Name:  "clair",
			Usage: "url to clair instance",
		},
	}
	app.Action = func(c *cli.Context) error {
		auth, err := utils.GetAuthConfig(c)
		if err != nil {
			return err
		}

		// create the registry client
		r, err := registry.New(auth, c.GlobalBool("debug"))
		if err != nil {
			return err
		}

		// get the path to the static directory
		wd, err := os.Getwd()
		if err != nil {
			return err
		}
		staticDir := filepath.Join(wd, "static")

		// create the initial index
		if err := createStaticIndex(r, staticDir, c.GlobalString("clair")); err != nil {
			return cli.NewExitError(fmt.Sprintf("Error creating index: %s", err.Error()), 1)
		}

		// parse the duration
		dur, err := time.ParseDuration(c.String("interval"))
		if err != nil {
			logrus.Fatalf("parsing %s as duration failed: %v", c.String("interval"), err)
		}
		ticker := time.NewTicker(dur)

		go func() {
			// create more indexes every X minutes based off interval
			for range ticker.C {
				if !updating {
					if err := createStaticIndex(r, staticDir, c.GlobalString("clair")); err != nil {
						logrus.Warnf("creating static index failed: %v", err)
						wg.Wait()
						updating = false
					}
					wg.Wait()
				}
			}
		}()

		// create mux server
		mux := http.NewServeMux()

		// static files handler
		staticHandler := http.FileServer(http.Dir(staticDir))
		mux.Handle("/", staticHandler)

		// set up the server
		port := c.String("port")
		server := &http.Server{
			Addr:    ":" + port,
			Handler: mux,
		}
		logrus.Infof("Starting server on port %q", port)
		if c.String("cert") != "" && c.String("key") != "" {
			logrus.Fatal(server.ListenAndServeTLS(c.String("cert"), c.String("key")))
		} else {
			logrus.Fatal(server.ListenAndServe())
		}

		return nil
	}

	app.Run(os.Args)
}

type data struct {
	RegistryURL string
	LastUpdated string
	Repos       []repository
}

type repository struct {
	Name        string
	Tag         string
	RepoURI     string
	CreatedDate string
	VulnURI     string
}

type v1Compatibility struct {
	ID      string    `json:"id"`
	Created time.Time `json:"created"`
}

func createStaticIndex(r *registry.Registry, staticDir, clairURI string) error {
	updating = true
	logrus.Info("fetching catalog")
	repoList, err := r.Catalog("")
	if err != nil {
		return fmt.Errorf("getting catalog failed: %v", err)
	}

	logrus.Info("fetching tags")
	var repos []repository
	for i, repo := range repoList {
		// get the tags
		tags, err := r.Tags(repo)
		if err != nil {
			return fmt.Errorf("getting tags for %s failed: %v", repo, err)
		}
		for j, tag := range tags {
			// get the manifest

			m1, err := r.ManifestV1(repo, tag)
			if err != nil {
				logrus.Warnf("getting v1 manifest for %s:%s failed: %v", repo, tag, err)
			}

			var createdDate string
			for _, h := range m1.History {
				var comp v1Compatibility
				if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil {
					return fmt.Errorf("unmarshal v1compatibility failed: %v", err)
				}
				createdDate = humanize.Time(comp.Created)
				break
			}

			repoURI := fmt.Sprintf("%s/%s", r.Domain, repo)
			if tag != "latest" {
				repoURI += ":" + tag
			}

			newrepo := repository{
				Name:        repo,
				Tag:         tag,
				RepoURI:     repoURI,
				CreatedDate: createdDate,
			}

			if clairURI != "" {
				wg.Add(1)

				go func(repo, tag string, i, j int) {
					defer wg.Done()

					throttle := time.Tick(time.Duration(time.Duration((i+1)*(j+1)*3) * time.Second))
					<-throttle

					logrus.Infof("creating vulns.txt for %s:%s", repo, tag)

					if err := createVulnStaticPage(r, staticDir, clairURI, repo, tag, m1); err != nil {
						// return fmt.Errorf("creating vuln static page for %s:%s failed: %v", repo, tag, err)
						logrus.Warnf("creating vuln static page for %s:%s failed: %v", repo, tag, err)
					}
				}(repo, tag, i, j)

				newrepo.VulnURI = filepath.Join(repo, tag, "vulns.txt")
			}
			repos = append(repos, newrepo)
		}
	}

	// create temporoary file to save template to
	logrus.Info("creating temporary file for template")
	f, err := ioutil.TempFile("", "reg-server")
	if err != nil {
		return fmt.Errorf("creating temp file failed: %v", err)
	}
	defer f.Close()
	defer os.Remove(f.Name())

	// parse & execute the template
	logrus.Info("parsing and executing the template")
	templateDir := filepath.Join(staticDir, "../templates")
	lp := filepath.Join(templateDir, "layout.html")

	d := data{
		RegistryURL: r.Domain,
		Repos:       repos,
		LastUpdated: time.Now().Local().Format(time.RFC1123),
	}
	tmpl := template.Must(template.New("").ParseFiles(lp))
	if err := tmpl.ExecuteTemplate(f, "layout", d); err != nil {
		return fmt.Errorf("execute template failed: %v", err)
	}
	f.Close()

	index := filepath.Join(staticDir, "index.html")
	logrus.Infof("renaming the temporary file %s to %s", f.Name(), index)
	if err := moveFile(f.Name(), index); err != nil {
		return fmt.Errorf("renaming result from %s to %s failed: %v", f.Name(), index, err)
	}
	updating = false
	return nil
}

func createVulnStaticPage(r *registry.Registry, staticDir, clairURI, repo, tag string, m schema1.SignedManifest) error {
	// filter out the empty layers
	var filteredLayers []schema1.FSLayer
	for _, layer := range m.FSLayers {
		if layer.BlobSum != clair.EmptyLayerBlobSum {
			filteredLayers = append(filteredLayers, layer)
		}
	}
	m.FSLayers = filteredLayers
	if len(m.FSLayers) == 0 {
		fmt.Printf("No need to analyse image %s:%s as there is no non-emtpy layer", repo, tag)
		return nil
	}

	// initialize clair
	cr, err := clair.New(clairURI, false)
	if err != nil {
		return err
	}

	for i := len(m.FSLayers) - 1; i >= 0; i-- {
		// form the clair layer
		l, err := utils.NewClairLayer(r, repo, m.FSLayers, i)
		if err != nil {
			return err
		}

		// post the layer
		if _, err := cr.PostLayer(l); err != nil {
			return err
		}
	}

	vl, err := cr.GetLayer(m.FSLayers[0].BlobSum.String(), false, true)
	if err != nil {
		return err
	}

	// get the vulns
	var vulns []clair.Vulnerability
	for _, f := range vl.Features {
		for _, v := range f.Vulnerabilities {
			vulns = append(vulns, v)
		}
	}

	path := filepath.Join(staticDir, repo, tag, "vulns.txt")
	if err := os.MkdirAll(filepath.Dir(path), 0644); err != nil {
		return err
	}
	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer file.Close()

	fmt.Fprintf(file, "Found %d vulnerabilities \n", len(vulns))
	fmt.Fprintln(file, "")

	vulnsBy := func(sev string, store map[string][]clair.Vulnerability) []clair.Vulnerability {
		items, found := store[sev]
		if !found {
			items = make([]clair.Vulnerability, 0)
			store[sev] = items
		}
		return items
	}

	// group by severity
	store := make(map[string][]clair.Vulnerability)
	for _, v := range vulns {
		sevRow := vulnsBy(v.Severity, store)
		store[v.Severity] = append(sevRow, v)
	}

	// iterate over the priorities list
	iteratePriorities := func(f func(sev string)) {
		for _, sev := range clair.Priorities {
			if len(store[sev]) != 0 {
				f(sev)
			}
		}
	}

	iteratePriorities(func(sev string) {
		fmt.Fprintf(file, "%s: %d\n", sev, len(store[sev]))
	})
	fmt.Fprintln(file, "")

	// return an error if there are more than 10 bad vulns
	lenBadVulns := len(store["High"]) + len(store["Critical"]) + len(store["Defcon1"])
	if lenBadVulns > 10 {
		fmt.Fprintln(file, "--------------- ALERT ---------------")
		fmt.Fprintf(file, "%d bad vunerabilities found", lenBadVulns)
	}
	fmt.Fprintln(file, "")

	iteratePriorities(func(sev string) {
		for _, v := range store[sev] {
			fmt.Fprintf(file, "%s: [%s] \n%s\n%s\n", v.Name, v.Severity, v.Description, v.Link)
			fmt.Fprintln(file, "-----------------------------------------")
		}
	})

	return nil
}
<|endoftext|>


<commit_before>package node

import (
	"crypto/md5"
	"errors"
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/node/file/index"
	"io"
	"math/rand"
	"os"
	"syscall"
	"time"
)

func (node *Node) SetFile(file FormFile) (err error) {
	fileStat, err := os.Stat(file.Path)
	if err != nil {
		return
	}
	os.Rename(file.Path, node.FilePath())
	node.File.Name = file.Name
	node.File.Size = fileStat.Size()
	node.File.Checksum = file.Checksum
	node.File.CreatedOn = fileStat.ModTime()

	//fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}
	err = node.Save()
	return
}

func (node *Node) SetFileFromSubset(subsetIndices FormFile) (err error) {
	// load parent node
	var n *Node
	n, err = Load(node.Subset.Parent.Id)
	if err != nil {
		return err
	}

	if _, indexExists := n.Indexes[node.Subset.Parent.IndexName]; !indexExists {
		return errors.New("Index '" + node.Subset.Parent.IndexName + "' does not exist for parent node.")
	}

	parentIndexFile := n.IndexPath() + "/" + node.Subset.Parent.IndexName + ".idx"
	if _, statErr := os.Stat(parentIndexFile); statErr != nil {
		return errors.New("Could not stat index file for parent node where parent node = '" + node.Subset.Parent.Id + "' and index = '" + node.Subset.Parent.IndexName + "'.")
	}

	// we default to "array" index format for backwards compatibility
	indexFormat := "array"
	if n.Indexes[node.Subset.Parent.IndexName].Format == "array" || n.Indexes[node.Subset.Parent.IndexName].Format == "matrix" {
		indexFormat = n.Indexes[node.Subset.Parent.IndexName].Format
	}

	if fi, statErr := os.Stat(subsetIndices.Path); statErr != nil {
		return errors.New("Could not stat uploaded subset_indices file.")
	} else {
		if fi.Size() == 0 {
			return errors.New("Uploaded subset_indices file is size zero.  This is prohibited.")
		}
	}

	f, _ := os.Open(subsetIndices.Path)
	defer f.Close()
	idxer := index.NewSubsetIndexer(f)
	coIndexPath := node.Path() + "/" + node.Id + ".subset.idx"
	oIndexPath := node.Path() + "/idx/" + node.Subset.Parent.IndexName + ".idx"

	coCount, oCount, oSize, err := index.CreateSubsetNodeIndexes(&idxer, coIndexPath, oIndexPath, parentIndexFile, indexFormat, n.Indexes[node.Subset.Parent.IndexName].TotalUnits)
	if err != nil {
		return
	}

	// this info refers to the compressed index for the subset node's data file
	node.Subset.Index.Path = coIndexPath
	node.Subset.Index.TotalUnits = coCount
	node.Subset.Index.AvgUnitSize = oSize / coCount
	node.Subset.Index.Format = "array"
	node.File.Size = oSize
	node.File.CreatedOn = time.Now()

	// this info is for the subset index that's been created in the index folder
	node.Indexes[node.Subset.Parent.IndexName] = IdxInfo{
		Type:        "subset",
		TotalUnits:  oCount,
		AvgUnitSize: oSize / oCount,
		Format:      indexFormat,
		CreatedOn:   time.Now(),
	}

	// fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}

	err = node.Save()
	return
}

func (node *Node) SetFileFromPath(path string, action string) (err error) {
	fileStat, err := os.Stat(path)
	if err != nil {
		return
	}
	node.File.Name = fileStat.Name()
	node.File.Size = fileStat.Size()
	node.File.CreatedOn = fileStat.ModTime()

	tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())

	if action != "copy_file" && action != "move_file" && action != "keep_file" {
		return errors.New("setting file from path requires action field equal to copy_file, move_file or keep_file")
	}

	if action == "move_file" {
		// Determine if device ID of src and target are the same before proceeding.
		var devID1 uint64
		var f, _ = os.Open(path)
		var fi, _ = f.Stat()
		s := fi.Sys()
		if s, ok := s.(*syscall.Stat_t); ok {
			devID1 = uint64(s.Dev)
		} else {
			return errors.New("Could not determine device ID of data input file, try copy_file action instead.")
		}

		var devID2 uint64
		var f2, _ = os.Open(tmpPath)
		var fi2, _ = f2.Stat()
		s2 := fi2.Sys()
		if s2, ok := s2.(*syscall.Stat_t); ok {
			devID2 = uint64(s2.Dev)
		} else {
			return errors.New("Could not complete move_file action, try copy_file action instead.")
		}

		if devID1 != devID2 {
			return errors.New("Will not be able to rename file because data input file is on a different device than Shock data directory, try copy_file action instead.")
		}
	}

	// Open file for reading.
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	// Set writer
	var dst io.Writer
	md5h := md5.New()

	if action == "copy_file" {
		tmpFile, err := os.Create(tmpPath)
		if err != nil {
			return err
		}
		defer tmpFile.Close()
		dst = io.MultiWriter(tmpFile, md5h)
	} else {
		dst = md5h
	}

	if _, err = io.Copy(dst, f); err != nil {
		return err
	}

	node.File.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))

	//fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}

	if action == "copy_file" {
		os.Rename(tmpPath, node.FilePath())
	} else if action == "move_file" {
		os.Rename(path, node.FilePath())
	} else {
		node.File.Path = path
	}
	err = node.Save()
	return
}

func (node *Node) SetFileFromParts(allowEmpty bool) (err error) {
	out, err := os.Create(fmt.Sprintf("%s/%s.data", node.Path(), node.Id))
	if err != nil {
		return
	}
	defer out.Close()
	md5h := md5.New()
	for i := 1; i <= node.Parts.Count; i++ {
		filename := fmt.Sprintf("%s/parts/%d", node.Path(), i)

		// skip this portion unless either
		// 1. file exists, or
		// 2. file does not exist and allowEmpty == false
		if _, errf := os.Stat(filename); errf == nil || (errf != nil && allowEmpty == false) {
			part, err := os.Open(filename)
			if err != nil {
				return err
			}
			defer part.Close()
			dst := io.MultiWriter(out, md5h)
			if _, err = io.Copy(dst, part); err != nil {
				return err
			}
		}
	}
	fileStat, err := os.Stat(fmt.Sprintf("%s/%s.data", node.Path(), node.Id))
	if err != nil {
		return
	}
	node.File.Name = node.Id
	node.File.Size = fileStat.Size()
	node.File.CreatedOn = fileStat.ModTime()
	node.File.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))

	//fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
	}

	err = node.Save()
	return
}

func (node *Node) Rmdir() (err error) {
	return os.RemoveAll(node.Path())
}

func (node *Node) Mkdir() (err error) {
	err = os.MkdirAll(node.Path(), 0777)
	if err != nil {
		return
	}
	err = os.MkdirAll(node.IndexPath(), 0777)
	if err != nil {
		return
	}
	return
}
<commit_msg>missed a line<commit_after>package node

import (
	"crypto/md5"
	"errors"
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/node/file/index"
	"io"
	"math/rand"
	"os"
	"syscall"
	"time"
)

func (node *Node) SetFile(file FormFile) (err error) {
	fileStat, err := os.Stat(file.Path)
	if err != nil {
		return
	}
	os.Rename(file.Path, node.FilePath())
	node.File.Name = file.Name
	node.File.Size = fileStat.Size()
	node.File.Checksum = file.Checksum
	node.File.CreatedOn = fileStat.ModTime()

	//fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}
	err = node.Save()
	return
}

func (node *Node) SetFileFromSubset(subsetIndices FormFile) (err error) {
	// load parent node
	var n *Node
	n, err = Load(node.Subset.Parent.Id)
	if err != nil {
		return err
	}

	if _, indexExists := n.Indexes[node.Subset.Parent.IndexName]; !indexExists {
		return errors.New("Index '" + node.Subset.Parent.IndexName + "' does not exist for parent node.")
	}

	parentIndexFile := n.IndexPath() + "/" + node.Subset.Parent.IndexName + ".idx"
	if _, statErr := os.Stat(parentIndexFile); statErr != nil {
		return errors.New("Could not stat index file for parent node where parent node = '" + node.Subset.Parent.Id + "' and index = '" + node.Subset.Parent.IndexName + "'.")
	}

	// we default to "array" index format for backwards compatibility
	indexFormat := "array"
	if n.Indexes[node.Subset.Parent.IndexName].Format == "array" || n.Indexes[node.Subset.Parent.IndexName].Format == "matrix" {
		indexFormat = n.Indexes[node.Subset.Parent.IndexName].Format
	}

	if fi, statErr := os.Stat(subsetIndices.Path); statErr != nil {
		return errors.New("Could not stat uploaded subset_indices file.")
	} else {
		if fi.Size() == 0 {
			return errors.New("Uploaded subset_indices file is size zero.  This is prohibited.")
		}
	}

	f, _ := os.Open(subsetIndices.Path)
	defer f.Close()
	idxer := index.NewSubsetIndexer(f)
	coIndexPath := node.Path() + "/" + node.Id + ".subset.idx"
	oIndexPath := node.Path() + "/idx/" + node.Subset.Parent.IndexName + ".idx"

	coCount, oCount, oSize, err := index.CreateSubsetNodeIndexes(&idxer, coIndexPath, oIndexPath, parentIndexFile, indexFormat, n.Indexes[node.Subset.Parent.IndexName].TotalUnits)
	if err != nil {
		return
	}

	// this info refers to the compressed index for the subset node's data file
	node.Subset.Index.Path = coIndexPath
	node.Subset.Index.TotalUnits = coCount
	node.Subset.Index.AvgUnitSize = oSize / coCount
	node.Subset.Index.Format = "array"
	node.File.Size = oSize
	node.File.CreatedOn = time.Now()

	// this info is for the subset index that's been created in the index folder
	node.Indexes[node.Subset.Parent.IndexName] = IdxInfo{
		Type:        "subset",
		TotalUnits:  oCount,
		AvgUnitSize: oSize / oCount,
		Format:      indexFormat,
		CreatedOn:   time.Now(),
	}

	// fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}

	err = node.Save()
	return
}

func (node *Node) SetFileFromPath(path string, action string) (err error) {
	fileStat, err := os.Stat(path)
	if err != nil {
		return
	}
	node.File.Name = fileStat.Name()
	node.File.Size = fileStat.Size()
	node.File.CreatedOn = fileStat.ModTime()

	tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())

	if action != "copy_file" && action != "move_file" && action != "keep_file" {
		return errors.New("setting file from path requires action field equal to copy_file, move_file or keep_file")
	}

	if action == "move_file" {
		// Determine if device ID of src and target are the same before proceeding.
		var devID1 uint64
		var f, _ = os.Open(path)
		var fi, _ = f.Stat()
		s := fi.Sys()
		if s, ok := s.(*syscall.Stat_t); ok {
			devID1 = uint64(s.Dev)
		} else {
			return errors.New("Could not determine device ID of data input file, try copy_file action instead.")
		}

		var devID2 uint64
		var f2, _ = os.Open(tmpPath)
		var fi2, _ = f2.Stat()
		s2 := fi2.Sys()
		if s2, ok := s2.(*syscall.Stat_t); ok {
			devID2 = uint64(s2.Dev)
		} else {
			return errors.New("Could not complete move_file action, try copy_file action instead.")
		}

		if devID1 != devID2 {
			return errors.New("Will not be able to rename file because data input file is on a different device than Shock data directory, try copy_file action instead.")
		}
	}

	// Open file for reading.
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	// Set writer
	var dst io.Writer
	md5h := md5.New()

	if action == "copy_file" {
		tmpFile, err := os.Create(tmpPath)
		if err != nil {
			return err
		}
		defer tmpFile.Close()
		dst = io.MultiWriter(tmpFile, md5h)
	} else {
		dst = md5h
	}

	if _, err = io.Copy(dst, f); err != nil {
		return err
	}

	node.File.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))

	//fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}

	if action == "copy_file" {
		os.Rename(tmpPath, node.FilePath())
	} else if action == "move_file" {
		os.Rename(path, node.FilePath())
	} else {
		node.File.Path = path
	}
	err = node.Save()
	return
}

func (node *Node) SetFileFromParts(allowEmpty bool) (err error) {
	out, err := os.Create(fmt.Sprintf("%s/%s.data", node.Path(), node.Id))
	if err != nil {
		return
	}
	defer out.Close()
	md5h := md5.New()
	for i := 1; i <= node.Parts.Count; i++ {
		filename := fmt.Sprintf("%s/parts/%d", node.Path(), i)

		// skip this portion unless either
		// 1. file exists, or
		// 2. file does not exist and allowEmpty == false
		if _, errf := os.Stat(filename); errf == nil || (errf != nil && allowEmpty == false) {
			part, err := os.Open(filename)
			if err != nil {
				return err
			}
			defer part.Close()
			dst := io.MultiWriter(out, md5h)
			if _, err = io.Copy(dst, part); err != nil {
				return err
			}
		}
	}
	fileStat, err := os.Stat(fmt.Sprintf("%s/%s.data", node.Path(), node.Id))
	if err != nil {
		return
	}
	node.File.Name = node.Id
	node.File.Size = fileStat.Size()
	node.File.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
	node.File.CreatedOn = fileStat.ModTime()

	//fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}

	err = node.Save()
	return
}

func (node *Node) Rmdir() (err error) {
	return os.RemoveAll(node.Path())
}

func (node *Node) Mkdir() (err error) {
	err = os.MkdirAll(node.Path(), 0777)
	if err != nil {
		return
	}
	err = os.MkdirAll(node.IndexPath(), 0777)
	if err != nil {
		return
	}
	return
}
<|endoftext|>


<commit_before>package product

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
)

var Endpoint string

type BonoboProduct map[string]interface{}
type ShopProduct map[string]string

const (
	DescriptionKey      = "Product Description"
	FloorPriceKey       = "Cost Price"
	StockKey            = "Product in stock"
	CategoryPathKey     = "Category Path"
	CategoryKey         = "Category"
	SubCategoryKey      = "Sub category"
	ShopCodeKey         = "Shop Code"
	VariantIDKey        = "Variant ID"
	ProductNameKey      = "Product Name"
	PictureLinkKey      = "Picture Link"
	DeeplinkKey         = "Deeplink"
	ProductEanKey       = "Product Ean"
	ProductBrandKey     = "Product Brand"
	DeliveryPeriodKey   = "Delivery Period"
	ProductInStockKey   = "Product in stock"
	StockStatusKey      = "Stock Status"
	EnabledKey          = "Enabled"
	DisabledAtKey       = "Disabled At"
	SellingPriceExclKey = "Selling Price Ex"
	SellingPriceInclKey = "Selling Price"
)

func (s ShopProduct) Description() string {
	return s[DescriptionKey]
}
func (s ShopProduct) FloorPrice() string {
	return s[FloorPriceKey]
}
func (s ShopProduct) Stock() (int, error) {
	return strconv.Atoi(s[StockKey])
}
func (s ShopProduct) Category() string {
	return s[CategoryKey]
}
func (s ShopProduct) CategoryPath() string {
	return s[CategoryPathKey]
}
func (s ShopProduct) SubCategory() string {
	return s[SubCategoryKey]
}
func (s ShopProduct) ShopCode() string {
	return s[ShopCodeKey]
}

func (s ShopProduct) VariantID() string {
	return s[VariantIDKey]
}
func (s ShopProduct) ProductName() string {
	return s[ProductNameKey]
}
func (s ShopProduct) PictureLink() string {
	return s[PictureLinkKey]
}
func (s ShopProduct) Deeplink() string {
	return s[DeeplinkKey]
}
func (s ShopProduct) ProductEan() string {
	return s[ProductEanKey]
}
func (s ShopProduct) ProductBrand() string {
	return s[ProductBrandKey]
}
func (s ShopProduct) DeliveryPeriod() string {
	return s[DeliveryPeriodKey]
}
func (s ShopProduct) ProductInStock() string {
	return s[ProductInStockKey]
}
func (s ShopProduct) StockStatus() string {
	return s[StockStatusKey]
}
func (s ShopProduct) SellingPriceIncl() string {
	return s[SellingPriceInclKey]
}
func (s ShopProduct) SellingPriceExcl() string {
	return s[SellingPriceExclKey]
}
func (s ShopProduct) Enabled() string {
	return s[EnabledKey]
}
func (s ShopProduct) DisabledAt() string {
	return s[DisabledAtKey]
}

type Finder func(int) (<-chan ShopProduct, <-chan error)

func fetchValue(hash map[string]interface{}, key string) string {
	value, ok := hash[key].(string)
	if ok {
		return value
	}
	return ""
}
func fetchNumberValue(hash map[string]interface{}, key string) string {
	value, ok := hash[key].(float64)
	if ok {
		return fmt.Sprintf("%.0f", value)
	}
	return ""
}
func fetchBool(hash map[string]interface{}, key string) string {
	value, ok := hash[key].(bool)
	if ok && value {
		return "true"
	}
	return "false"
}
func (bp BonoboProduct) toShopProducts() []ShopProduct {
	shopProducts := []ShopProduct{}
	variants := bp["variants"].([]interface{})
	for _, variantInterface := range variants {
		shopProduct := ShopProduct{}
		shopProduct[ShopCodeKey] = fetchValue(bp, "shop_code")
		shopProduct[ProductNameKey] = fetchValue(bp, "product_name")
		shopProduct[DescriptionKey] = fetchValue(bp, "description")
		shopProduct[ProductBrandKey] = fetchValue(bp, "brand")
		shopProduct[DeeplinkKey] = fetchValue(bp, "deeplink")
		shopProduct[CategoryPathKey] = fetchValue(bp, "category_path")
		variant := variantInterface.(map[string]interface{})
		shopProduct[VariantIDKey] = variant["variant_id"].(string)
		shopProduct[PictureLinkKey] = fetchValue(variant, "picture_link")
		shopProduct[ProductEanKey] = fetchValue(variant, "ean")
		shopProduct[FloorPriceKey] = fetchNumberValue(variant, "cost_price_excl")
		shopProduct[ProductInStockKey] = fetchNumberValue(variant, "product_in_stock")
		shopProduct[StockStatusKey] = fetchValue(variant, "stock_status")
		shopProduct[DeliveryPeriodKey] = fetchValue(variant, "delivery_period")
		shopProduct[SellingPriceInclKey] = fetchNumberValue(variant, "price_incl")
		shopProduct[SellingPriceExclKey] = fetchNumberValue(variant, "price_excl")
		shopProduct[EnabledKey] = fetchBool(variant, "enabled")
		shopProduct[DisabledAtKey] = fetchValue(variant, "disabled_at")

		shopProducts = append(shopProducts, shopProduct)
	}
	return shopProducts
}

func FindAll(shopId int) (<-chan ShopProduct, <-chan error) {
	shopProductChannel := make(chan ShopProduct)
	errorChannel := make(chan error, 5)
	request, err := http.NewRequest("GET", catalogUrl(shopId), nil)
	if err != nil {
		errorChannel <- err
		return shopProductChannel, errorChannel
	}
	resp, err := http.DefaultClient.Do(request)
	if err != nil {
		errorChannel <- err
		return shopProductChannel, errorChannel
	}
	go func() {
		defer close(shopProductChannel)
		defer close(errorChannel)
		defer resp.Body.Close()
		decoder := json.NewDecoder(resp.Body)
		var product BonoboProduct
		var shopProduct ShopProduct
		for {
			err := decoder.Decode(&product)
			if err == io.EOF {
				return
			}
			if err != nil {
				errorChannel <- err
				return
			}
			for _, shopProduct = range product.toShopProducts() {
				shopProductChannel <- shopProduct
			}

		}
	}()
	return shopProductChannel, errorChannel
}

func catalogUrl(shopId int) string {
	return fmt.Sprintf("%s/shops/%d/mongo/products?sorted=true", Endpoint, shopId)
}
<commit_msg>[+] Add logger for bonobo catalog<commit_after>package product

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
)

var Endpoint string

type BonoboProduct map[string]interface{}
type ShopProduct map[string]string

const (
	DescriptionKey      = "Product Description"
	FloorPriceKey       = "Cost Price"
	StockKey            = "Product in stock"
	CategoryPathKey     = "Category Path"
	CategoryKey         = "Category"
	SubCategoryKey      = "Sub category"
	ShopCodeKey         = "Shop Code"
	VariantIDKey        = "Variant ID"
	ProductNameKey      = "Product Name"
	PictureLinkKey      = "Picture Link"
	DeeplinkKey         = "Deeplink"
	ProductEanKey       = "Product Ean"
	ProductBrandKey     = "Product Brand"
	DeliveryPeriodKey   = "Delivery Period"
	ProductInStockKey   = "Product in stock"
	StockStatusKey      = "Stock Status"
	EnabledKey          = "Enabled"
	DisabledAtKey       = "Disabled At"
	SellingPriceExclKey = "Selling Price Ex"
	SellingPriceInclKey = "Selling Price"
)

func (s ShopProduct) Description() string {
	return s[DescriptionKey]
}
func (s ShopProduct) FloorPrice() string {
	return s[FloorPriceKey]
}
func (s ShopProduct) Stock() (int, error) {
	return strconv.Atoi(s[StockKey])
}
func (s ShopProduct) Category() string {
	return s[CategoryKey]
}
func (s ShopProduct) CategoryPath() string {
	return s[CategoryPathKey]
}
func (s ShopProduct) SubCategory() string {
	return s[SubCategoryKey]
}
func (s ShopProduct) ShopCode() string {
	return s[ShopCodeKey]
}

func (s ShopProduct) VariantID() string {
	return s[VariantIDKey]
}
func (s ShopProduct) ProductName() string {
	return s[ProductNameKey]
}
func (s ShopProduct) PictureLink() string {
	return s[PictureLinkKey]
}
func (s ShopProduct) Deeplink() string {
	return s[DeeplinkKey]
}
func (s ShopProduct) ProductEan() string {
	return s[ProductEanKey]
}
func (s ShopProduct) ProductBrand() string {
	return s[ProductBrandKey]
}
func (s ShopProduct) DeliveryPeriod() string {
	return s[DeliveryPeriodKey]
}
func (s ShopProduct) ProductInStock() string {
	return s[ProductInStockKey]
}
func (s ShopProduct) StockStatus() string {
	return s[StockStatusKey]
}
func (s ShopProduct) SellingPriceIncl() string {
	return s[SellingPriceInclKey]
}
func (s ShopProduct) SellingPriceExcl() string {
	return s[SellingPriceExclKey]
}
func (s ShopProduct) Enabled() string {
	return s[EnabledKey]
}
func (s ShopProduct) DisabledAt() string {
	return s[DisabledAtKey]
}

type Finder func(int) (<-chan ShopProduct, <-chan error)

func fetchValue(hash map[string]interface{}, key string) string {
	value, ok := hash[key].(string)
	if ok {
		return value
	}
	return ""
}
func fetchNumberValue(hash map[string]interface{}, key string) string {
	value, ok := hash[key].(float64)
	if ok {
		return fmt.Sprintf("%.0f", value)
	}
	return ""
}
func fetchBool(hash map[string]interface{}, key string) string {
	value, ok := hash[key].(bool)
	if ok && value {
		return "true"
	}
	return "false"
}
func (bp BonoboProduct) toShopProducts() []ShopProduct {
	shopProducts := []ShopProduct{}
	variants := bp["variants"].([]interface{})
	for _, variantInterface := range variants {
		shopProduct := ShopProduct{}
		shopProduct[ShopCodeKey] = fetchValue(bp, "shop_code")
		shopProduct[ProductNameKey] = fetchValue(bp, "product_name")
		shopProduct[DescriptionKey] = fetchValue(bp, "description")
		shopProduct[ProductBrandKey] = fetchValue(bp, "brand")
		shopProduct[DeeplinkKey] = fetchValue(bp, "deeplink")
		shopProduct[CategoryPathKey] = fetchValue(bp, "category_path")
		variant := variantInterface.(map[string]interface{})
		shopProduct[VariantIDKey] = variant["variant_id"].(string)
		shopProduct[PictureLinkKey] = fetchValue(variant, "picture_link")
		shopProduct[ProductEanKey] = fetchValue(variant, "ean")
		shopProduct[FloorPriceKey] = fetchNumberValue(variant, "cost_price_excl")
		shopProduct[ProductInStockKey] = fetchNumberValue(variant, "product_in_stock")
		shopProduct[StockStatusKey] = fetchValue(variant, "stock_status")
		shopProduct[DeliveryPeriodKey] = fetchValue(variant, "delivery_period")
		shopProduct[SellingPriceInclKey] = fetchNumberValue(variant, "price_incl")
		shopProduct[SellingPriceExclKey] = fetchNumberValue(variant, "price_excl")
		shopProduct[EnabledKey] = fetchBool(variant, "enabled")
		shopProduct[DisabledAtKey] = fetchValue(variant, "disabled_at")

		shopProducts = append(shopProducts, shopProduct)
	}
	return shopProducts
}

func FindAll(shopId int) (<-chan ShopProduct, <-chan error) {
	shopProductChannel := make(chan ShopProduct)
	errorChannel := make(chan error, 5)
	request, err := http.NewRequest("GET", catalogUrl(shopId), nil)
	if err != nil {
		errorChannel <- err
		return shopProductChannel, errorChannel
	}
	resp, err := http.DefaultClient.Do(request)
	if err != nil {
		errorChannel <- err
		return shopProductChannel, errorChannel
	}
	fmt.Printf("Opened stream for shop %d\n", shopId)
	go func() {
		defer close(shopProductChannel)
		defer close(errorChannel)
		defer resp.Body.Close()
		defer fmt.Printf("Closed stream for shop %d\n", shopId)

		decoder := json.NewDecoder(resp.Body)
		var product BonoboProduct
		var shopProduct ShopProduct
		for {
			err := decoder.Decode(&product)
			if err == io.EOF {
				return
			}
			if err != nil {
				errorChannel <- err
				return
			}
			for _, shopProduct = range product.toShopProducts() {
				shopProductChannel <- shopProduct
			}

		}
	}()
	return shopProductChannel, errorChannel
}

func catalogUrl(shopId int) string {
	return fmt.Sprintf("%s/shops/%d/mongo/products?sorted=true", Endpoint, shopId)
}
<|endoftext|>


<commit_before>package contacts

import (
	"context"

	"io/ioutil"

	"github.com/pkg/errors"
	"github.com/uber-go/zap"
	"github.com/uber/zanzibar/examples/example-gateway/build/clients"
	contactsClientStructs "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/github.com/uber/zanzibar/clients/contacts/contacts"
	zanzibar "github.com/uber/zanzibar/runtime"
)

// HandleSaveContactsRequest "/contacts/:userUUID/contacts"
func HandleSaveContactsRequest(
	ctx context.Context,
	req *zanzibar.ServerHTTPRequest,
	res *zanzibar.ServerHTTPResponse,
	clients *clients.Clients,
) {
	var body SaveContactsRequest
	ok := req.ReadAndUnmarshalBody(&body)
	if !ok {
		return
	}

	// TODO AuthenticatedRequest()
	// TODO MatchedIdRequest({paramName: 'userUUID'})

	body.UserUUID = req.Params[0].Value
	body.AppType = req.Header.Get("x-uber-client-name")
	body.DeviceType = req.Header.Get("x-uber-device")
	body.AppVersion = req.Header.Get("x-uber-client-version")

	clientBody := convertToClient(&body)
	cres, err := clients.Contacts.SaveContacts(ctx, clientBody, nil)
	if err != nil {
		req.Logger.Error("Could not make client request",
			zap.String("error", err.Error()),
		)
		res.SendError(500, errors.Wrap(err, "Could not make client request:"))
		return
	}

	defer func() {
		if cerr := cres.Body.Close(); cerr != nil && err == nil {
			err = cerr
		}
	}()

	// Handle client respnse.
	if !res.IsOKResponse(cres.StatusCode, []int{200, 202}) {
		req.Logger.Warn("Unknown response status code",
			zap.Int("status code", cres.StatusCode),
		)
	}

	bytes, err := ioutil.ReadAll(cres.Body)
	if err != nil {
		res.SendError(500, errors.Wrap(err, "could not read client response body:"))
		return
	}
	var clientRespBody contactsClientStructs.SaveContactsResponse
	if err := clientRespBody.UnmarshalJSON(bytes); err != nil {
		res.SendError(500, errors.Wrap(err, "could not unmarshal client response body:"))
		return
	}
	response := convertToResponse(&clientRespBody)
	res.WriteJSON(cres.StatusCode, response)
}

func convertToResponse(
	body *contactsClientStructs.SaveContactsResponse,
) *SaveContactsResponse {
	return &SaveContactsResponse{}
}

func convertToClient(
	body *SaveContactsRequest,
) *contactsClientStructs.SaveContactsRequest {
	clientBody := &contactsClientStructs.SaveContactsRequest{}
	clientBody.UserUUID = contactsClientStructs.UUID(body.UserUUID)

	for _, contact := range body.Contacts {
		clientContact := &contactsClientStructs.Contact{}
		clientAttributes := &contactsClientStructs.ContactAttributes{}
		attributes := contact.Attributes

		clientAttributes.FirstName = attributes.FirstName
		clientAttributes.LastName = attributes.LastName
		clientAttributes.Nickname = attributes.Nickname
		clientAttributes.HasPhoto = attributes.HasPhoto
		clientAttributes.NumFields = attributes.NumFields
		clientAttributes.TimesContacted = attributes.TimesContacted
		clientAttributes.LastTimeContacted = attributes.LastTimeContacted
		clientAttributes.IsStarred = attributes.IsStarred
		clientAttributes.HasCustomRingtone = attributes.HasCustomRingtone
		clientAttributes.IsSendToVoicemail = attributes.IsSendToVoiceMail
		clientAttributes.HasThumbnail = attributes.HasThumbnail
		clientAttributes.NamePrefix = attributes.NamePrefix
		clientAttributes.NameSuffix = attributes.NameSuffix

		for _, fragment := range contact.Fragments {
			clientFragment := &contactsClientStructs.ContactFragment{}
			clientFragment.Text = fragment.Text
			clientFragmentType := contactsClientStructs.
				ContactFragmentType(*fragment.Type)
			clientFragment.Type = &clientFragmentType

			clientContact.Fragments = append(
				clientContact.Fragments, clientFragment,
			)
		}

		clientContact.Attributes = clientAttributes
		clientBody.Contacts = append(clientBody.Contacts, clientContact)
	}

	return clientBody
}
<commit_msg>examples: fixup save-contacts to call Flush()<commit_after>package contacts

import (
	"context"

	"io/ioutil"

	"github.com/pkg/errors"
	"github.com/uber-go/zap"
	"github.com/uber/zanzibar/examples/example-gateway/build/clients"
	contactsClientStructs "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/github.com/uber/zanzibar/clients/contacts/contacts"
	zanzibar "github.com/uber/zanzibar/runtime"
)

// HandleSaveContactsRequest "/contacts/:userUUID/contacts"
func HandleSaveContactsRequest(
	ctx context.Context,
	req *zanzibar.ServerHTTPRequest,
	res *zanzibar.ServerHTTPResponse,
	clients *clients.Clients,
) {
	var body SaveContactsRequest
	ok := req.ReadAndUnmarshalBody(&body)
	if !ok {
		return
	}

	// TODO AuthenticatedRequest()
	// TODO MatchedIdRequest({paramName: 'userUUID'})

	body.UserUUID = req.Params[0].Value
	body.AppType = req.Header.Get("x-uber-client-name")
	body.DeviceType = req.Header.Get("x-uber-device")
	body.AppVersion = req.Header.Get("x-uber-client-version")

	clientBody := convertToClient(&body)
	cres, err := clients.Contacts.SaveContacts(ctx, clientBody, nil)
	if err != nil {
		req.Logger.Error("Could not make client request",
			zap.String("error", err.Error()),
		)
		res.SendError(500, errors.Wrap(err, "Could not make client request:"))
		return
	}

	defer func() {
		if cerr := cres.Body.Close(); cerr != nil && err == nil {
			err = cerr
		}
	}()

	// Handle client respnse.
	if !res.IsOKResponse(cres.StatusCode, []int{200, 202}) {
		req.Logger.Warn("Unknown response status code",
			zap.Int("status code", cres.StatusCode),
		)
	}

	bytes, err := ioutil.ReadAll(cres.Body)
	if err != nil {
		res.SendError(500, errors.Wrap(err, "could not read client response body:"))
		return
	}
	var clientRespBody contactsClientStructs.SaveContactsResponse
	if err := clientRespBody.UnmarshalJSON(bytes); err != nil {
		res.SendError(500, errors.Wrap(err, "could not unmarshal client response body:"))
		return
	}
	response := convertToResponse(&clientRespBody)
	res.WriteJSON(cres.StatusCode, response)
	res.Flush()
}

func convertToResponse(
	body *contactsClientStructs.SaveContactsResponse,
) *SaveContactsResponse {
	return &SaveContactsResponse{}
}

func convertToClient(
	body *SaveContactsRequest,
) *contactsClientStructs.SaveContactsRequest {
	clientBody := &contactsClientStructs.SaveContactsRequest{}
	clientBody.UserUUID = contactsClientStructs.UUID(body.UserUUID)

	for _, contact := range body.Contacts {
		clientContact := &contactsClientStructs.Contact{}
		clientAttributes := &contactsClientStructs.ContactAttributes{}
		attributes := contact.Attributes

		clientAttributes.FirstName = attributes.FirstName
		clientAttributes.LastName = attributes.LastName
		clientAttributes.Nickname = attributes.Nickname
		clientAttributes.HasPhoto = attributes.HasPhoto
		clientAttributes.NumFields = attributes.NumFields
		clientAttributes.TimesContacted = attributes.TimesContacted
		clientAttributes.LastTimeContacted = attributes.LastTimeContacted
		clientAttributes.IsStarred = attributes.IsStarred
		clientAttributes.HasCustomRingtone = attributes.HasCustomRingtone
		clientAttributes.IsSendToVoicemail = attributes.IsSendToVoiceMail
		clientAttributes.HasThumbnail = attributes.HasThumbnail
		clientAttributes.NamePrefix = attributes.NamePrefix
		clientAttributes.NameSuffix = attributes.NameSuffix

		for _, fragment := range contact.Fragments {
			clientFragment := &contactsClientStructs.ContactFragment{}
			clientFragment.Text = fragment.Text
			clientFragmentType := contactsClientStructs.
				ContactFragmentType(*fragment.Type)
			clientFragment.Type = &clientFragmentType

			clientContact.Fragments = append(
				clientContact.Fragments, clientFragment,
			)
		}

		clientContact.Attributes = clientAttributes
		clientBody.Contacts = append(clientBody.Contacts, clientContact)
	}

	return clientBody
}
<|endoftext|>


<commit_before>package reverseproxy

import (
    "io"
    "errors"
    "fmt"
    "log"
    "net/http"
    "net/url"
    "strings"
    "sync"
    "time"

    "github.com/oxtoacart/bpool"
)

// HTTP proxy application
// This interface encapsulates the methods that the HTTP
// processing application needs to implement to
// use the reverse proxy service
// - a handler to process HTTP Requests
// - a handler to process HTTP Responses
type HttpApplication interface {
    RequestHandler(req *http.Request) error
    ResponseHandler(resp *http.Response) error
}

// onExitFlushLoop is a callback set by tests to detect the state of the
// flushLoop() goroutine.
var onExitFlushLoop func()

// ErrResponseShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrResponseShortWrite = errors.New("short write")

// A BufferPool is an interface for getting and returing temporary
// byte slices for use by io.CopyBuffer.
type BufferPool struct {
    size int
    buf  []byte
}

func (pool *BufferPool) Get() []byte {
    buffer := pool.buf
    pool.buf = nil
    return buffer
}

func (pool *BufferPool) Put(v []byte) {
    if pool.buf != nil {
        fmt.Printf("Buffer already given out... should be nil")
        return
    }
    pool.buf = v
}

/*
 *  HTTP reverse proxy using the standard library reverse proxy
 *
 *    target:
 */
type HttpReverseProxy struct {
    // Director must be a function that modified the request into a new
    // request to be sent using Transport. Its response is then copied
    // back to the client
    Director func(*http.Request)

    // The transport used to perform proxy requests.
    // If nil, http.DefaultTransport is used
    Transport *ProxyTransport

    // FlushInterval specifies the flush interval to flush to the client
    // while copying the response body.
    // If zero, no periodic flushing is done
    FlushInterval time.Duration

    // ErrorLog specifies an optional logger for errors that occur when
    // attempting to process the request. If nil, logging goes to os.Stderr
    // using the standard logger
    ErrorLog *log.Logger

    // BufferPool specifies a buffer pool to get byte slices for use by
    // io.CopyBuffer when copying HTTP request and response bodies
    BufferPool *BufferPool

    // The application that is processes the HTTP data as it is
    // streamed to/from the server being proxied
    app    HttpApplication
}

func NewHttpReverseProxy(target *url.URL, app HttpApplication) (*HttpReverseProxy, error) {

    pool := bpool.BytePool(20, 524288)
    director := func(req *http.Request) {
        targetQuery := target.RawQuery
        req.URL.Scheme = target.Scheme
        req.URL.Host = target.Host
        if targetQuery == "" || req.URL.RawQuery == "" {
            req.URL.RawQuery = targetQuery + req.URL.RawQuery
        } else {
            req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
        }
        //update the host header in the request
        req.Host = target.Host
    }

    return &HttpReverseProxy{
        Director: director,
        Transport: &ProxyTransport{},
        BufferPool: pool,
        app: app,
    }, nil
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func (p *HttpReverseProxy) Start() error {
    fmt.Printf("Starting Object Store service on port 8080\n");
    http.ListenAndServe(":8080", p)
    return nil
}

// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te",      // canonicalized version of "TE"
	"Trailer", // not Trailers per URL above; http://www.rfc-editor.org/errata_search.php?eid=4522
	"Transfer-Encoding",
	"Upgrade",
}

func (p *HttpReverseProxy) processRequest(req *http.Request) (*http.Response, error) {
    transport := p.Transport

    err := transport.WriteHeader(req)
    if err != nil {
        fmt.Printf("proxy: Failed to send Request Headers to backend: %s\n", err)
        return nil, err
    }

    written := 0
    // write the body out, if there is one
    if req.Body != nil {
        src := req.Body
        fmt.Printf("Content Length is %d\n", req.Header.Get("Content-Length"))

        var buf []byte
        if p.BufferPool != nil {
            buf = p.BufferPool.Get()
        }
        for {
            nr, err := src.Read(buf)
            if nr > 0 {
                nw, err := transport.Write(req, buf[0:nr])
                if err != nil {
                    fmt.Printf("Error: Writing request body\n")
                    break
                }
                if nw != nr {
                    fmt.Printf("Error: ShortRequestWrite\n")
                    break
                }
                written += nw
                fmt.Printf("Written %d bytes\n", written)
            }
            if err == io.EOF {
                break
            }
        }
        if p.BufferPool != nil {
            p.BufferPool.Put(buf)
        }
    }
    resp, err := transport.ReadResponse(req)
    return resp, err
}

func (p *HttpReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {

	outreq := new(http.Request)
	*outreq = *req // includes shallow copies of maps, but okay
    if req.ContentLength == 0 {
        outreq.Body = nil
    }

	p.Director(outreq)
	outreq.Proto = "HTTP/1.1"
	outreq.ProtoMajor = 1
	outreq.ProtoMinor = 1
	outreq.Close = false

    // Remove hop-by-hop headers listed in the "Connection" header.
	// See RFC 2616, section 14.10.
	copiedHeaders := false
	if c := outreq.Header.Get("Connection"); c != "" {
		for _, f := range strings.Split(c, ",") {
			if f = strings.TrimSpace(f); f != "" {
				if !copiedHeaders {
					outreq.Header = make(http.Header)
					copyHeader(outreq.Header, req.Header)
					copiedHeaders = true
				}
				outreq.Header.Del(f)
			}
		}
	}
	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		if outreq.Header.Get(h) != "" {
			if !copiedHeaders {
				outreq.Header = make(http.Header)
				copyHeader(outreq.Header, req.Header)
				copiedHeaders = true
			}
			outreq.Header.Del(h)
		}
	}

    //now call the app handlers if registered
    //p.app.RequestHandler(outreq)

    //send this request on its way out
    res, err := p.processRequest(outreq)

    //res, err := transport.RoundTrip(outreq)
	if err != nil {
        p.logf("http: proxy error: %v", err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}
	defer res.Body.Close()

	for _, h := range hopHeaders {
		res.Header.Del(h)
	}

    //now call the app handlers if registered
    p.app.ResponseHandler(res)

	copyHeader(rw.Header(), res.Header)

	rw.WriteHeader(res.StatusCode)
	p.copyResponse(rw, res.Body)
}

func (p *HttpReverseProxy) copyResponse(dst io.Writer, src io.Reader) (written int64, err error) {
	if p.FlushInterval != 0 {
		if wf, ok := dst.(writeFlusher); ok {
			mlw := &maxLatencyWriter{
				dst:     wf,
				latency: p.FlushInterval,
				done:    make(chan bool),
			}
			go mlw.flushLoop()
			defer mlw.stop()
			dst = mlw
		}
	}
    var buf []byte
    if p.BufferPool != nil {
        buf = p.BufferPool.Get()
    }
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
            // invoke the application callback, if registered
            // app may modify the request and return a different
            // buffer. Also buffer here needs to be larger than
            // the data being read, so that app handler can
            // modify contents and return update buffer.
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = ErrResponseShortWrite
				break
			}
		}
		if er == io.EOF {
			break
		}
		if er != nil {
			err = er
			break
		}
	}
    if p.BufferPool != nil {
        p.BufferPool.Put(buf)
    }
    return written, err
}

func (p *HttpReverseProxy) logf(format string, args ...interface{}) {
	if p.ErrorLog != nil {
		p.ErrorLog.Printf(format, args...)
	} else {
		log.Printf(format, args...)
	}
}

type writeFlusher interface {
	io.Writer
	http.Flusher
}

type maxLatencyWriter struct {
	dst     writeFlusher
	latency time.Duration

	lk   sync.Mutex // protects Write + Flush
	done chan bool
}

func (m *maxLatencyWriter) Write(p []byte) (int, error) {
	m.lk.Lock()
	defer m.lk.Unlock()
	return m.dst.Write(p)
}

func (m *maxLatencyWriter) flushLoop() {
	t := time.NewTicker(m.latency)
	defer t.Stop()
	for {
		select {
		case <-m.done:
			if onExitFlushLoop != nil {
				onExitFlushLoop()
			}
			return
		case <-t.C:
			m.lk.Lock()
			m.dst.Flush()
			m.lk.Unlock()
		}
	}
}

func (m *maxLatencyWriter) stop() {
    m.done <- true
}
<commit_msg>Use BytePool for buffer management<commit_after>package reverseproxy

import (
    "io"
    "errors"
    "fmt"
    "log"
    "net/http"
    "net/url"
    "strings"
    "sync"
    "time"

    "github.com/oxtoacart/bpool"
)

// HTTP proxy application
// This interface encapsulates the methods that the HTTP
// processing application needs to implement to
// use the reverse proxy service
// - a handler to process HTTP Requests
// - a handler to process HTTP Responses
type HttpApplication interface {
    RequestHandler(req *http.Request) error
    ResponseHandler(resp *http.Response) error
}

// onExitFlushLoop is a callback set by tests to detect the state of the
// flushLoop() goroutine.
var onExitFlushLoop func()

// ErrResponseShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrResponseShortWrite = errors.New("short write")

/*
 *  HTTP reverse proxy using the standard library reverse proxy
 *
 *    target:
 */
type HttpReverseProxy struct {
    // Director must be a function that modified the request into a new
    // request to be sent using Transport. Its response is then copied
    // back to the client
    Director func(*http.Request)

    // The transport used to perform proxy requests.
    // If nil, http.DefaultTransport is used
    Transport *ProxyTransport

    // FlushInterval specifies the flush interval to flush to the client
    // while copying the response body.
    // If zero, no periodic flushing is done
    FlushInterval time.Duration

    // ErrorLog specifies an optional logger for errors that occur when
    // attempting to process the request. If nil, logging goes to os.Stderr
    // using the standard logger
    ErrorLog *log.Logger

    // BufferPool specifies a buffer pool to get byte slices for use by
    // io.CopyBuffer when copying HTTP request and response bodies
    BufferPool *bpool.BytePool

    // The application that is processes the HTTP data as it is
    // streamed to/from the server being proxied
    app    HttpApplication
}

func NewHttpReverseProxy(target *url.URL, app HttpApplication) (*HttpReverseProxy, error) {

    pool := bpool.NewBytePool(20, 524288)
    director := func(req *http.Request) {
        targetQuery := target.RawQuery
        req.URL.Scheme = target.Scheme
        req.URL.Host = target.Host
        if targetQuery == "" || req.URL.RawQuery == "" {
            req.URL.RawQuery = targetQuery + req.URL.RawQuery
        } else {
            req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
        }
        //update the host header in the request
        req.Host = target.Host
    }

    return &HttpReverseProxy{
        Director: director,
        Transport: &ProxyTransport{},
        BufferPool: pool,
        app: app,
    }, nil
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func (p *HttpReverseProxy) Start() error {
    fmt.Printf("Starting Object Store service on port 8080\n");
    http.ListenAndServe(":8080", p)
    return nil
}

// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te",      // canonicalized version of "TE"
	"Trailer", // not Trailers per URL above; http://www.rfc-editor.org/errata_search.php?eid=4522
	"Transfer-Encoding",
	"Upgrade",
}

func (p *HttpReverseProxy) processRequest(req *http.Request) (*http.Response, error) {
    transport := p.Transport

    err := transport.WriteHeader(req)
    if err != nil {
        fmt.Printf("proxy: Failed to send Request Headers to backend: %s\n", err)
        return nil, err
    }

    written := 0
    // write the body out, if there is one
    if req.Body != nil {
        src := req.Body
        fmt.Printf("Content Length is %d\n", req.Header.Get("Content-Length"))

        var buf []byte
        for {
            buf := p.BufferPool.Get()
            nr, err := src.Read(buf)
            if nr > 0 {
                nw, err := transport.Write(req, buf[0:nr])
                if err != nil {
                    fmt.Printf("Error: Writing request body\n")
                    break
                }
                if nw != nr {
                    fmt.Printf("Error: ShortRequestWrite\n")
                    break
                }
                written += nw
                fmt.Printf("Written %d bytes\n", written)
            }
            if err == io.EOF {
                break
            }
        }
        p.BufferPool.Put(buf)
    }
    resp, err := transport.ReadResponse(req)
    return resp, err
}

func (p *HttpReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {

	outreq := new(http.Request)
	*outreq = *req // includes shallow copies of maps, but okay
    if req.ContentLength == 0 {
        outreq.Body = nil
    }

	p.Director(outreq)
	outreq.Proto = "HTTP/1.1"
	outreq.ProtoMajor = 1
	outreq.ProtoMinor = 1
	outreq.Close = false

    // Remove hop-by-hop headers listed in the "Connection" header.
	// See RFC 2616, section 14.10.
	copiedHeaders := false
	if c := outreq.Header.Get("Connection"); c != "" {
		for _, f := range strings.Split(c, ",") {
			if f = strings.TrimSpace(f); f != "" {
				if !copiedHeaders {
					outreq.Header = make(http.Header)
					copyHeader(outreq.Header, req.Header)
					copiedHeaders = true
				}
				outreq.Header.Del(f)
			}
		}
	}
	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		if outreq.Header.Get(h) != "" {
			if !copiedHeaders {
				outreq.Header = make(http.Header)
				copyHeader(outreq.Header, req.Header)
				copiedHeaders = true
			}
			outreq.Header.Del(h)
		}
	}

    //now call the app handlers if registered
    //p.app.RequestHandler(outreq)

    //send this request on its way out
    res, err := p.processRequest(outreq)

    //res, err := transport.RoundTrip(outreq)
	if err != nil {
        p.logf("http: proxy error: %v", err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}
	defer res.Body.Close()

	for _, h := range hopHeaders {
		res.Header.Del(h)
	}

    //now call the app handlers if registered
    p.app.ResponseHandler(res)

	copyHeader(rw.Header(), res.Header)

	rw.WriteHeader(res.StatusCode)
	p.copyResponse(rw, res.Body)
}

func (p *HttpReverseProxy) copyResponse(dst io.Writer, src io.Reader) (written int64, err error) {
	if p.FlushInterval != 0 {
		if wf, ok := dst.(writeFlusher); ok {
			mlw := &maxLatencyWriter{
				dst:     wf,
				latency: p.FlushInterval,
				done:    make(chan bool),
			}
			go mlw.flushLoop()
			defer mlw.stop()
			dst = mlw
		}
	}
    var buf []byte
	for {
        buf = p.BufferPool.Get()
		nr, er := src.Read(buf)
		if nr > 0 {
            // invoke the application callback, if registered
            // app may modify the request and return a different
            // buffer. Also buffer here needs to be larger than
            // the data being read, so that app handler can
            // modify contents and return update buffer.
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = ErrResponseShortWrite
				break
			}
		}
		if er == io.EOF {
			break
		}
		if er != nil {
			err = er
			break
		}
	}
    p.BufferPool.Put(buf)
    return written, err
}

func (p *HttpReverseProxy) logf(format string, args ...interface{}) {
	if p.ErrorLog != nil {
		p.ErrorLog.Printf(format, args...)
	} else {
		log.Printf(format, args...)
	}
}

type writeFlusher interface {
	io.Writer
	http.Flusher
}

type maxLatencyWriter struct {
	dst     writeFlusher
	latency time.Duration

	lk   sync.Mutex // protects Write + Flush
	done chan bool
}

func (m *maxLatencyWriter) Write(p []byte) (int, error) {
	m.lk.Lock()
	defer m.lk.Unlock()
	return m.dst.Write(p)
}

func (m *maxLatencyWriter) flushLoop() {
	t := time.NewTicker(m.latency)
	defer t.Stop()
	for {
		select {
		case <-m.done:
			if onExitFlushLoop != nil {
				onExitFlushLoop()
			}
			return
		case <-t.C:
			m.lk.Lock()
			m.dst.Flush()
			m.lk.Unlock()
		}
	}
}

func (m *maxLatencyWriter) stop() {
    m.done <- true
}
<|endoftext|>