<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"
)

const packageTemplateString = `<!DOCTYPE html>

		<meta charset="utf-8">
		<title>{{.Repo.PackageName}}.{{.Repo.MajorVersion}}{{.Repo.SubPath}} - {{.Repo.GopkgPath}}</title>
		<link href='//fonts.googleapis.com/css?family=Ubuntu+Mono|Ubuntu' rel='stylesheet' >
		<link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet" >
		<link href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet" >

{{.Repo.GopkgPath}}

						 Source Code
						 API Documentation

Getting started

To get the package, execute:

go get {{.Repo.GopkgPath}}

To import this package, add the following line to your code:

import "{{.Repo.GopkgPath}}"

								{{if .CleanPackageName}}
Refer to it as {{.CleanPackageName}}.{{end}}

								{{if .Synopsis}}
{{.Synopsis}}
{{end}}

For more details, see the API documentation.

Versions

						{{ if .LatestVersions }}
							{{ range .LatestVersions }}

									v{{.Major}}
									→
									{{.}}

							{{ end }}
						{{ else }}

								v0
								→
								master

						{{ end }}

gopkg.in

`

var packageTemplate *template.Template

func gopkgVersionRoot(repo *Repo, version Version) string {
	return repo.GopkgVersionRoot(version)
}

var packageFuncs = template.FuncMap{
	"gopkgVersionRoot": gopkgVersionRoot,
}

func init() {
	var err error
	packageTemplate, err = template.New("page").Funcs(packageFuncs).Parse(packageTemplateString)
	if err != nil {
		fmt.Fprintf(os.Stderr, "fatal: parsing package template failed: %s\n", err)
		os.Exit(1)
	}
}

type packageData struct {
	Repo             *Repo
	LatestVersions   VersionList // Contains only the latest version for each major
	FullVersion      Version     // Version that the major requested resolves to
	CleanPackageName string
	Synopsis         string
}

type gddoApiSynopsisResult struct {
	Results []struct {
		Path     string `json:"path"`
		Synopsis string `json:"synopsis"`
	} `json:"results"`
}

func renderPackagePage(resp http.ResponseWriter, req *http.Request, repo *Repo) {
	data := &packageData{
		Repo: repo,
	}

	// calculate version mapping
	latestVersionsMap := make(map[int]Version)
	for _, v := range repo.AllVersions {
		v2, exists := latestVersionsMap[v.Major]
		if !exists || v2.Less(v) {
			latestVersionsMap[v.Major] = v
		}
	}
	data.FullVersion = latestVersionsMap[repo.MajorVersion.Major]
	data.LatestVersions = make(VersionList, 0, len(latestVersionsMap))
	for _, v := range latestVersionsMap {
		data.LatestVersions = append(data.LatestVersions, v)
	}
	sort.Sort(sort.Reverse(data.LatestVersions))

	// find clean package name
	data.CleanPackageName = repo.PackageName
	if strings.HasPrefix(data.CleanPackageName, "go-") {
		data.CleanPackageName = data.CleanPackageName[3:]
	}
	if strings.HasSuffix(data.CleanPackageName, "-go") {
		data.CleanPackageName = data.CleanPackageName[:len(data.CleanPackageName)-3]
	}
	for i, c := range data.CleanPackageName {
		if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' {
			continue
		}
		if i > 0 && (c == '_' || c >= '0' && c <= '9') {
			continue
		}
		data.CleanPackageName = ""
		break
	}

	// retrieve synopsis
	gddoResp, err := http.Get(`http://api.godoc.org/search?q=` + url.QueryEscape(repo.GopkgPath()))
	if err == nil {
		synopsisResult := &gddoApiSynopsisResult{}
		err = json.NewDecoder(gddoResp.Body).Decode(&synopsisResult)
		gddoResp.Body.Close()
		if err == nil {
			for _, apiPkg := range synopsisResult.Results {
				if apiPkg.Path == repo.GopkgPath() {
					data.Synopsis = apiPkg.Synopsis
					break
				}
			}
		}
	}

	err = packageTemplate.Execute(resp, data)
	if err != nil {
		log.Printf("error executing tmplPackage: %s\n", err)
	}
}
<commit_msg>Some styling changes<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"
)

const packageTemplateString = `<!DOCTYPE html>

		<meta charset="utf-8">
		<title>{{.Repo.PackageName}}.{{.Repo.MajorVersion}}{{.Repo.SubPath}} - {{.Repo.GopkgPath}}</title>
		<link href='//fonts.googleapis.com/css?family=Ubuntu+Mono|Ubuntu' rel='stylesheet' >
		<link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet" >
		<link href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet" >

{{.Repo.GopkgPath}}

						{{if .Synopsis}}
{{.Synopsis}}
{{end}}

						 Source Code
						 API Documentation

Getting started

To get the package, execute:

go get {{.Repo.GopkgPath}}

To import this package, add the following line to your code:

import "{{.Repo.GopkgPath}}"

								{{if .CleanPackageName}}
Refer to it as {{.CleanPackageName}}.{{end}}

For more details, see the API documentation.

Versions

						{{ if .LatestVersions }}
							{{ range .LatestVersions }}

									v{{.Major}}
									→
									{{.}}

							{{ end }}
						{{ else }}

								v0
								→
								master

						{{ end }}

gopkg.in

`

var packageTemplate *template.Template

func gopkgVersionRoot(repo *Repo, version Version) string {
	return repo.GopkgVersionRoot(version)
}

var packageFuncs = template.FuncMap{
	"gopkgVersionRoot": gopkgVersionRoot,
}

func init() {
	var err error
	packageTemplate, err = template.New("page").Funcs(packageFuncs).Parse(packageTemplateString)
	if err != nil {
		fmt.Fprintf(os.Stderr, "fatal: parsing package template failed: %s\n", err)
		os.Exit(1)
	}
}

type packageData struct {
	Repo             *Repo
	LatestVersions   VersionList // Contains only the latest version for each major
	FullVersion      Version     // Version that the major requested resolves to
	CleanPackageName string
	Synopsis         string
}

type gddoApiSynopsisResult struct {
	Results []struct {
		Path     string `json:"path"`
		Synopsis string `json:"synopsis"`
	} `json:"results"`
}

func renderPackagePage(resp http.ResponseWriter, req *http.Request, repo *Repo) {
	data := &packageData{
		Repo: repo,
	}

	// calculate version mapping
	latestVersionsMap := make(map[int]Version)
	for _, v := range repo.AllVersions {
		v2, exists := latestVersionsMap[v.Major]
		if !exists || v2.Less(v) {
			latestVersionsMap[v.Major] = v
		}
	}
	data.FullVersion = latestVersionsMap[repo.MajorVersion.Major]
	data.LatestVersions = make(VersionList, 0, len(latestVersionsMap))
	for _, v := range latestVersionsMap {
		data.LatestVersions = append(data.LatestVersions, v)
	}
	sort.Sort(sort.Reverse(data.LatestVersions))

	// find clean package name
	data.CleanPackageName = repo.PackageName
	if strings.HasPrefix(data.CleanPackageName, "go-") {
		data.CleanPackageName = data.CleanPackageName[3:]
	}
	if strings.HasSuffix(data.CleanPackageName, "-go") {
		data.CleanPackageName = data.CleanPackageName[:len(data.CleanPackageName)-3]
	}
	for i, c := range data.CleanPackageName {
		if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' {
			continue
		}
		if i > 0 && (c == '_' || c >= '0' && c <= '9') {
			continue
		}
		data.CleanPackageName = ""
		break
	}

	// retrieve synopsis
	str := `http://api.godoc.org/search?q=` + url.QueryEscape(repo.GopkgPath())
	fmt.Println(str)
	gddoResp, err := http.Get(str)
	if err == nil {
		synopsisResult := &gddoApiSynopsisResult{}
		err = json.NewDecoder(gddoResp.Body).Decode(&synopsisResult)
		gddoResp.Body.Close()
		if err == nil {
			for _, apiPkg := range synopsisResult.Results {
				if apiPkg.Path == repo.GopkgPath() {
					data.Synopsis = apiPkg.Synopsis
					break
				}
			}
		}
	}

	err = packageTemplate.Execute(resp, data)
	if err != nil {
		log.Printf("error executing tmplPackage: %s\n", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"

	"github.com/conformal/btcwire"
)

type Peer struct {
	client  *Client
	address string
	conn    net.Conn
	nonce   uint64 // Nonce we're sending to the peer
}

func NewPeer(client *Client, address string) *Peer {
	p := Peer{
		client:  client,
		address: address,
	}
	return &p
}

func (p *Peer) Connect() error {
	if p.conn != nil {
		return fmt.Errorf("Peer already connected, can't connect again.")
	}
	conn, err := net.Dial("tcp", p.address)
	if err != nil {
		return err
	}

	p.conn = conn
	return nil
}

func (p *Peer) Disconnect() {
	p.conn.Close()
	logger.Debugf("[%s] Closed.", p.address)
}

func (p *Peer) Handshake() error {
	if p.conn == nil {
		return fmt.Errorf("Peer is not connected, can't handshake.")
	}

	logger.Debugf("[%s] Starting handshake.", p.address)

	nonce, err := btcwire.RandomUint64()
	if err != nil {
		return err
	}
	p.nonce = nonce

	pver, btcnet := p.client.pver, p.client.btcnet

	msgVersion, err := btcwire.NewMsgVersionFromConn(p.conn, p.nonce, p.client.userAgent, 0)
	msgVersion.DisableRelayTx = true
	if err := btcwire.WriteMessage(p.conn, msgVersion, pver, btcnet); err != nil {
		return err
	}

	// Read the response version.
	msg, _, err := btcwire.ReadMessage(p.conn, pver, btcnet)
	if err != nil {
		return err
	}
	vmsg, ok := msg.(*btcwire.MsgVersion)
	if !ok {
		return fmt.Errorf("Did not receive version message: %T", vmsg)
	}
	// Negotiate protocol version.
	if uint32(vmsg.ProtocolVersion) < pver {
		pver = uint32(vmsg.ProtocolVersion)
	}
	logger.Debugf("[%s] -> Version: %s", p.address, vmsg.UserAgent)

	// Normally we'd check if vmsg.Nonce == p.nonce but the crawler does not
	// accept external connections so we skip it.

	// Send verack.
	if err := btcwire.WriteMessage(p.conn, btcwire.NewMsgVerAck(), pver, btcnet); err != nil {
		return err
	}

	return nil
}

func (p *Peer) WriteMessage(msg btcwire.Message) error {
	return btcwire.WriteMessage(p.conn, btcwire.NewMsgGetAddr(), p.client.pver, p.client.btcnet)
}

func (p *Peer) ReadMessage() (btcwire.Message, []byte, error) {
	return btcwire.ReadMessage(p.conn, p.client.pver, p.client.btcnet)
}
<commit_msg>Refactor peer.<commit_after>package main

import (
	"fmt"
	"net"

	"github.com/conformal/btcwire"
)

type Peer struct {
	client  *Client
	address string
	conn    net.Conn
	nonce   uint64 // Nonce we're sending to the peer
}

func NewPeer(client *Client, address string) *Peer {
	p := Peer{
		client:  client,
		address: address,
	}
	return &p
}

func (p *Peer) Connect() error {
	if p.conn != nil {
		return fmt.Errorf("Peer already connected, can't connect again.")
	}
	conn, err := net.Dial("tcp", p.address)
	if err != nil {
		return err
	}

	p.conn = conn
	return nil
}

func (p *Peer) Disconnect() {
	p.conn.Close()
	logger.Debugf("[%s] Closed.", p.address)
}

func (p *Peer) Handshake() error {
	if p.conn == nil {
		return fmt.Errorf("Peer is not connected, can't handshake.")
	}

	logger.Debugf("[%s] Starting handshake.", p.address)

	nonce, err := btcwire.RandomUint64()
	if err != nil {
		return err
	}
	p.nonce = nonce

	pver, btcnet := p.client.pver, p.client.btcnet

	msgVersion, err := btcwire.NewMsgVersionFromConn(p.conn, p.nonce, p.client.userAgent, 0)
	msgVersion.DisableRelayTx = true
	if err := btcwire.WriteMessage(p.conn, msgVersion, pver, btcnet); err != nil {
		return err
	}

	// Read the response version.
	msg, _, err := btcwire.ReadMessage(p.conn, pver, btcnet)
	if err != nil {
		return err
	}
	vmsg, ok := msg.(*btcwire.MsgVersion)
	if !ok {
		return fmt.Errorf("Did not receive version message: %T", vmsg)
	}
	// Negotiate protocol version.
	if uint32(vmsg.ProtocolVersion) < pver {
		pver = uint32(vmsg.ProtocolVersion)
	}
	logger.Debugf("[%s] -> Version: %s", p.address, vmsg.UserAgent)

	// Normally we'd check if vmsg.Nonce == p.nonce but the crawler does not
	// accept external connections so we skip it.

	// Send verack.
	if err := btcwire.WriteMessage(p.conn, btcwire.NewMsgVerAck(), pver, btcnet); err != nil {
		return err
	}

	return nil
}

func (p *Peer) WriteMessage(msg btcwire.Message) error {
	return btcwire.WriteMessage(p.conn, msg, p.client.pver, p.client.btcnet)
}

func (p *Peer) ReadMessage() (btcwire.Message, []byte, error) {
	return btcwire.ReadMessage(p.conn, p.client.pver, p.client.btcnet)
}
<|endoftext|>


<commit_before>package pget

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/asaskevich/govalidator"
	"github.com/pkg/errors"
)

// Pget structs
type Pget struct {
	Trace  bool
	Output string
	Procs  int
	URLs   []string

	args      []string
	timeout   int
	useragent string
	referer   string
}

// New for pget package
func New() *Pget {
	return &Pget{
		Trace:   false,
		Procs:   runtime.NumCPU(), // default
		timeout: 10,
	}
}

// Run execute methods in pget package
func (pget *Pget) Run(ctx context.Context, version string, args []string) error {
	if err := pget.Ready(version, args); err != nil {
		return errTop(err)
	}

	client := newDownloadClient(16)

	target, err := Check(ctx, &CheckConfig{
		URLs:    pget.URLs,
		Timeout: time.Duration(pget.timeout) * time.Second,
		Client:  client,
	})
	if err != nil {
		return err
	}

	filename := target.Filename

	var dir string
	if pget.Output != "" {
		fi, err := os.Stat(pget.Output)
		if err == nil && fi.IsDir() {
			dir = pget.Output
		} else {
			dir, filename = filepath.Split(pget.Output)
			if dir != "" {
				if err := os.MkdirAll(dir, 0755); err != nil {
					return errors.Wrapf(err, "failed to create diretory at %s", dir)
				}
			}
		}
	}

	opts := []DownloadOption{
		WithUserAgent(pget.useragent),
		WithReferer(pget.referer),
	}

	return Download(ctx, &DownloadConfig{
		Filename:      filename,
		Dirname:       dir,
		ContentLength: target.ContentLength,
		Procs:         pget.Procs,
		URLs:          target.URLs,
		Client:        client,
	}, opts...)
}

// Ready method define the variables required to Download.
func (pget *Pget) Ready(version string, args []string) error {
	if procs := os.Getenv("GOMAXPROCS"); procs == "" {
		runtime.GOMAXPROCS(pget.Procs)
	}

	opts, err := pget.parseOptions(args, version)
	if err != nil {
		return errors.Wrap(errTop(err), "failed to parse command line args")
	}

	if opts.Trace {
		pget.Trace = opts.Trace
	}

	if opts.Procs > 2 {
		pget.Procs = opts.Procs
	}

	if opts.Timeout > 0 {
		pget.timeout = opts.Timeout
	}

	if err := pget.parseURLs(); err != nil {
		return errors.Wrap(err, "failed to parse of url")
	}

	if opts.Output != "" {
		pget.Output = opts.Output
	}

	if opts.UserAgent != "" {
		pget.useragent = opts.UserAgent
	}

	if opts.Referer != "" {
		pget.referer = opts.Referer
	}

	return nil
}

func (pget *Pget) parseOptions(argv []string, version string) (*Options, error) {
	var opts Options
	if len(argv) == 0 {
		stdout.Write(opts.usage(version))
		return nil, makeIgnoreErr()
	}

	o, err := opts.parse(argv, version)
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse command line options")
	}

	if opts.Help {
		stdout.Write(opts.usage(version))
		return nil, makeIgnoreErr()
	}

	if opts.Update {
		result, err := opts.isupdate(version)
		if err != nil {
			return nil, errors.Wrap(err, "failed to parse command line options")
		}

		stdout.Write(result)
		return nil, makeIgnoreErr()
	}

	pget.args = o

	return &opts, nil
}

func (pget *Pget) parseURLs() error {

	// find url in args
	for _, argv := range pget.args {
		if govalidator.IsURL(argv) {
			pget.URLs = append(pget.URLs, argv)
		}
	}

	if len(pget.URLs) < 1 {
		fmt.Fprintf(stdout, "Please input url separate with space or newline\n")
		fmt.Fprintf(stdout, "Start download with ^D\n")

		// scanning url from stdin
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			scan := scanner.Text()
			urls := strings.Split(scan, " ")
			for _, url := range urls {
				if govalidator.IsURL(url) {
					pget.URLs = append(pget.URLs, url)
				}
			}
		}

		if err := scanner.Err(); err != nil {
			return errors.Wrap(err, "failed to parse url from stdin")
		}

		if len(pget.URLs) < 1 {
			return errors.New("urls not found in the arguments passed")
		}
	}

	return nil
}
<commit_msg>added comment<commit_after>package pget

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/asaskevich/govalidator"
	"github.com/pkg/errors"
)

// Pget structs
type Pget struct {
	Trace  bool
	Output string
	Procs  int
	URLs   []string

	args      []string
	timeout   int
	useragent string
	referer   string
}

// New for pget package
func New() *Pget {
	return &Pget{
		Trace:   false,
		Procs:   runtime.NumCPU(), // default
		timeout: 10,
	}
}

// Run execute methods in pget package
func (pget *Pget) Run(ctx context.Context, version string, args []string) error {
	if err := pget.Ready(version, args); err != nil {
		return errTop(err)
	}

	// TODO(codehex): calc maxIdleConnsPerHost
	client := newDownloadClient(16)

	target, err := Check(ctx, &CheckConfig{
		URLs:    pget.URLs,
		Timeout: time.Duration(pget.timeout) * time.Second,
		Client:  client,
	})
	if err != nil {
		return err
	}

	filename := target.Filename

	var dir string
	if pget.Output != "" {
		fi, err := os.Stat(pget.Output)
		if err == nil && fi.IsDir() {
			dir = pget.Output
		} else {
			dir, filename = filepath.Split(pget.Output)
			if dir != "" {
				if err := os.MkdirAll(dir, 0755); err != nil {
					return errors.Wrapf(err, "failed to create diretory at %s", dir)
				}
			}
		}
	}

	opts := []DownloadOption{
		WithUserAgent(pget.useragent),
		WithReferer(pget.referer),
	}

	return Download(ctx, &DownloadConfig{
		Filename:      filename,
		Dirname:       dir,
		ContentLength: target.ContentLength,
		Procs:         pget.Procs,
		URLs:          target.URLs,
		Client:        client,
	}, opts...)
}

// Ready method define the variables required to Download.
func (pget *Pget) Ready(version string, args []string) error {
	if procs := os.Getenv("GOMAXPROCS"); procs == "" {
		runtime.GOMAXPROCS(pget.Procs)
	}

	opts, err := pget.parseOptions(args, version)
	if err != nil {
		return errors.Wrap(errTop(err), "failed to parse command line args")
	}

	if opts.Trace {
		pget.Trace = opts.Trace
	}

	if opts.Procs > 2 {
		pget.Procs = opts.Procs
	}

	if opts.Timeout > 0 {
		pget.timeout = opts.Timeout
	}

	if err := pget.parseURLs(); err != nil {
		return errors.Wrap(err, "failed to parse of url")
	}

	if opts.Output != "" {
		pget.Output = opts.Output
	}

	if opts.UserAgent != "" {
		pget.useragent = opts.UserAgent
	}

	if opts.Referer != "" {
		pget.referer = opts.Referer
	}

	return nil
}

func (pget *Pget) parseOptions(argv []string, version string) (*Options, error) {
	var opts Options
	if len(argv) == 0 {
		stdout.Write(opts.usage(version))
		return nil, makeIgnoreErr()
	}

	o, err := opts.parse(argv, version)
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse command line options")
	}

	if opts.Help {
		stdout.Write(opts.usage(version))
		return nil, makeIgnoreErr()
	}

	if opts.Update {
		result, err := opts.isupdate(version)
		if err != nil {
			return nil, errors.Wrap(err, "failed to parse command line options")
		}

		stdout.Write(result)
		return nil, makeIgnoreErr()
	}

	pget.args = o

	return &opts, nil
}

func (pget *Pget) parseURLs() error {

	// find url in args
	for _, argv := range pget.args {
		if govalidator.IsURL(argv) {
			pget.URLs = append(pget.URLs, argv)
		}
	}

	if len(pget.URLs) < 1 {
		fmt.Fprintf(stdout, "Please input url separate with space or newline\n")
		fmt.Fprintf(stdout, "Start download with ^D\n")

		// scanning url from stdin
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			scan := scanner.Text()
			urls := strings.Split(scan, " ")
			for _, url := range urls {
				if govalidator.IsURL(url) {
					pget.URLs = append(pget.URLs, url)
				}
			}
		}

		if err := scanner.Err(); err != nil {
			return errors.Wrap(err, "failed to parse url from stdin")
		}

		if len(pget.URLs) < 1 {
			return errors.New("urls not found in the arguments passed")
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/gopacket"
	"code.google.com/p/gopacket/layers"
	"code.google.com/p/gopacket/pcap"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"runtime/pprof"
	"sync"
	"time"
)

var (
	dcache dns_cache
)

type dns_cache struct {
	mu             sync.Mutex
	resolved_names map[string]string
}

func (d *dns_cache) Put(key string, value string) {
	d.mu.Lock()
	d.resolved_names[key] = value
	d.mu.Unlock()
}

func (d *dns_cache) Get(key string) string {
	result := ""
	d.mu.Lock()
	if r, ok := d.resolved_names[key]; ok {
		result = r
	}
	d.mu.Unlock()
	return result
}

// Used for generating network names for Nodes
type Network struct {
	CIDR  string
	Name  string
	ipnet *net.IPNet
}

// Used for gathering most of the data
type Object struct {
	SourceIP   string
	SourceName string
	TargetIP   string
	TargetName string
	Service    string
	Protocol   string
	Packages   int
	Bytes      int
	Syns       int
	Fins       int
}

// Helper struct for write_nodes_to_file
type Node struct {
	ID      string
	Label   string
	Network string
}

// Used in tracking the duration of connections
type Connection struct {
	SourceIP string
	TargetIP string
	Service  string
	Sequence uint32
	SynTime  time.Time
	FinTime  time.Time
}

// Tries to reverse resolve ip address. Returns addr if unsuccessful.
func lookup_addr(addr string) string {
	result := dcache.Get(addr)
	if result == "" {
		addrs, _ := net.LookupAddr(addr)
		if addrs != nil && len(addrs) > 0 {
			result = addrs[0]
			dcache.Put(addr, addrs[0])
		}
	}
	if result == "" {
		result = addr
	}
	return result
}

// Gets source and target IPs from packet, if it is either IPv4 or IPv6
func get_ips(packet *gopacket.Packet) (string, string) {
	src := ""
	tgt := ""

	// IPv4
	if ipv4_layer := (*packet).Layer(layers.LayerTypeIPv4); ipv4_layer != nil {
		ipv4, _ := ipv4_layer.(*layers.IPv4)
		src = ipv4.SrcIP.String()
		tgt = ipv4.DstIP.String()
	} else // Can't be both v4 and v6!

	// IPv6
	if ipv6_layer := (*packet).Layer(layers.LayerTypeIPv6); ipv6_layer != nil {
		ipv6, _ := ipv6_layer.(*layers.IPv6)
		src = ipv6.SrcIP.String()
		tgt = ipv6.DstIP.String()
	}

	return src, tgt
}

// Read the packets from file, gather some very basic information, and forward the information to other goroutines.
// Does not check for pcap parsing errors, but those should be rare for the most common layer types...
func read_packets(messages chan int, cs chan Object, cs2 chan gopacket.Packet, in_filename string) {
	if handle, err := pcap.OpenOffline(in_filename); err != nil {
		panic(err)
	} else {
		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
		packetSource.Lazy = true
		packetSource.NoCopy = true
		for packet := range packetSource.Packets() {
			result := Object{
				Packages: 1,
				Bytes:    packet.Metadata().Length}

			result.SourceIP, result.TargetIP = get_ips(&packet)

			// TCP
			if tcp_layer := packet.Layer(layers.LayerTypeTCP); tcp_layer != nil {
				tcp, _ := tcp_layer.(*layers.TCP)
				// Calls String(), which may return "number(name)" or "number"
				result.Service = tcp.DstPort.String()
				result.Protocol = "TCP"
				if tcp.SYN {
					result.Syns = 1
				}
				if tcp.FIN {
					result.Fins = 1
				}
			} else // Try UDP next

			// UDP
			if udp_layer := packet.Layer(layers.LayerTypeUDP); udp_layer != nil {
				udp, _ := udp_layer.(*layers.UDP)
				// Calls String(), which may return "number(name)" or "number"
				result.Service = udp.DstPort.String()
				result.Protocol = "UDP"
			} else // Try SCTP next

			// SCTP
			if sctp_layer := packet.Layer(layers.LayerTypeSCTP); sctp_layer != nil {
				sctp, _ := sctp_layer.(*layers.SCTP)
				// Calls String(), which may return "number(name)" or "number"
				result.Service = sctp.DstPort.String()
				result.Protocol = "SCTP"
			}

			// Only forward this packet, if we got the basic information about the packet
			// This rules out packages such as dhcp
			if result.SourceIP != "" && result.TargetIP != "" && result.Protocol != "" {
				cs <- result
				cs2 <- packet
			}

		}
	}

	fmt.Printf("Read all packets\n")
	messages <- 1
	close(cs)
	close(cs2)
}

// Aggregate statistics for edges
func count_packets(messages chan int, cs chan Object, packets map[string]Object) {
	for packet := range cs {
		key := packet.SourceIP + "," + packet.TargetIP + "," + packet.Service + "," + packet.Protocol

		// Exists already?
		if p, ok := packets[key]; ok {
			// Existed, add just Packages + Bytes + Syns + Fins
			p.Bytes = p.Bytes + packet.Bytes
			p.Packages = p.Packages + 1
			p.Syns = p.Syns + packet.Syns
			p.Fins = p.Fins + packet.Fins
			packets[key] = p
		} else {
			// Did not exist, add
			packets[key] = packet
		}
	}
	fmt.Printf("Counted basic statistics\n")
	messages <- 2
}

// Tracks TCP connection statuses
func track_tcp_connections(messages chan int, cs chan gopacket.Packet, conns *[]Connection) {
	for packet := range cs {

		src, tgt := get_ips(&packet)
		svc := ""
		ts := packet.Metadata().Timestamp
		seq := uint32(0)
		syn := false
		fin := false

		// Gather info from TCP layer
		if tcp_layer := packet.Layer(layers.LayerTypeTCP); tcp_layer != nil {
			tcp, _ := tcp_layer.(*layers.TCP)
			// Calls String(), which may return "number(name)" or "number"
			svc = tcp.DstPort.String()
			seq = tcp.Seq
			if tcp.SYN {
				syn = true

			}
			if tcp.FIN {
				fin = true
			}

			// Handle package information
			if syn {
				// New, so make a new connection
				conn := Connection{
					SynTime:  ts,
					SourceIP: src,
					TargetIP: tgt,
					Service:  svc,
					Sequence: seq}
				// Add to connections
				*conns = append(*conns, conn)
			} else {
				// Try to find matching package
				// .. This takes account only packages to one direction, but that should be okayish
				// (there should be a matching FIN anyways, discarding only the second one doesn't usually change the end result)
				for i := 0; i < len(*conns); i++ {
					if (*conns)[i].SourceIP == src && (*conns)[i].TargetIP == tgt && (*conns)[i].Service == svc && (*conns)[i].Sequence+1 == seq {
						// Update sequence, and if fin the timestamp for that
						(*conns)[i].Sequence = seq
						if fin {
							(*conns)[i].FinTime = ts
						}
					}
				}
			}
		}
	}

	fmt.Printf("Tracking TCP status ready\n")
	messages <- 3
}

// Calculate average time (TCP, limited by generated input data) connections on edge have taken
// Process only connections that have both start and end timestamps
func get_avg_tcp(conns []Connection, packet Object) int {
	result := 0

	matching := []Connection{}
	// First let's get all matching connections
	for _, x := range conns {
		if packet.SourceIP == x.SourceIP && packet.TargetIP == x.TargetIP && !x.SynTime.IsZero() && !x.FinTime.IsZero() {
			matching = append(matching, x)
		}
	}

	// If we found any, calculate avg
	if len(matching) > 0 {
		sum := int64(0)
		for _, k := range matching {
			diff := k.FinTime.Sub(k.SynTime)
			diffms := diff.Nanoseconds() / 1000000
			sum = sum + diffms
		}
		result = int(sum / int64(len(matching)))
	}

	return result
}

// Writes node information to file, in format Gephi knows how to import
func write_nodes_to_file(out_filename string, packets map[string]Object, networks []Network) {
	f, err := os.Create(out_filename)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	// Headers
	f.WriteString("Id,Label,Network\n")

	// Get all mentioned nodes, remove duplicates..
	nodes := make(map[string]*Node)
	for _, packet := range packets {
		nodes[packet.SourceIP] = &(Node{ID: packet.SourceIP})
		nodes[packet.TargetIP] = &(Node{ID: packet.TargetIP})
	}

	// Parallel lookups
	var wg sync.WaitGroup
	for x, _ := range nodes {
		wg.Add(1)
		go func(n *Node, networks []Network) {
			// Decrement counter
			defer wg.Done()
			// Lookup
			n.Label = lookup_addr(n.ID)
			n.Network = get_network(networks, n.ID)
		}(nodes[x], networks)
	}
	// Wait for all lookups to complete
	wg.Wait()

	// Print the nodes
	for x, _ := range nodes {
		if nodes[x].ID != "" {
			f.WriteString(fmt.Sprintf("%v,%v,%v\n", nodes[x].ID, nodes[x].Label, nodes[x].Network))
		}
	}
}

// Writes edge information to file, in format Gephi knows how to import
func write_edges_to_file(out_filename string, packets map[string]Object, conns []Connection) {
	f, err := os.Create(out_filename)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	// Headers
	f.WriteString("Source,Target,Type,Label,Protocol,Weight,Bytes,Packages,SYNs,FINs,Unfinished,Avg\n")

	for _, packet := range packets {
		unf := 0
		avg := 0
		if packet.Protocol == "TCP" {
			unf = packet.Syns - packet.Fins
			// It's possible we missed some in the capture itself...
			// There's no point reporting negative number
			if unf < 0 {
				unf = 0
			}
			// Average time TCP connections took on this edge
			avg = get_avg_tcp(conns, packet)
		}
		f.WriteString(fmt.Sprintf("%v,%v,Directed,%v,%v,%v,%v,%v,%v,%v,%v,%v\n",
			packet.SourceIP, packet.TargetIP, packet.Service, packet.Protocol,
			packet.Bytes, packet.Bytes, packet.Packages, packet.Syns, packet.Fins, unf, avg))
	}

}

// Names the network the IP belongs to, if the name is known
func get_network(networks []Network, ip_str string) string {
	result := ""

	if ip := net.ParseIP(ip_str); ip != nil {
		for x := 0; x < len(networks); x++ {
			if networks[x].ipnet.Contains(ip) {
				result = networks[x].Name
			}
		}
	}

	return result
}

// Reads networks information in JSON format for naming networks of Nodes
func read_networks(networks_filename string) []Network {
	networks := []Network{}

	// Reading the string versions
	if networks_filename != "" {
		net_file, err := ioutil.ReadFile(networks_filename)
		if err != nil {
			panic(err)
		}

		err = json.Unmarshal(net_file, &networks)
		if err != nil {
			panic(err)
		}
	}

	// Generating IPNets
	// Alternatively could have implemented custom JSON unmarshaler
	for x, _ := range networks {
		_, net, err := net.ParseCIDR(networks[x].CIDR)
		if err != nil {
			panic(err)
		}

		networks[x].ipnet = net
	}

	return networks
}

func main() {
	var pcap_file = flag.String("in", "", "pcap file")
	var nodes_file = flag.String("nodes", "", "node file")
	var edges_file = flag.String("edges", "", "edge file")
	var networks_file = flag.String("networks", "", "networks file")
	var cpu_profile = flag.String("cpuprofile", "", "cpu profile to file")

	flag.Parse()

	if *pcap_file == "" || *nodes_file == "" || *edges_file == "" {
		flag.PrintDefaults()
		return
	}

	if *cpu_profile != "" {
		// Enable profiling
		f, err := os.Create(*cpu_profile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	dcache.resolved_names = make(map[string]string)
	/*
	   var (
	   	dcache dns_cache
	   )

	   type dns_cache struct {
	   	mu             sync.Mutex
	   	resolved_names map[string]string
	   }

	*/
	nets := read_networks(*networks_file)

	conns := []Connection{}

	objs := make(map[string]Object)
	cs := make(chan Object, 1000000)
	cs2 := make(chan gopacket.Packet, 1000000)
	messages := make(chan int)

	// Read packets and their basic information
	go read_packets(messages, cs, cs2, *pcap_file)
	// Aggregate basic statistics
	go count_packets(messages, cs, objs)
	// Track the TCP connection durations
	go track_tcp_connections(messages, cs2, &conns)

	for i := 0; i < 3; i++ {
		// Wait for every goroutine to signal something
		<-messages
	}

	// Easier approach than attempting to do this with channels
	write_nodes_to_file(*nodes_file, objs, nets)
	fmt.Printf("Writing nodes finished\n")
	write_edges_to_file(*edges_file, objs, conns)
	fmt.Printf("Writing edges finished\n")
}
<commit_msg>Fixed location of Google's GoPacket library (was moved to github)<commit_after>package main

// Corrected location of gopacket 9/29/17
import (
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"runtime/pprof"
	"sync"
	"time"
)

var (
	dcache dns_cache
)

type dns_cache struct {
	mu             sync.Mutex
	resolved_names map[string]string
}

func (d *dns_cache) Put(key string, value string) {
	d.mu.Lock()
	d.resolved_names[key] = value
	d.mu.Unlock()
}

func (d *dns_cache) Get(key string) string {
	result := ""
	d.mu.Lock()
	if r, ok := d.resolved_names[key]; ok {
		result = r
	}
	d.mu.Unlock()
	return result
}

// Used for generating network names for Nodes
type Network struct {
	CIDR  string
	Name  string
	ipnet *net.IPNet
}

// Used for gathering most of the data
type Object struct {
	SourceIP   string
	SourceName string
	TargetIP   string
	TargetName string
	Service    string
	Protocol   string
	Packages   int
	Bytes      int
	Syns       int
	Fins       int
}

// Helper struct for write_nodes_to_file
type Node struct {
	ID      string
	Label   string
	Network string
}

// Used in tracking the duration of connections
type Connection struct {
	SourceIP string
	TargetIP string
	Service  string
	Sequence uint32
	SynTime  time.Time
	FinTime  time.Time
}

// Tries to reverse resolve ip address. Returns addr if unsuccessful.
func lookup_addr(addr string) string {
	result := dcache.Get(addr)
	if result == "" {
		addrs, _ := net.LookupAddr(addr)
		if addrs != nil && len(addrs) > 0 {
			result = addrs[0]
			dcache.Put(addr, addrs[0])
		}
	}
	if result == "" {
		result = addr
	}
	return result
}

// Gets source and target IPs from packet, if it is either IPv4 or IPv6
func get_ips(packet *gopacket.Packet) (string, string) {
	src := ""
	tgt := ""

	// IPv4
	if ipv4_layer := (*packet).Layer(layers.LayerTypeIPv4); ipv4_layer != nil {
		ipv4, _ := ipv4_layer.(*layers.IPv4)
		src = ipv4.SrcIP.String()
		tgt = ipv4.DstIP.String()
	} else // Can't be both v4 and v6!

	// IPv6
	if ipv6_layer := (*packet).Layer(layers.LayerTypeIPv6); ipv6_layer != nil {
		ipv6, _ := ipv6_layer.(*layers.IPv6)
		src = ipv6.SrcIP.String()
		tgt = ipv6.DstIP.String()
	}

	return src, tgt
}

// Read the packets from file, gather some very basic information, and forward the information to other goroutines.
// Does not check for pcap parsing errors, but those should be rare for the most common layer types...
func read_packets(messages chan int, cs chan Object, cs2 chan gopacket.Packet, in_filename string) {
	if handle, err := pcap.OpenOffline(in_filename); err != nil {
		panic(err)
	} else {
		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
		packetSource.Lazy = true
		packetSource.NoCopy = true
		for packet := range packetSource.Packets() {
			result := Object{
				Packages: 1,
				Bytes:    packet.Metadata().Length}

			result.SourceIP, result.TargetIP = get_ips(&packet)

			// TCP
			if tcp_layer := packet.Layer(layers.LayerTypeTCP); tcp_layer != nil {
				tcp, _ := tcp_layer.(*layers.TCP)
				// Calls String(), which may return "number(name)" or "number"
				result.Service = tcp.DstPort.String()
				result.Protocol = "TCP"
				if tcp.SYN {
					result.Syns = 1
				}
				if tcp.FIN {
					result.Fins = 1
				}
			} else // Try UDP next

			// UDP
			if udp_layer := packet.Layer(layers.LayerTypeUDP); udp_layer != nil {
				udp, _ := udp_layer.(*layers.UDP)
				// Calls String(), which may return "number(name)" or "number"
				result.Service = udp.DstPort.String()
				result.Protocol = "UDP"
			} else // Try SCTP next

			// SCTP
			if sctp_layer := packet.Layer(layers.LayerTypeSCTP); sctp_layer != nil {
				sctp, _ := sctp_layer.(*layers.SCTP)
				// Calls String(), which may return "number(name)" or "number"
				result.Service = sctp.DstPort.String()
				result.Protocol = "SCTP"
			}

			// Only forward this packet, if we got the basic information about the packet
			// This rules out packages such as dhcp
			if result.SourceIP != "" && result.TargetIP != "" && result.Protocol != "" {
				cs <- result
				cs2 <- packet
			}

		}
	}

	fmt.Printf("Read all packets\n")
	messages <- 1
	close(cs)
	close(cs2)
}

// Aggregate statistics for edges
func count_packets(messages chan int, cs chan Object, packets map[string]Object) {
	for packet := range cs {
		key := packet.SourceIP + "," + packet.TargetIP + "," + packet.Service + "," + packet.Protocol

		// Exists already?
		if p, ok := packets[key]; ok {
			// Existed, add just Packages + Bytes + Syns + Fins
			p.Bytes = p.Bytes + packet.Bytes
			p.Packages = p.Packages + 1
			p.Syns = p.Syns + packet.Syns
			p.Fins = p.Fins + packet.Fins
			packets[key] = p
		} else {
			// Did not exist, add
			packets[key] = packet
		}
	}
	fmt.Printf("Counted basic statistics\n")
	messages <- 2
}

// Tracks TCP connection statuses
func track_tcp_connections(messages chan int, cs chan gopacket.Packet, conns *[]Connection) {
	for packet := range cs {

		src, tgt := get_ips(&packet)
		svc := ""
		ts := packet.Metadata().Timestamp
		seq := uint32(0)
		syn := false
		fin := false

		// Gather info from TCP layer
		if tcp_layer := packet.Layer(layers.LayerTypeTCP); tcp_layer != nil {
			tcp, _ := tcp_layer.(*layers.TCP)
			// Calls String(), which may return "number(name)" or "number"
			svc = tcp.DstPort.String()
			seq = tcp.Seq
			if tcp.SYN {
				syn = true

			}
			if tcp.FIN {
				fin = true
			}

			// Handle package information
			if syn {
				// New, so make a new connection
				conn := Connection{
					SynTime:  ts,
					SourceIP: src,
					TargetIP: tgt,
					Service:  svc,
					Sequence: seq}
				// Add to connections
				*conns = append(*conns, conn)
			} else {
				// Try to find matching package
				// .. This takes account only packages to one direction, but that should be okayish
				// (there should be a matching FIN anyways, discarding only the second one doesn't usually change the end result)
				for i := 0; i < len(*conns); i++ {
					if (*conns)[i].SourceIP == src && (*conns)[i].TargetIP == tgt && (*conns)[i].Service == svc && (*conns)[i].Sequence+1 == seq {
						// Update sequence, and if fin the timestamp for that
						(*conns)[i].Sequence = seq
						if fin {
							(*conns)[i].FinTime = ts
						}
					}
				}
			}
		}
	}

	fmt.Printf("Tracking TCP status ready\n")
	messages <- 3
}

// Calculate average time (TCP, limited by generated input data) connections on edge have taken
// Process only connections that have both start and end timestamps
func get_avg_tcp(conns []Connection, packet Object) int {
	result := 0

	matching := []Connection{}
	// First let's get all matching connections
	for _, x := range conns {
		if packet.SourceIP == x.SourceIP && packet.TargetIP == x.TargetIP && !x.SynTime.IsZero() && !x.FinTime.IsZero() {
			matching = append(matching, x)
		}
	}

	// If we found any, calculate avg
	if len(matching) > 0 {
		sum := int64(0)
		for _, k := range matching {
			diff := k.FinTime.Sub(k.SynTime)
			diffms := diff.Nanoseconds() / 1000000
			sum = sum + diffms
		}
		result = int(sum / int64(len(matching)))
	}

	return result
}

// Writes node information to file, in format Gephi knows how to import
func write_nodes_to_file(out_filename string, packets map[string]Object, networks []Network) {
	f, err := os.Create(out_filename)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	// Headers
	f.WriteString("Id,Label,Network\n")

	// Get all mentioned nodes, remove duplicates..
	nodes := make(map[string]*Node)
	for _, packet := range packets {
		nodes[packet.SourceIP] = &(Node{ID: packet.SourceIP})
		nodes[packet.TargetIP] = &(Node{ID: packet.TargetIP})
	}

	// Parallel lookups
	var wg sync.WaitGroup
	for x, _ := range nodes {
		wg.Add(1)
		go func(n *Node, networks []Network) {
			// Decrement counter
			defer wg.Done()
			// Lookup
			n.Label = lookup_addr(n.ID)
			n.Network = get_network(networks, n.ID)
		}(nodes[x], networks)
	}
	// Wait for all lookups to complete
	wg.Wait()

	// Print the nodes
	for x, _ := range nodes {
		if nodes[x].ID != "" {
			f.WriteString(fmt.Sprintf("%v,%v,%v\n", nodes[x].ID, nodes[x].Label, nodes[x].Network))
		}
	}
}

// Writes edge information to file, in format Gephi knows how to import
func write_edges_to_file(out_filename string, packets map[string]Object, conns []Connection) {
	f, err := os.Create(out_filename)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	// Headers
	f.WriteString("Source,Target,Type,Label,Protocol,Weight,Bytes,Packages,SYNs,FINs,Unfinished,Avg\n")

	for _, packet := range packets {
		unf := 0
		avg := 0
		if packet.Protocol == "TCP" {
			unf = packet.Syns - packet.Fins
			// It's possible we missed some in the capture itself...
			// There's no point reporting negative number
			if unf < 0 {
				unf = 0
			}
			// Average time TCP connections took on this edge
			avg = get_avg_tcp(conns, packet)
		}
		f.WriteString(fmt.Sprintf("%v,%v,Directed,%v,%v,%v,%v,%v,%v,%v,%v,%v\n",
			packet.SourceIP, packet.TargetIP, packet.Service, packet.Protocol,
			packet.Bytes, packet.Bytes, packet.Packages, packet.Syns, packet.Fins, unf, avg))
	}

}

// Names the network the IP belongs to, if the name is known
func get_network(networks []Network, ip_str string) string {
	result := ""

	if ip := net.ParseIP(ip_str); ip != nil {
		for x := 0; x < len(networks); x++ {
			if networks[x].ipnet.Contains(ip) {
				result = networks[x].Name
			}
		}
	}

	return result
}

// Reads networks information in JSON format for naming networks of Nodes
func read_networks(networks_filename string) []Network {
	networks := []Network{}

	// Reading the string versions
	if networks_filename != "" {
		net_file, err := ioutil.ReadFile(networks_filename)
		if err != nil {
			panic(err)
		}

		err = json.Unmarshal(net_file, &networks)
		if err != nil {
			panic(err)
		}
	}

	// Generating IPNets
	// Alternatively could have implemented custom JSON unmarshaler
	for x, _ := range networks {
		_, net, err := net.ParseCIDR(networks[x].CIDR)
		if err != nil {
			panic(err)
		}

		networks[x].ipnet = net
	}

	return networks
}

func main() {
	var pcap_file = flag.String("in", "", "pcap file")
	var nodes_file = flag.String("nodes", "", "node file")
	var edges_file = flag.String("edges", "", "edge file")
	var networks_file = flag.String("networks", "", "networks file")
	var cpu_profile = flag.String("cpuprofile", "", "cpu profile to file")

	flag.Parse()

	if *pcap_file == "" || *nodes_file == "" || *edges_file == "" {
		flag.PrintDefaults()
		return
	}

	if *cpu_profile != "" {
		// Enable profiling
		f, err := os.Create(*cpu_profile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	dcache.resolved_names = make(map[string]string)
	/*
	   var (
	   	dcache dns_cache
	   )

	   type dns_cache struct {
	   	mu             sync.Mutex
	   	resolved_names map[string]string
	   }

	*/
	nets := read_networks(*networks_file)

	conns := []Connection{}

	objs := make(map[string]Object)
	cs := make(chan Object, 1000000)
	cs2 := make(chan gopacket.Packet, 1000000)
	messages := make(chan int)

	// Read packets and their basic information
	go read_packets(messages, cs, cs2, *pcap_file)
	// Aggregate basic statistics
	go count_packets(messages, cs, objs)
	// Track the TCP connection durations
	go track_tcp_connections(messages, cs2, &conns)

	for i := 0; i < 3; i++ {
		// Wait for every goroutine to signal something
		<-messages
	}

	// Easier approach than attempting to do this with channels
	write_nodes_to_file(*nodes_file, objs, nets)
	fmt.Printf("Writing nodes finished\n")
	write_edges_to_file(*edges_file, objs, conns)
	fmt.Printf("Writing edges finished\n")
}
<|endoftext|>


<commit_before>package GoHttpService

import (
	"net/http"
)

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 400. This is useful for telling a caller that the
request is malformed.
*/
func BadRequest(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 400)
}

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 500. This is useful for indicating some type of
error ocurred during processing.
*/
func Error(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 500)
}

/*
Convenience method sending our a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 403. This is useful in telling a caller that some
type of authentication failed.
*/
func Forbidden(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 403)
}

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 404. This is useful in telling a caller that
the resources they are trying to get is not found.
*/
func NotFound(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 404)
}

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of TRUE. This will set the HTTP status
code to 200.
*/
func Success(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(true, message), 200)
}
<commit_msg>Added new method for sending a session expired response<commit_after>package GoHttpService

import (
	"net/http"
)

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 400. This is useful for telling a caller that the
request is malformed.
*/
func BadRequest(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 400)
}

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 500. This is useful for indicating some type of
error ocurred during processing.
*/
func Error(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 500)
}

/*
Convenience method sending our a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 403. This is useful in telling a caller that some
type of authentication failed.
*/
func Forbidden(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 403)
}

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of FALSE. This will set the HTTP status
code to 404. This is useful in telling a caller that
the resources they are trying to get is not found.
*/
func NotFound(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(false, message), 404)
}

/*
Convenience method for telling clients that a session has
expired. This sends a SessionExpiredResponse structure and
a status of 401 Unauthorized.
*/
func SessionExpired(writer http.ResponseWriter) {
	WriteJson(writer, NewSessionExpiredResponse(), 401)
}

/*
Convenience method sending out a BasicJsonResponse in JSON
format with a success of TRUE. This will set the HTTP status
code to 200.
*/
func Success(writer http.ResponseWriter, message string) {
	WriteJson(writer, NewBasicJsonResponse(true, message), 200)
}
<|endoftext|>


<commit_before>package xlistener

import (
	"net"
	"strconv"
	"sync"
	"syscall"

	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/xmetrics"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics/discard"
)

// netListen is the factory function for creating a net.Listener.  Defaults to net.Listen.  Only tests would change this variable.
var netListen = net.Listen

// Options defines the available options for configuring a listener
type Options struct {
	// Logger is the go-kit logger to use for output.  If unset, logging.DefaultLogger() is used.
	Logger log.Logger

	// MaxConnections is the maximum number of active connections the listener will permit.  If this
	// value is not positive, there is no limit to the number of connections.
	MaxConnections int

	// Rejected is is incremented each time the listener rejects a connection.  If unset, a go-kit discard Counter is used.
	Rejected xmetrics.Adder

	// Active is updated to reflect the current number of active connections.  If unset, a go-kit discard Gauge is used.
	Active xmetrics.Adder

	// Network is the network to listen on.  This value is only used if Next is unset.  Defaults to "tcp" if unset.
	Network string

	// Address is the address to listen on.  This value is only used if Next is unset.  Defaults to ":http" if unset.
	Address string

	// Next is the net.Listener to decorate.  If this field is set, Network and Address are ignored.
	Next net.Listener
}

// New constructs a new net.Listener using a set of options.
//
// If Next is set, that listener is decorated with connection limiting and other options specfied in Options.
// Otherwise, a new net.Listener is created, and that new listener is decorated.  Note that in the case
// where this function creates a new net.Listener, that listener will be occupying a port and should be cleaned
// up via Close() if higher level errors occur.
func New(o Options) (net.Listener, error) {
	if o.Logger == nil {
		o.Logger = logging.DefaultLogger()
	}

	var semaphore chan struct{}
	if o.MaxConnections > 0 {
		semaphore = make(chan struct{}, o.MaxConnections)
	}

	if o.Rejected == nil {
		o.Rejected = discard.NewCounter()
	}

	if o.Active == nil {
		o.Active = discard.NewGauge()
	}

	next := o.Next
	if next == nil {
		if len(o.Network) == 0 {
			o.Network = "tcp"
		}

		if len(o.Address) == 0 {
			o.Address = ":http"
		}

		var err error
		next, err = netListen(o.Network, o.Address)
		if err != nil {
			return nil, err
		}
	}

	return &listener{
		Listener:  next,
		logger:    log.With(o.Logger, "listenNetwork", next.Addr().Network(), "listenAddress", next.Addr().String()),
		semaphore: semaphore,
		rejected:  xmetrics.NewIncrementer(o.Rejected),
		active:    o.Active,
	}, nil
}

// listener decorates a net.Listener with metrics and optional maximum connection enforcement
type listener struct {
	net.Listener
	logger    log.Logger
	semaphore chan struct{}
	rejected  xmetrics.Incrementer
	active    xmetrics.Adder
}

// acquire attempts to obtain a semaphore resource.  If the semaphore has not been set (i.e. no maximum connections),
// this method immediately returns true.  Otherwise, the semaphore must be immediately acquired or this method returns false.
// In all cases, the active connections gauge is updated if appropriate.
func (l *listener) acquire() bool {
	if l.semaphore == nil {
		l.active.Add(1.0)
		return true
	}

	select {
	case l.semaphore <- struct{}{}:
		l.active.Add(1.0)
		return true
	default:
		return false
	}
}

// release returns a semaphore resource to the pool, if set.  This method also decrements the active connection gauge.
func (l *listener) release() {
	l.active.Add(-1.0)
	if l.semaphore != nil {
		<-l.semaphore
	}
}

// Accept invokes the delegate net.Listener's Accept method, then attempts to acquire the semaphore.
// If the semaphore was set and could not be acquired, the accepted connection is immediately closed.
func (l *listener) Accept() (net.Conn, error) {
	for {
		c, err := l.Listener.Accept()
		if err != nil {
			sysValue := ""
			if errno, ok := err.(syscall.Errno); ok {
				sysValue = "0x" + strconv.Itoa(int(errno))
			}

			l.logger.Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "failed to accept connection", logging.ErrorKey(), err, "sysValue", sysValue)
			if err == syscall.ENFILE {
				l.logger.Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "ENFILE received.  translating to EMFILE")
				return nil, syscall.EMFILE
			}

			return nil, err
		}

		if !l.acquire() {
			l.logger.Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "rejected connection", "remoteAddress", c.RemoteAddr().String())
			l.rejected.Inc()
			c.Close()
			continue
		}

		l.logger.Log(level.Key(), level.DebugValue(), logging.MessageKey(), "accepted connection", "remoteAddress", c.RemoteAddr().String())
		return &conn{Conn: c, release: l.release}, nil
	}
}

// conn is a decorated net.Conn that supplies feedback to a listener when the connection is closed.
type conn struct {
	net.Conn
	releaseOnce sync.Once
	release     func()
}

// Close closes the decorated connection and invokes release on the listener that created it.  The release
// operation is idempotent.
func (c *conn) Close() error {
	err := c.Conn.Close()
	c.releaseOnce.Do(c.release)
	return err
}
<commit_msg>Use the hex format, for easier lookup in godoc<commit_after>package xlistener

import (
	"net"
	"strconv"
	"sync"
	"syscall"

	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/xmetrics"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics/discard"
)

// netListen is the factory function for creating a net.Listener.  Defaults to net.Listen.  Only tests would change this variable.
var netListen = net.Listen

// Options defines the available options for configuring a listener
type Options struct {
	// Logger is the go-kit logger to use for output.  If unset, logging.DefaultLogger() is used.
	Logger log.Logger

	// MaxConnections is the maximum number of active connections the listener will permit.  If this
	// value is not positive, there is no limit to the number of connections.
	MaxConnections int

	// Rejected is is incremented each time the listener rejects a connection.  If unset, a go-kit discard Counter is used.
	Rejected xmetrics.Adder

	// Active is updated to reflect the current number of active connections.  If unset, a go-kit discard Gauge is used.
	Active xmetrics.Adder

	// Network is the network to listen on.  This value is only used if Next is unset.  Defaults to "tcp" if unset.
	Network string

	// Address is the address to listen on.  This value is only used if Next is unset.  Defaults to ":http" if unset.
	Address string

	// Next is the net.Listener to decorate.  If this field is set, Network and Address are ignored.
	Next net.Listener
}

// New constructs a new net.Listener using a set of options.
//
// If Next is set, that listener is decorated with connection limiting and other options specfied in Options.
// Otherwise, a new net.Listener is created, and that new listener is decorated.  Note that in the case
// where this function creates a new net.Listener, that listener will be occupying a port and should be cleaned
// up via Close() if higher level errors occur.
func New(o Options) (net.Listener, error) {
	if o.Logger == nil {
		o.Logger = logging.DefaultLogger()
	}

	var semaphore chan struct{}
	if o.MaxConnections > 0 {
		semaphore = make(chan struct{}, o.MaxConnections)
	}

	if o.Rejected == nil {
		o.Rejected = discard.NewCounter()
	}

	if o.Active == nil {
		o.Active = discard.NewGauge()
	}

	next := o.Next
	if next == nil {
		if len(o.Network) == 0 {
			o.Network = "tcp"
		}

		if len(o.Address) == 0 {
			o.Address = ":http"
		}

		var err error
		next, err = netListen(o.Network, o.Address)
		if err != nil {
			return nil, err
		}
	}

	return &listener{
		Listener:  next,
		logger:    log.With(o.Logger, "listenNetwork", next.Addr().Network(), "listenAddress", next.Addr().String()),
		semaphore: semaphore,
		rejected:  xmetrics.NewIncrementer(o.Rejected),
		active:    o.Active,
	}, nil
}

// listener decorates a net.Listener with metrics and optional maximum connection enforcement
type listener struct {
	net.Listener
	logger    log.Logger
	semaphore chan struct{}
	rejected  xmetrics.Incrementer
	active    xmetrics.Adder
}

// acquire attempts to obtain a semaphore resource.  If the semaphore has not been set (i.e. no maximum connections),
// this method immediately returns true.  Otherwise, the semaphore must be immediately acquired or this method returns false.
// In all cases, the active connections gauge is updated if appropriate.
func (l *listener) acquire() bool {
	if l.semaphore == nil {
		l.active.Add(1.0)
		return true
	}

	select {
	case l.semaphore <- struct{}{}:
		l.active.Add(1.0)
		return true
	default:
		return false
	}
}

// release returns a semaphore resource to the pool, if set.  This method also decrements the active connection gauge.
func (l *listener) release() {
	l.active.Add(-1.0)
	if l.semaphore != nil {
		<-l.semaphore
	}
}

// Accept invokes the delegate net.Listener's Accept method, then attempts to acquire the semaphore.
// If the semaphore was set and could not be acquired, the accepted connection is immediately closed.
func (l *listener) Accept() (net.Conn, error) {
	for {
		c, err := l.Listener.Accept()
		if err != nil {
			sysValue := ""
			if errno, ok := err.(syscall.Errno); ok {
				sysValue = "0x" + strconv.FormatInt(int64(errno), 16)
			}

			l.logger.Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "failed to accept connection", logging.ErrorKey(), err, "sysValue", sysValue)
			if err == syscall.ENFILE {
				l.logger.Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "ENFILE received.  translating to EMFILE")
				return nil, syscall.EMFILE
			}

			return nil, err
		}

		if !l.acquire() {
			l.logger.Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "rejected connection", "remoteAddress", c.RemoteAddr().String())
			l.rejected.Inc()
			c.Close()
			continue
		}

		l.logger.Log(level.Key(), level.DebugValue(), logging.MessageKey(), "accepted connection", "remoteAddress", c.RemoteAddr().String())
		return &conn{Conn: c, release: l.release}, nil
	}
}

// conn is a decorated net.Conn that supplies feedback to a listener when the connection is closed.
type conn struct {
	net.Conn
	releaseOnce sync.Once
	release     func()
}

// Close closes the decorated connection and invokes release on the listener that created it.  The release
// operation is idempotent.
func (c *conn) Close() error {
	err := c.Conn.Close()
	c.releaseOnce.Do(c.release)
	return err
}
<|endoftext|>


<commit_before>package elasticthought

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"sync"

	"github.com/couchbaselabs/logg"
	"github.com/tleyden/go-couch"
)

// A classify job tries to classify images given by user against
// the given trained model
type ClassifyJob struct {
	ElasticThoughtDoc
	ProcessingState ProcessingState `json:"processing-state"`
	ProcessingLog   string          `json:"processing-log"`
	StdOutUrl       string          `json:"std-out-url"`
	StdErrUrl       string          `json:"std-err-url"`
	ClassifierID    string          `json:"classifier-id"`

	// Key: image url of image in cbfs
	// Value: the classification result for that image
	Results map[string]string `json:"results"`

	// had to make exported, due to https://github.com/gin-gonic/gin/pull/123
	// waiting for this to get merged into master branch, since go get
	// pulls from master branch.
	Configuration Configuration
}

// Create a new classify job.  If you don't use this, you must set the
// embedded ElasticThoughtDoc Type field.
func NewClassifyJob(c Configuration) *ClassifyJob {
	return &ClassifyJob{
		ElasticThoughtDoc: ElasticThoughtDoc{Type: DOC_TYPE_CLASSIFY_JOB},
		Configuration:     c,
	}
}

// Run this job
func (c *ClassifyJob) Run(wg *sync.WaitGroup) {

	defer wg.Done()

	logg.LogTo("CLASSIFY_JOB", "Run() called!")

	updatedState, err := c.UpdateProcessingState(Processing)
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	if !updatedState {
		logg.LogTo("CLASSIFY_JOB", "%+v already processed.  Ignoring.", c)
		return
	}

	// TODO: add code to run job

	logg.LogTo("CLASSIFY_JOB", "lazily create dir.  images: %+v", c.Results)

	if err := c.createWorkDirectory(); err != nil {
		c.recordProcessingError(err)
		return
	}

	if err := c.downloadWorkAssets(); err != nil {
		c.recordProcessingError(err)
		return
	}

	// invoke caffe
	resultsMap, err := c.invokeCaffe()
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	// update classifyjob with results
	logg.LogTo("CLASSIFY_JOB", "resultsMap: %+v", resultsMap)
	_, err = c.SetResults(resultsMap)
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	logg.LogTo("CLASSIFY_JOB", "Finished")

}

func (c ClassifyJob) invokeCaffe() (map[string]string, error) {

	// call "python classifier.py"
	// build command args
	cmdArgs := []string{"classifier.py"}
	python := "python"

	// debugging
	logg.LogTo("CLASSIFY_JOB", "Running %v with args %v", python, cmdArgs)
	logg.LogTo("CLASSIFY_JOB", "Path %v", os.Getenv("PATH"))

	// TEMP DEBUGGING -- remove this
	exec.Command("ls", "-alh", "/usr/local/bin").Run()

	// explicitly check if caffe binary found on the PATH
	lookPathResult, err := exec.LookPath("python")
	if err != nil {
		logg.LogError(fmt.Errorf("python not found on path: %v", err))
	}
	logg.LogTo("CLASSIFY_JOB", "python found on path: %v", lookPathResult)

	// Create command, but don't actually run it yet
	cmd := exec.Command(python, cmdArgs...)

	// set the directory where the command will be run in (important
	// because we depend on relative file paths to work)
	cmd.Dir = c.getWorkDirectory()

	// run the command and save stdio to files and tee to stdio streams
	if err := runCmdTeeStdio(cmd, c.getStdOutPath(), c.getStdErrPath()); err != nil {
		return nil, err
	}

	// read from temp files and write to cbfs.
	// initially I tried to write the stdout/stderr streams directly
	// to cbfs, but ran into an error related to the io.Seeker interface.
	if err := c.saveCmdOutputToCbfs(c.getStdOutPath()); err != nil {
		return nil, fmt.Errorf("Could not save output to cbfs. Err: %v", err)
	}

	if err := c.saveCmdOutputToCbfs(c.getStdErrPath()); err != nil {
		return nil, fmt.Errorf("Could not save output to cbfs. Err: %v", err)
	}

	// read output.json file into map
	result := map[string]string{}
	resultFilePath := filepath.Join(c.getWorkDirectory(), "result.json")
	resultFile, err := os.Open(resultFilePath)
	if err != nil {
		return nil, err
	}
	jsonParser := json.NewDecoder(resultFile)
	if err = jsonParser.Decode(&result); err != nil {
		return nil, err
	}

	return result, nil

}

func (c ClassifyJob) getStdOutPath() string {
	return path.Join(c.getWorkDirectory(), "stdout")
}

func (c ClassifyJob) getStdErrPath() string {
	return path.Join(c.getWorkDirectory(), "stderr")
}

func (c ClassifyJob) getWorkDirectory() string {
	return filepath.Join(c.Configuration.WorkDirectory, c.Id)
}

func (c ClassifyJob) getWorkImagesDirectory() string {
	return filepath.Join(c.getWorkDirectory(), "images")
}

func (c ClassifyJob) saveCmdOutputToCbfs(sourcePath string) error {

	base := path.Base(sourcePath)
	destPath := fmt.Sprintf("%v/%v", c.Id, base)

	cbfsclient, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}

	if err := saveFileToCbfs(sourcePath, destPath, "text/plain", cbfsclient); err != nil {
		return err
	}

	return nil

}

// lazily create work dir
func (c ClassifyJob) createWorkDirectory() error {
	workDir := c.getWorkDirectory()
	logg.LogTo("CLASSIFY_JOB", "Creating dir: %v", workDir)
	if err := Mkdir(workDir); err != nil {
		return err
	}

	imagesSubdir := c.getWorkImagesDirectory()
	if err := Mkdir(imagesSubdir); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) getClassifier() (*Classifier, error) {

	classifier := NewClassifier(c.Configuration)
	err := classifier.Find(c.ClassifierID)
	return classifier, err
}

func (c ClassifyJob) getTrainingJob() (*TrainingJob, error) {

	classifier, err := c.getClassifier()
	if err != nil {
		return nil, err
	}

	trainingJob := NewTrainingJob(c.Configuration)
	err = trainingJob.Find(classifier.TrainingJobID)
	return trainingJob, err

}

func (c ClassifyJob) downloadWorkAssets() error {

	// caffe model
	if err := c.downloadCaffeModel(); err != nil {
		return err
	}

	// prototxt
	if err := c.downloadPrototxt(); err != nil {
		return err
	}

	// images
	if err := c.downloadImagesToClassify(); err != nil {
		return err
	}

	// python classify script
	workDirectory := c.getWorkDirectory()
	if err := c.copyPythonClassifier(workDirectory); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) copyPythonClassifier(destDirPath string) error {

	// find out $GOPATH env variable
	gopath := os.Getenv("GOPATH")

	// find path to python classifier file
	elasticThoughtRoot := path.Join(
		gopath,
		"src",
		"github.com",
		"tleyden",
		"elastic-thought",
	)

	pythonScript := path.Join(
		elasticThoughtRoot,
		"scripts",
		"python-classifier",
		"classifier.py",
	)

	destPath := path.Join(destDirPath, "classifier.py")

	if err := CopyFileContents(pythonScript, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadCaffeModel() error {

	// get training job id, instantiate object
	trainingJob, err := c.getTrainingJob()
	if err != nil {
		return err
	}

	// make sure that training job state == Finished
	if trainingJob.GetProcessingState() != FinishedSuccessfully {
		return fmt.Errorf("TrainingJob is not finished yet")
	}

	// make sure the trained model url starts with cbfs
	trainedModelUrl := trainingJob.TrainedModelUrl

	// download from cbfs to local file system
	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "caffe.model")
	if err := downloadFromCbfs(cbfs, trainedModelUrl, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadPrototxt() error {

	classifier, err := c.getClassifier()
	if err != nil {
		return err
	}

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "classifier.prototxt")
	if err := downloadFromCbfs(cbfs, classifier.SpecificationUrl, destPath); err != nil {
		return err
	}

	return nil
}

func (c ClassifyJob) downloadImagesToClassify() error {

	// for now, just download a single image
	// (if multiple images specified, all bust last will be clobbered)
	// TODO: fix this and download all images

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}

	i := 0
	for imageUrl, _ := range c.Results {

		// url will be cbfs://<classify_job_id>/<imageurl_sha1_hash>
		_, imageSha1Hash := path.Split(imageUrl)
		destPath := path.Join(c.getWorkImagesDirectory(), imageSha1Hash)
		if err := downloadFromCbfs(cbfs, imageUrl, destPath); err != nil {
			return err
		}
		i += 1
	}

	return nil

}

// Update the processing state to new state.
func (c *ClassifyJob) UpdateProcessingState(newState ProcessingState) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingState = newState
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingState == newState
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) SetResults(results map[string]string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.Results = results
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return reflect.DeepEqual(results, classifyJob.Results)
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) UpdateProcessingLog(val string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingLog = val
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingLog == val
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) casUpdate(updater func(*ClassifyJob), doneMetric func(ClassifyJob) bool) (bool, error) {

	db := c.Configuration.DbConnection()

	genUpdater := func(classifyJobPtr interface{}) {
		cjp := classifyJobPtr.(*ClassifyJob)
		updater(cjp)
	}

	genDoneMetric := func(classifyJobPtr interface{}) bool {
		cjp := classifyJobPtr.(*ClassifyJob)
		return doneMetric(*cjp)
	}

	refresh := func(classifyJobPtr interface{}) error {
		cjp := classifyJobPtr.(*ClassifyJob)
		return cjp.RefreshFromDB(db)
	}

	return casUpdate(db, c, genUpdater, genDoneMetric, refresh)

}

// Insert into database (only call this if you know it doesn't arleady exist,
// or else you'll end up w/ unwanted dupes)
func (c *ClassifyJob) Insert() error {

	db := c.Configuration.DbConnection()

	id, rev, err := db.Insert(c)
	if err != nil {
		err := fmt.Errorf("Error inserting classify job: %v.  Err: %v", c, err)
		return err
	}

	c.Id = id
	c.Revision = rev

	return nil

}

// CodeReview: duplication with RefreshFromDB in many places
func (c *ClassifyJob) RefreshFromDB(db couch.Database) error {
	classifyJob := ClassifyJob{}
	err := db.Retrieve(c.Id, &classifyJob)
	if err != nil {
		return err
	}
	*c = classifyJob
	return nil
}

// Find a classify Job in the db with the given id, or error if not found
// CodeReview: duplication with Find in many places
func (c *ClassifyJob) Find(id string) error {
	db := c.Configuration.DbConnection()
	c.Id = id
	if err := c.RefreshFromDB(db); err != nil {
		return err
	}
	return nil
}

// Codereview: de-dupe
func (c ClassifyJob) recordProcessingError(err error) {
	logg.LogError(err)
	db := c.Configuration.DbConnection()
	if err := c.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting training job as failed: %v", err)
		logg.LogError(errMsg)
	}
}

func (c ClassifyJob) Failed(db couch.Database, processingErr error) error {

	_, err := c.UpdateProcessingState(Failed)
	if err != nil {
		return err
	}

	logg.LogTo("CLASSIFY_JOB", "updating processing log")

	logValue := fmt.Sprintf("%v", processingErr)
	_, err = c.UpdateProcessingLog(logValue)
	if err != nil {
		return err
	}

	return nil

}
<commit_msg>update processing state<commit_after>package elasticthought

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"sync"

	"github.com/couchbaselabs/logg"
	"github.com/tleyden/go-couch"
)

// A classify job tries to classify images given by user against
// the given trained model
type ClassifyJob struct {
	ElasticThoughtDoc
	ProcessingState ProcessingState `json:"processing-state"`
	ProcessingLog   string          `json:"processing-log"`
	StdOutUrl       string          `json:"std-out-url"`
	StdErrUrl       string          `json:"std-err-url"`
	ClassifierID    string          `json:"classifier-id"`

	// Key: image url of image in cbfs
	// Value: the classification result for that image
	Results map[string]string `json:"results"`

	// had to make exported, due to https://github.com/gin-gonic/gin/pull/123
	// waiting for this to get merged into master branch, since go get
	// pulls from master branch.
	Configuration Configuration
}

// Create a new classify job.  If you don't use this, you must set the
// embedded ElasticThoughtDoc Type field.
func NewClassifyJob(c Configuration) *ClassifyJob {
	return &ClassifyJob{
		ElasticThoughtDoc: ElasticThoughtDoc{Type: DOC_TYPE_CLASSIFY_JOB},
		Configuration:     c,
	}
}

// Run this job
func (c *ClassifyJob) Run(wg *sync.WaitGroup) {

	defer wg.Done()

	logg.LogTo("CLASSIFY_JOB", "Run() called!")

	updatedState, err := c.UpdateProcessingState(Processing)
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	if !updatedState {
		logg.LogTo("CLASSIFY_JOB", "%+v already processed.  Ignoring.", c)
		return
	}

	// TODO: add code to run job

	logg.LogTo("CLASSIFY_JOB", "lazily create dir.  images: %+v", c.Results)

	if err := c.createWorkDirectory(); err != nil {
		c.recordProcessingError(err)
		return
	}

	if err := c.downloadWorkAssets(); err != nil {
		c.recordProcessingError(err)
		return
	}

	// invoke caffe
	resultsMap, err := c.invokeCaffe()
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	// update classifyjob with results
	logg.LogTo("CLASSIFY_JOB", "resultsMap: %+v", resultsMap)
	_, err = c.SetResults(resultsMap)
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	// set job state to finished
	_, err = c.UpdateProcessingState(FinishedSuccessfully)
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	logg.LogTo("CLASSIFY_JOB", "Finished")

}

func (c ClassifyJob) invokeCaffe() (map[string]string, error) {

	// call "python classifier.py"
	// build command args
	cmdArgs := []string{"classifier.py"}
	python := "python"

	// debugging
	logg.LogTo("CLASSIFY_JOB", "Running %v with args %v", python, cmdArgs)
	logg.LogTo("CLASSIFY_JOB", "Path %v", os.Getenv("PATH"))

	// TEMP DEBUGGING -- remove this
	exec.Command("ls", "-alh", "/usr/local/bin").Run()

	// explicitly check if caffe binary found on the PATH
	lookPathResult, err := exec.LookPath("python")
	if err != nil {
		logg.LogError(fmt.Errorf("python not found on path: %v", err))
	}
	logg.LogTo("CLASSIFY_JOB", "python found on path: %v", lookPathResult)

	// Create command, but don't actually run it yet
	cmd := exec.Command(python, cmdArgs...)

	// set the directory where the command will be run in (important
	// because we depend on relative file paths to work)
	cmd.Dir = c.getWorkDirectory()

	// run the command and save stdio to files and tee to stdio streams
	if err := runCmdTeeStdio(cmd, c.getStdOutPath(), c.getStdErrPath()); err != nil {
		return nil, err
	}

	// read from temp files and write to cbfs.
	// initially I tried to write the stdout/stderr streams directly
	// to cbfs, but ran into an error related to the io.Seeker interface.
	if err := c.saveCmdOutputToCbfs(c.getStdOutPath()); err != nil {
		return nil, fmt.Errorf("Could not save output to cbfs. Err: %v", err)
	}

	if err := c.saveCmdOutputToCbfs(c.getStdErrPath()); err != nil {
		return nil, fmt.Errorf("Could not save output to cbfs. Err: %v", err)
	}

	// read output.json file into map
	result := map[string]string{}
	resultFilePath := filepath.Join(c.getWorkDirectory(), "result.json")
	resultFile, err := os.Open(resultFilePath)
	if err != nil {
		return nil, err
	}
	jsonParser := json.NewDecoder(resultFile)
	if err = jsonParser.Decode(&result); err != nil {
		return nil, err
	}

	return result, nil

}

func (c ClassifyJob) getStdOutPath() string {
	return path.Join(c.getWorkDirectory(), "stdout")
}

func (c ClassifyJob) getStdErrPath() string {
	return path.Join(c.getWorkDirectory(), "stderr")
}

func (c ClassifyJob) getWorkDirectory() string {
	return filepath.Join(c.Configuration.WorkDirectory, c.Id)
}

func (c ClassifyJob) getWorkImagesDirectory() string {
	return filepath.Join(c.getWorkDirectory(), "images")
}

func (c ClassifyJob) saveCmdOutputToCbfs(sourcePath string) error {

	base := path.Base(sourcePath)
	destPath := fmt.Sprintf("%v/%v", c.Id, base)

	cbfsclient, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}

	if err := saveFileToCbfs(sourcePath, destPath, "text/plain", cbfsclient); err != nil {
		return err
	}

	return nil

}

// lazily create work dir
func (c ClassifyJob) createWorkDirectory() error {
	workDir := c.getWorkDirectory()
	logg.LogTo("CLASSIFY_JOB", "Creating dir: %v", workDir)
	if err := Mkdir(workDir); err != nil {
		return err
	}

	imagesSubdir := c.getWorkImagesDirectory()
	if err := Mkdir(imagesSubdir); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) getClassifier() (*Classifier, error) {

	classifier := NewClassifier(c.Configuration)
	err := classifier.Find(c.ClassifierID)
	return classifier, err
}

func (c ClassifyJob) getTrainingJob() (*TrainingJob, error) {

	classifier, err := c.getClassifier()
	if err != nil {
		return nil, err
	}

	trainingJob := NewTrainingJob(c.Configuration)
	err = trainingJob.Find(classifier.TrainingJobID)
	return trainingJob, err

}

func (c ClassifyJob) downloadWorkAssets() error {

	// caffe model
	if err := c.downloadCaffeModel(); err != nil {
		return err
	}

	// prototxt
	if err := c.downloadPrototxt(); err != nil {
		return err
	}

	// images
	if err := c.downloadImagesToClassify(); err != nil {
		return err
	}

	// python classify script
	workDirectory := c.getWorkDirectory()
	if err := c.copyPythonClassifier(workDirectory); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) copyPythonClassifier(destDirPath string) error {

	// find out $GOPATH env variable
	gopath := os.Getenv("GOPATH")

	// find path to python classifier file
	elasticThoughtRoot := path.Join(
		gopath,
		"src",
		"github.com",
		"tleyden",
		"elastic-thought",
	)

	pythonScript := path.Join(
		elasticThoughtRoot,
		"scripts",
		"python-classifier",
		"classifier.py",
	)

	destPath := path.Join(destDirPath, "classifier.py")

	if err := CopyFileContents(pythonScript, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadCaffeModel() error {

	// get training job id, instantiate object
	trainingJob, err := c.getTrainingJob()
	if err != nil {
		return err
	}

	// make sure that training job state == Finished
	if trainingJob.GetProcessingState() != FinishedSuccessfully {
		return fmt.Errorf("TrainingJob is not finished yet")
	}

	// make sure the trained model url starts with cbfs
	trainedModelUrl := trainingJob.TrainedModelUrl

	// download from cbfs to local file system
	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "caffe.model")
	if err := downloadFromCbfs(cbfs, trainedModelUrl, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadPrototxt() error {

	classifier, err := c.getClassifier()
	if err != nil {
		return err
	}

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "classifier.prototxt")
	if err := downloadFromCbfs(cbfs, classifier.SpecificationUrl, destPath); err != nil {
		return err
	}

	return nil
}

func (c ClassifyJob) downloadImagesToClassify() error {

	// for now, just download a single image
	// (if multiple images specified, all bust last will be clobbered)
	// TODO: fix this and download all images

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}

	i := 0
	for imageUrl, _ := range c.Results {

		// url will be cbfs://<classify_job_id>/<imageurl_sha1_hash>
		_, imageSha1Hash := path.Split(imageUrl)
		destPath := path.Join(c.getWorkImagesDirectory(), imageSha1Hash)
		if err := downloadFromCbfs(cbfs, imageUrl, destPath); err != nil {
			return err
		}
		i += 1
	}

	return nil

}

// Update the processing state to new state.
func (c *ClassifyJob) UpdateProcessingState(newState ProcessingState) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingState = newState
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingState == newState
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) SetResults(results map[string]string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.Results = results
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return reflect.DeepEqual(results, classifyJob.Results)
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) UpdateProcessingLog(val string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingLog = val
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingLog == val
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) casUpdate(updater func(*ClassifyJob), doneMetric func(ClassifyJob) bool) (bool, error) {

	db := c.Configuration.DbConnection()

	genUpdater := func(classifyJobPtr interface{}) {
		cjp := classifyJobPtr.(*ClassifyJob)
		updater(cjp)
	}

	genDoneMetric := func(classifyJobPtr interface{}) bool {
		cjp := classifyJobPtr.(*ClassifyJob)
		return doneMetric(*cjp)
	}

	refresh := func(classifyJobPtr interface{}) error {
		cjp := classifyJobPtr.(*ClassifyJob)
		return cjp.RefreshFromDB(db)
	}

	return casUpdate(db, c, genUpdater, genDoneMetric, refresh)

}

// Insert into database (only call this if you know it doesn't arleady exist,
// or else you'll end up w/ unwanted dupes)
func (c *ClassifyJob) Insert() error {

	db := c.Configuration.DbConnection()

	id, rev, err := db.Insert(c)
	if err != nil {
		err := fmt.Errorf("Error inserting classify job: %v.  Err: %v", c, err)
		return err
	}

	c.Id = id
	c.Revision = rev

	return nil

}

// CodeReview: duplication with RefreshFromDB in many places
func (c *ClassifyJob) RefreshFromDB(db couch.Database) error {
	classifyJob := ClassifyJob{}
	err := db.Retrieve(c.Id, &classifyJob)
	if err != nil {
		return err
	}
	*c = classifyJob
	return nil
}

// Find a classify Job in the db with the given id, or error if not found
// CodeReview: duplication with Find in many places
func (c *ClassifyJob) Find(id string) error {
	db := c.Configuration.DbConnection()
	c.Id = id
	if err := c.RefreshFromDB(db); err != nil {
		return err
	}
	return nil
}

// Codereview: de-dupe
func (c ClassifyJob) recordProcessingError(err error) {
	logg.LogError(err)
	db := c.Configuration.DbConnection()
	if err := c.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting training job as failed: %v", err)
		logg.LogError(errMsg)
	}
}

func (c ClassifyJob) Failed(db couch.Database, processingErr error) error {

	_, err := c.UpdateProcessingState(Failed)
	if err != nil {
		return err
	}

	logg.LogTo("CLASSIFY_JOB", "updating processing log")

	logValue := fmt.Sprintf("%v", processingErr)
	_, err = c.UpdateProcessingLog(logValue)
	if err != nil {
		return err
	}

	return nil

}
<|endoftext|>


<commit_before>package google

import (
	"regexp"
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
)

// Destroy and recreate the mapping, testing that Terraform doesn't return a 409
func TestAccCloudRunDomainMapping_foregroundDeletion(t *testing.T) {
	t.Parallel()

	context := map[string]interface{}{
		"namespace":     getTestProjectFromEnv(),
		"random_suffix": randString(t, 10),
	}

	vcrTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		ExternalProviders: map[string]resource.ExternalProvider{
			"random": {},
		},
		CheckDestroy: testAccCheckCloudRunDomainMappingDestroyProducer(t),
		Steps: []resource.TestStep{
			{
				Config: testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated1(context),
			},
			{
				ResourceName:            "google_cloud_run_domain_mapping.default",
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"name", "location", "status", "metadata.0.resource_version"},
			},
			{
				Config:      testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated2(context),
				ExpectError: regexp.MustCompile("Domain mapping already exists"), // when this error is no longer returned, remove ExpectError and add the ISV below.
			},
			/*{
				ResourceName:            "google_cloud_run_domain_mapping.default",
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"name", "location", "status", "metadata.0.resource_version"},
			},*/
		},
	})
}

func testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated1(context map[string]interface{}) string {
	return Nprintf(`
resource "google_cloud_run_service" "default" {
    name     = "tf-test-cloudrun-srv%{random_suffix}"
    location = "us-central1"

    metadata {
      namespace = "%{namespace}"
    }

    template {
      spec {
        containers {
          image = "us-docker.pkg.dev/cloudrun/container/hello"
        }
      }
    }
  }

resource "google_cloud_run_domain_mapping" "default" {
  location = "us-central1"
  name     = "tf-test-domain%{random_suffix}.gcp.tfacc.hashicorptest.com"

  metadata {
    namespace = "%{namespace}"
  }

  spec {
    route_name = google_cloud_run_service.default.name
  }
}
`, context)
}

func testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated2(context map[string]interface{}) string {
	return Nprintf(`

resource "google_cloud_run_service" "default" {
  name     = "tf-test-cloudrun-srv%{random_suffix}"
  location = "us-central1"

  metadata {
    namespace = "%{namespace}"
  }

  template {
    spec {
      containers {
        image = "us-docker.pkg.dev/cloudrun/container/hello"
      }
    }
  }
}

resource "google_cloud_run_domain_mapping" "default" {
  location = "us-central1"
  name     = "tf-test-domain%{random_suffix}.gcp.tfacc.hashicorptest.com"

  metadata {
    namespace = "%{namespace}"
    labels = {
      "my-label" = "my-value"
    }
  }

  spec {
    route_name = google_cloud_run_service.default.name
  }
}
`, context)
}
<commit_msg>Remove test step as the error has been fixed in backend (#4654)<commit_after>package google

import (
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
)

// Destroy and recreate the mapping, testing that Terraform doesn't return a 409
func TestAccCloudRunDomainMapping_foregroundDeletion(t *testing.T) {
	t.Parallel()

	context := map[string]interface{}{
		"namespace":     getTestProjectFromEnv(),
		"random_suffix": randString(t, 10),
	}

	vcrTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		ExternalProviders: map[string]resource.ExternalProvider{
			"random": {},
		},
		CheckDestroy: testAccCheckCloudRunDomainMappingDestroyProducer(t),
		Steps: []resource.TestStep{
			{
				Config: testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated1(context),
			},
			{
				ResourceName:            "google_cloud_run_domain_mapping.default",
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"name", "location", "status", "metadata.0.resource_version"},
			},
			{
				Config:      testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated2(context),
			},
			{
				ResourceName:            "google_cloud_run_domain_mapping.default",
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"name", "location", "status", "metadata.0.resource_version"},
			},
		},
	})
}

func testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated1(context map[string]interface{}) string {
	return Nprintf(`
resource "google_cloud_run_service" "default" {
    name     = "tf-test-cloudrun-srv%{random_suffix}"
    location = "us-central1"

    metadata {
      namespace = "%{namespace}"
    }

    template {
      spec {
        containers {
          image = "us-docker.pkg.dev/cloudrun/container/hello"
        }
      }
    }
  }

resource "google_cloud_run_domain_mapping" "default" {
  location = "us-central1"
  name     = "tf-test-domain%{random_suffix}.gcp.tfacc.hashicorptest.com"

  metadata {
    namespace = "%{namespace}"
  }

  spec {
    route_name = google_cloud_run_service.default.name
  }
}
`, context)
}

func testAccCloudRunDomainMapping_cloudRunDomainMappingUpdated2(context map[string]interface{}) string {
	return Nprintf(`
resource "google_cloud_run_service" "default" {
  name     = "tf-test-cloudrun-srv%{random_suffix}"
  location = "us-central1"
  metadata {
    namespace = "%{namespace}"
  }
  template {
    spec {
      containers {
        image = "us-docker.pkg.dev/cloudrun/container/hello"
      }
    }
  }
}
resource "google_cloud_run_domain_mapping" "default" {
  location = "us-central1"
  name     = "tf-test-domain%{random_suffix}.gcp.tfacc.hashicorptest.com"
  metadata {
    namespace = "%{namespace}"
    labels = {
      "my-label" = "my-value"
    }
  }
  spec {
    route_name = google_cloud_run_service.default.name
  }
}
`, context)
}
<|endoftext|>


<commit_before>package db

const BLANK = ""

const (
	DELETE_OPERATION = 1
	INSERT_OPERATION = 2
	UPDATE_OPERATION = 3

	SentCollection       = "sent_notifications"
	StatsCollection      = "last_seen_at"
	TopicCollection      = "topics"
	GullyCollection      = "gullys"
	UserLocaleCollection = "user_locales"

	SavedEmailCollection     = "saved_email"
	SavedPushCollection      = "saved_push"
	SavedWebCollection       = "saved_web"
	AvailableTopicCollection = "topics_available"

	ProcessedCollection = "processed"
)

type Processed struct {
	BaseModel    `bson:",inline"`
	User         string `bson:"user"`
	Organization string `bson:"org"`
}

type AvailableTopic struct {
	BaseModel `bson:",inline"`
	Ident     string   `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
	AppName   string   `json:"app_name" bson:"app_name" required:"true" form:"app_name" binding:"required"`
	Channels  []string `json:"channels" bson:"channels" required:"true" form:"channels" binding:"required"`
}

type SentItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Text           string                 `json:"text" bson:"text" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Context        map[string]interface{} `json:"context" bson:"context"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

type SavedItem struct {
	BaseModel `bson:",inline"`
	Data      interface{} `bson:"data"`
	Details   PendingItem `bson:"details"`
}

type PendingItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	IsPending      bool                   `json:"is_pending" bson:"is_pending" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Context        map[string]interface{} `json:"context" bson:"context" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

func (self *PendingItem) IsValid() bool {
	if len(self.Context) == 0 {
		return false
	}
	return true
}

type LastSeen struct {
	BaseModel    `bson:",inline"`
	User         string `json:"user" bson:"user" required:"true"`
	Organization string `json:"org" bson:"org"`
	AppName      string `json:"app_name" bson:"app_name"`
	Timestamp    int64  `json:"timestamp" bson:"timestamp" required:"true"`
}

type Gully struct {
	BaseModel    `bson:",inline"`
	User         string                 `json:"user" bson:"user"`
	Organization string                 `json:"org" bson:"org"`
	AppName      string                 `json:"app_name" bson:"app_name"`
	Data         map[string]interface{} `json:"data" bson:"data" required:"true"`
	Ident        string                 `json:"ident" bson:"ident" required:"true"`
}

func (self *Gully) IsValid(op_type int) bool {

	if len(self.Ident) == 0 {
		return false
	}

	if op_type == INSERT_OPERATION {
		if len(self.Data) == 0 {
			return false
		}
	}

	return true
}

type UserLocale struct {
	BaseModel `bson:",inline"`
	User      string `json:"user" bson:"user" required:"true"`
	Locale    string `json:"locale" bson:"locale" required:"true"`
	TimeZone  string `json:"timezone" bson:"timezone" required:"true"`
}

func (self *UserLocale) IsValid() bool {
	if len(self.Locale) == 0 || len(self.User) == 0 || len(self.TimeZone) == 0 {
		return false
	}
	return true
}

type Channel struct {
	Name    string `json:"name" bson:"name"`
	Enabled bool   `json:"enabled" bson:"enabled"`
	Default bool   `json:"default" bson:"default"`
	Locked  bool   `json:"locked" bson:"locked"`
}

type Topic struct {
	BaseModel    `bson:",inline"`
	User         string    `json:"user" bson:"user"`
	Organization string    `json:"org" bson:"org"`
	Channels     []Channel `json:"channels" bson:"channels" required:"true"`
	Ident        string    `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
}
<commit_msg>remove gullys from models<commit_after>package db

const BLANK = ""

const (
	DELETE_OPERATION = 1
	INSERT_OPERATION = 2
	UPDATE_OPERATION = 3

	SentCollection       = "sent_notifications"
	StatsCollection      = "last_seen_at"
	TopicCollection      = "topics"
	GullyCollection      = "gullys"
	UserLocaleCollection = "user_locales"

	SavedEmailCollection     = "saved_email"
	SavedPushCollection      = "saved_push"
	SavedWebCollection       = "saved_web"
	AvailableTopicCollection = "topics_available"

	ProcessedCollection = "processed"
)

type Processed struct {
	BaseModel    `bson:",inline"`
	User         string `bson:"user"`
	Organization string `bson:"org"`
}

type AvailableTopic struct {
	BaseModel `bson:",inline"`
	Ident     string   `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
	AppName   string   `json:"app_name" bson:"app_name" required:"true" form:"app_name" binding:"required"`
	Channels  []string `json:"channels" bson:"channels" required:"true" form:"channels" binding:"required"`
}

type SentItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Text           string                 `json:"text" bson:"text" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Context        map[string]interface{} `json:"context" bson:"context"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

type SavedItem struct {
	BaseModel `bson:",inline"`
	Data      interface{} `bson:"data"`
	Details   PendingItem `bson:"details"`
}

type PendingItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	IsPending      bool                   `json:"is_pending" bson:"is_pending" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Context        map[string]interface{} `json:"context" bson:"context" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

func (self *PendingItem) IsValid() bool {
	if len(self.Context) == 0 {
		return false
	}
	return true
}

type LastSeen struct {
	BaseModel    `bson:",inline"`
	User         string `json:"user" bson:"user" required:"true"`
	Organization string `json:"org" bson:"org"`
	AppName      string `json:"app_name" bson:"app_name"`
	Timestamp    int64  `json:"timestamp" bson:"timestamp" required:"true"`
}

type UserLocale struct {
	BaseModel `bson:",inline"`
	User      string `json:"user" bson:"user" required:"true"`
	Locale    string `json:"locale" bson:"locale" required:"true"`
	TimeZone  string `json:"timezone" bson:"timezone" required:"true"`
}

func (self *UserLocale) IsValid() bool {
	if len(self.Locale) == 0 || len(self.User) == 0 || len(self.TimeZone) == 0 {
		return false
	}
	return true
}

type Channel struct {
	Name    string `json:"name" bson:"name"`
	Enabled bool   `json:"enabled" bson:"enabled"`
	Default bool   `json:"default" bson:"default"`
	Locked  bool   `json:"locked" bson:"locked"`
}

type Topic struct {
	BaseModel    `bson:",inline"`
	User         string    `json:"user" bson:"user"`
	Organization string    `json:"org" bson:"org"`
	Channels     []Channel `json:"channels" bson:"channels" required:"true"`
	Ident        string    `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
}
<|endoftext|>