<commit_before>package main

import (
	"bytes"
	"bufio"
	"flag"
	"fmt"
	"json"
	"os"
)

var newline = []uint8("\n")

func main() {
	bufIn := bufio.NewReader(fileFromArguments())
	lastLine := []uint8("")
	// One meg is good enough for anybody
	arr := make([]byte, 0, 1024*1024)
	buf := bytes.NewBuffer(arr)
	lineNum := int64(1)
	for {
		line, isPrefix, err := bufIn.ReadLine()
		if err != nil && err != os.EOF {
			genericError(err)
		}

		lastLine = append(lastLine, line...)
		if !isPrefix && len(lastLine) != 0 {
			indentAndPrint(buf, lastLine, lineNum)
			lineNum += 1
			lastLine = []uint8("")
		}

		if err == os.EOF {
			os.Exit(0)
		}
	}
}

func indentAndPrint(buf *bytes.Buffer, js []uint8, lineNum int64) {
	jsErr := json.Indent(buf, js, "", "  ")
	if (jsErr != nil) {
		malformedJson(jsErr, js, lineNum)
	}
	os.Stdout.Write(buf.Bytes())
	os.Stdout.Write(newline)
	buf.Reset()
}

func malformedJson(jsErr os.Error, js []uint8, lineNum int64) {
	os.Stdout.Sync()

	synErr, isSynError := (jsErr).(*json.SyntaxError)

	fmt.Fprintf(os.Stderr, "ERROR: Broken json on line %d: %s\n", lineNum, jsErr)

	if (isSynError) {
		prefix := ""
		suffix := ""

		begin := 0
		if synErr.Offset > 15 {
			begin = int(synErr.Offset - 15)
			prefix = "..."
		}

		end := begin + 30
		if end > len(js) {
			end = len(js)
		} else {
			suffix = "..."
		}

		fmt.Fprintf(os.Stderr, "  Context: %s%s%s\n", prefix, js[begin:end], suffix)
	}

	os.Exit(1)
}

func fileFromArguments() *os.File {
	flag.Parse()
	args := flag.Args()
	if len(args) > 1 {
		msg := fmt.Sprintf("only specify 0 or 1 files in the arguments, not %d\n", len(args))
		genericError(os.NewError(msg))
	}
	if len(args) == 0 {
		return os.Stdin
	}

	file, err := os.OpenFile(args[0], os.O_RDONLY, 0)
	if err != nil {
		genericError(err)
	}
	return file
}

func genericError(err os.Error) {
	os.Stdout.Sync()
	fmt.Fprintf(os.Stderr, "ERROR: %s", err)
	os.Exit(2)
}
<commit_msg>remove buffer comment<commit_after>package main

import (
	"bytes"
	"bufio"
	"flag"
	"fmt"
	"json"
	"os"
)

var newline = []uint8("\n")

func main() {
	bufIn := bufio.NewReader(fileFromArguments())
	lastLine := []uint8("")

	arr := make([]byte, 0, 1024*1024)
	buf := bytes.NewBuffer(arr)
	lineNum := int64(1)
	for {
		line, isPrefix, err := bufIn.ReadLine()
		if err != nil && err != os.EOF {
			genericError(err)
		}

		lastLine = append(lastLine, line...)
		if !isPrefix && len(lastLine) != 0 {
			indentAndPrint(buf, lastLine, lineNum)
			lineNum += 1
			lastLine = []uint8("")
		}

		if err == os.EOF {
			os.Exit(0)
		}
	}
}

func indentAndPrint(buf *bytes.Buffer, js []uint8, lineNum int64) {
	jsErr := json.Indent(buf, js, "", "  ")
	if (jsErr != nil) {
		malformedJson(jsErr, js, lineNum)
	}
	os.Stdout.Write(buf.Bytes())
	os.Stdout.Write(newline)
	buf.Reset()
}

func malformedJson(jsErr os.Error, js []uint8, lineNum int64) {
	os.Stdout.Sync()

	synErr, isSynError := (jsErr).(*json.SyntaxError)

	fmt.Fprintf(os.Stderr, "ERROR: Broken json on line %d: %s\n", lineNum, jsErr)

	if (isSynError) {
		prefix := ""
		suffix := ""

		begin := 0
		if synErr.Offset > 15 {
			begin = int(synErr.Offset - 15)
			prefix = "..."
		}

		end := begin + 30
		if end > len(js) {
			end = len(js)
		} else {
			suffix = "..."
		}

		fmt.Fprintf(os.Stderr, "  Context: %s%s%s\n", prefix, js[begin:end], suffix)
	}

	os.Exit(1)
}

func fileFromArguments() *os.File {
	flag.Parse()
	args := flag.Args()
	if len(args) > 1 {
		msg := fmt.Sprintf("only specify 0 or 1 files in the arguments, not %d\n", len(args))
		genericError(os.NewError(msg))
	}
	if len(args) == 0 {
		return os.Stdin
	}

	file, err := os.OpenFile(args[0], os.O_RDONLY, 0)
	if err != nil {
		genericError(err)
	}
	return file
}

func genericError(err os.Error) {
	os.Stdout.Sync()
	fmt.Fprintf(os.Stderr, "ERROR: %s", err)
	os.Exit(2)
}
<|endoftext|>


<commit_before>package hpkp

import (
	"net/http"
	"strconv"
	"strings"
	"time"
)

// Header holds a domain's hpkp information
type Header struct {
	Created           int64
	MaxAge            int64
	IncludeSubDomains bool
	Permanent         bool
	Sha256Pins        []string
}

// Matches checks whether the provided pin is in the header list
func (h *Header) Matches(pin string) bool {
	for i := range h.Sha256Pins {
		if h.Sha256Pins[i] == pin {
			return true
		}
	}
	return false
}

// ParseHeader parses the hpkp information from an http.Response. It should only
// be used on HTTPS connections.
func ParseHeader(resp *http.Response) *Header {
	header := &Header{
		Sha256Pins: []string{},
	}

	v, ok := resp.Header["Public-Key-Pins"]
	if !ok {
		return header
	}

	for _, field := range strings.Split(v[0], ";") {
		field = strings.TrimSpace(field)

		i := strings.Index(field, "pin-sha256")
		if i >= 0 {
			header.Sha256Pins = append(header.Sha256Pins, field[i+12:len(field)-1])
			continue
		}

		i = strings.Index(field, "max-age=")
		if i >= 0 {
			ma, err := strconv.Atoi(field[i+8:])
			if err == nil {
				header.MaxAge = int64(ma)
			}
			continue
		}

		if strings.Contains(field, "includeSubDomains") {
			header.IncludeSubDomains = true
			continue
		}
	}

	header.Created = time.Now().Unix()
	return header
}
<commit_msg>header: nil and TLS checks<commit_after>package hpkp

import (
	"net/http"
	"strconv"
	"strings"
	"time"
)

// Header holds a domain's hpkp information
type Header struct {
	Created           int64
	MaxAge            int64
	IncludeSubDomains bool
	Permanent         bool
	Sha256Pins        []string
}

// Matches checks whether the provided pin is in the header list
func (h *Header) Matches(pin string) bool {
	for i := range h.Sha256Pins {
		if h.Sha256Pins[i] == pin {
			return true
		}
	}
	return false
}

// ParseHeader parses the hpkp information from an http.Response.
func ParseHeader(resp *http.Response) *Header {
	if resp == nil {
		return nil
	}

	// only make a header when using TLS
	if resp.TLS == nil {
		return nil
	}

	v, ok := resp.Header["Public-Key-Pins"]
	if !ok {
		return nil
	}

	header := &Header{
		Sha256Pins: []string{},
	}

	for _, field := range strings.Split(v[0], ";") {
		field = strings.TrimSpace(field)

		i := strings.Index(field, "pin-sha256")
		if i >= 0 {
			header.Sha256Pins = append(header.Sha256Pins, field[i+12:len(field)-1])
			continue
		}

		i = strings.Index(field, "max-age=")
		if i >= 0 {
			ma, err := strconv.Atoi(field[i+8:])
			if err == nil {
				header.MaxAge = int64(ma)
			}
			continue
		}

		if strings.Contains(field, "includeSubDomains") {
			header.IncludeSubDomains = true
			continue
		}
	}

	header.Created = time.Now().Unix()
	return header
}
<|endoftext|>


<commit_before>package python

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/graph"
	"sourcegraph.com/sourcegraph/srcgraph/grapher2"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	grapher2.Register(&fauxPackage{}, grapher2.DockerGrapher{defaultPythonEnv})
}

const srcRoot = "/src"
const stdLibRepo = repo.URI("hg.python.org/cpython")

var builtinPrefixes = map[string]string{"sys": "sys", "os": "os", "path": "os/path"}

var grapherDockerfileTemplate = template.Must(template.New("").Parse(`FROM dockerfile/java
RUN apt-get update
RUN apt-get install -qy curl
RUN apt-get install -qy git
RUN apt-get install -qy {{.Python}}
RUN ln -s $(which {{.Python}}) /usr/bin/python
RUN curl https://raw.github.com/pypa/pip/master/contrib/get-pip.py > get-pip.py
RUN python get-pip.py
RUN pip install virtualenv

# PyDep
RUN pip install git+git://github.com/sourcegraph/pydep@0.0

# Pysonar
RUN apt-get install -qy maven
RUN git clone --depth 1 --branch v0.0 https://github.com/sourcegraph/pysonar2.git /pysonar2
WORKDIR pysonar2
RUN mvn clean package
WORKDIR /

# Set up virtualenv (will contain dependencies)
RUN virtualenv /venv
`))

var grapherDockerCmdTemplate = template.Must(template.New("").Parse(`
/venv/bin/pip install {{.SrcDir}} 1>&2 || /venv/bin/pip install -r {{.SrcDir}}/requirements.txt 1>&2;

# Compute requirements
REQDATA=$(pydep-run.py {{.SrcDir}});

# Compute graph
echo 'Running graphing step...' 1>&2;
mkfifo /tmp/pysonar.err;
cat -v /tmp/pysonar.err &> /dev/null &  # bug: container hangs if we print this output
GRAPHDATA=$(java {{.JavaOpts}} -classpath /pysonar2/target/pysonar-2.0-SNAPSHOT.jar org.yinwang.pysonar.JSONDump {{.SrcDir}} '{{.IncludePaths}}' '' 2>/tmp/pysonar.err);
echo 'Graphing done.' 1>&2;

echo "{ \"graph\": $GRAPHDATA, \"reqs\": $REQDATA }";
`))

func (p *pythonEnv) grapherDockerfile() []byte {
	var buf bytes.Buffer
	grapherDockerfileTemplate.Execute(&buf, struct {
		Python string
		SrcDir string
	}{
		Python: p.PythonVersion,
		SrcDir: srcRoot,
	})
	return buf.Bytes()
}

func (p *pythonEnv) stdLibDir() string {
	return fmt.Sprintf("/usr/lib/%s", p.PythonVersion)
}

func (p *pythonEnv) sitePackagesDir() string {
	return filepath.Join("/venv", "lib", p.PythonVersion, "site-packages")
}

func (p *pythonEnv) grapherCmd() []string {
	javaOpts := os.Getenv("PYGRAPH_JAVA_OPTS")
	inclpaths := []string{srcRoot, p.stdLibDir(), p.sitePackagesDir()}

	var buf bytes.Buffer
	grapherDockerCmdTemplate.Execute(&buf, struct {
		JavaOpts     string
		SrcDir       string
		IncludePaths string
	}{
		JavaOpts:     javaOpts,
		SrcDir:       srcRoot,
		IncludePaths: strings.Join(inclpaths, ":"),
	})
	return []string{"/bin/bash", "-c", buf.String()}
}

func (p *pythonEnv) BuildGrapher(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	return &container.Command{
		Container: container.Container{
			RunOptions: []string{"-v", dir + ":" + srcRoot},
			Dockerfile: p.grapherDockerfile(),
			Cmd:        p.grapherCmd(),
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(orig []byte) ([]byte, error) {
			var o rawGraphData
			err := json.Unmarshal(orig, &o)
			if err != nil {
				return nil, err
			}

			o2 := grapher2.Output{
				Symbols: make([]*graph.Symbol, 0),
				Refs:    make([]*graph.Ref, 0),
				Docs:    make([]*graph.Doc, 0),
			}

			selfrefs := make(map[graph.Ref]struct{})
			for _, psym := range o.Graph.Syms {
				sym, selfref, err := p.convertSym(psym, c, o.Reqs)
				if err != nil {
					return nil, err
				}

				if sym != nil {
					o2.Symbols = append(o2.Symbols, sym)
				}
				if selfref != nil {
					selfrefs[*selfref] = struct{}{}
					o2.Refs = append(o2.Refs, selfref)
				}
			}
			for _, pref := range o.Graph.Refs {
				if ref, err := p.convertRef(pref, c, o.Reqs); err == nil {
					if _, exists := selfrefs[*ref]; !exists {
						o2.Refs = append(o2.Refs, ref)
					}
				} else {
					log.Printf("  (warn) unable to convert reference %+v", pref)
				}
			}
			for _, pdoc := range o.Graph.Docs {
				doc, err := p.convertDoc(pdoc, c, o.Reqs)
				if err != nil {
					return nil, err
				}
				o2.Docs = append(o2.Docs, doc)
			}

			return json.Marshal(o2)
		},
	}, nil
}

func (p *pythonEnv) convertSym(pySym *pySym, c *config.Repository, reqs []requirement) (sym *graph.Symbol, selfref *graph.Ref, err error) {
	symKey, err := p.pysonarSymPathToSymKey(pySym.Path, c, reqs)
	if err != nil {
		return
	}
	file, err := p.pysonarFilePathToFile(pySym.File)
	if err != nil {
		return
	}

	sym = &graph.Symbol{
		SymbolKey:    *symKey,
		Name:         pySym.Name,
		File:         file,
		DefStart:     pySym.DefStart,
		DefEnd:       pySym.DefEnd,
		Exported:     pySym.Exported,
		Callable:     callableSymbolKinds[pySym.Kind],
		Kind:         symbolKinds[pySym.Kind],
		SpecificKind: symbolSpecificKinds[pySym.Kind],
	}
	if pySym.Exported {
		components := strings.Split(string(sym.Path), "/")
		if len(components) == 1 {
			sym.SpecificPath = components[0]
		} else {
			// take the last 2 path components
			sym.SpecificPath = components[len(components)-2] + "." + components[len(components)-1]
		}
	} else {
		sym.SpecificPath = pySym.Name
	}
	if pySym.FuncData != nil {
		sym.TypeExpr = pySym.FuncData.Signature
	}
	if pySym.Kind == "MODULE" && strings.HasSuffix(pySym.File, "__init__.py") {
		sym.SpecificKind = Package
		sym.Kind = graph.Package
	}

	if sym.File != "" && pySym.IdentStart != pySym.IdentEnd {
		var symFile string
		symFile, err = p.pysonarFilePathToFile(pySym.File)
		if err != nil {
			return
		}
		selfref = &graph.Ref{
			SymbolRepo:     symKey.Repo,
			SymbolUnitType: symKey.UnitType,
			SymbolUnit:     symKey.Unit,
			SymbolPath:     symKey.Path,
			Def:            true,

			Repo:     symKey.Repo,
			UnitType: symKey.UnitType,
			Unit:     symKey.Unit,

			File:  symFile,
			Start: pySym.IdentStart,
			End:   pySym.IdentEnd,
		}
	}

	return
}

func (p *pythonEnv) convertRef(pyRef *pyRef, c *config.Repository, reqs []requirement) (*graph.Ref, error) {
	symKey, err := p.pysonarSymPathToSymKey(pyRef.Sym, c, reqs)
	if err != nil {
		return nil, err
	}
	refRepo, refFile, err := p.pysonarFilePathToRepoAndFile(pyRef.File, c, reqs)
	if err != nil {
		return nil, err
	}

	return &graph.Ref{
		SymbolRepo:     symKey.Repo,
		SymbolUnitType: symKey.UnitType,
		SymbolUnit:     symKey.Unit,
		SymbolPath:     symKey.Path,
		Def:            false,

		Repo:     refRepo,
		UnitType: unit.Type(&fauxPackage{}),
		Unit:     (&fauxPackage{}).Name(),

		File:  refFile,
		Start: pyRef.Start,
		End:   pyRef.End,
	}, nil
}

func (p *pythonEnv) convertDoc(pyDoc *pyDoc, c *config.Repository, reqs []requirement) (*graph.Doc, error) {
	// TODO: handle null byte (\x00) in doc body?
	symKey, err := p.pysonarSymPathToSymKey(pyDoc.Sym, c, reqs)
	if err != nil {
		return nil, err
	}
	docFile, err := p.pysonarFilePathToFile(pyDoc.File)
	if err != nil {
		return nil, err
	}
	return &graph.Doc{
		SymbolKey: *symKey,
		Format:    "", // TODO
		Data:      formatDocs(pyDoc.Body),
		File:      docFile,
		Start:     pyDoc.Start,
		End:       pyDoc.End,
	}, nil
}

func (p *pythonEnv) pysonarFilePathToFile(pth string) (string, error) {
	if newpath, err := filepath.Rel(srcRoot, pth); err == nil {
		return newpath, nil
	} else if newpath, err := filepath.Rel(p.sitePackagesDir(), pth); err == nil {
		return newpath, nil
	} else if newpath, err := filepath.Rel(p.stdLibDir(), pth); err == nil {
		return newpath, nil
	} else {
		return "", fmt.Errorf("Could not relativize file path %s", pth)
	}
}

func (p *pythonEnv) pysonarFilePathToRepoAndFile(pth string, c *config.Repository, reqs []requirement) (repo.URI, string, error) {
	if relpath, err := filepath.Rel(srcRoot, pth); err == nil {
		return c.URI, relpath, nil
	} else if relpath, err := filepath.Rel(p.sitePackagesDir(), pth); err == nil {
		var foundReq *requirement
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				pkgpath := strings.Replace(pkg, ".", "/", -1)
				if _, err := filepath.Rel(pkgpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				modpath := mod + ".py"
				if _, err := filepath.Rel(modpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
		}
		if foundReq == nil {
			return "", "", fmt.Errorf("Could not resolve repo URL for file path %s", pth)
		}
		return repo.MakeURI(foundReq.RepoURL), relpath, nil
	} else if relpath, err := filepath.Rel(p.stdLibDir(), pth); err == nil {
		return stdLibRepo, relpath, nil
	} else {
		return "", "", fmt.Errorf("Could not resolve repo URL for file path %s", pth)
	}
}

func (p *pythonEnv) pysonarSymPathToSymKey(pth string, c *config.Repository, reqs []requirement) (*graph.SymbolKey, error) {
	fauxUnit := &fauxPackage{}
	if relpath, err := filepath.Rel(srcRoot, pth); err == nil {
		return &graph.SymbolKey{
			Repo:     c.URI,
			UnitType: unit.Type(fauxUnit),
			Unit:     fauxUnit.Name(),
			Path:     graph.SymbolPath(relpath),
		}, nil
	} else if relpath, err := filepath.Rel(p.sitePackagesDir(), pth); err == nil {
		var foundReq *requirement
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				pkgpath := strings.Replace(pkg, ".", "/", -1)
				if _, err := filepath.Rel(pkgpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				modpath := mod
				if _, err := filepath.Rel(modpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
		}
		if foundReq == nil {
			return nil, fmt.Errorf("Could not find requirement matching path %s", pth)
		}

		return &graph.SymbolKey{
			Repo:     repo.MakeURI(foundReq.RepoURL),
			UnitType: unit.Type(fauxUnit),
			Unit:     fauxUnit.Name(),
			Path:     graph.SymbolPath(relpath),
		}, nil
	} else if relpath, err := filepath.Rel(p.stdLibDir(), pth); err == nil {
		return &graph.SymbolKey{
			Repo:     stdLibRepo,
			UnitType: unit.Type(fauxUnit),
			Unit:     fauxUnit.Name(),
			Path:     graph.SymbolPath(relpath),
		}, nil
	} else {
		for prefix, newPrefix := range builtinPrefixes {
			if strings.HasPrefix(pth, prefix) {
				return &graph.SymbolKey{
					Repo:     stdLibRepo,
					UnitType: unit.Type(fauxUnit),
					Unit:     fauxUnit.Name(),
					Path:     graph.SymbolPath(strings.Replace(pth, prefix, newPrefix, 1)),
				}, nil
			}
		}
		return nil, fmt.Errorf("Could not find requirement matching path %s", pth)
	}
}

type rawGraphData struct {
	Graph struct {
		Syms []*pySym
		Refs []*pyRef
		Docs []*pyDoc
	}
	Reqs []requirement
}

type pySym struct {
	Path       string
	Name       string
	File       string
	IdentStart int
	IdentEnd   int
	DefStart   int
	DefEnd     int
	Exported   bool
	Kind       string
	FuncData   *struct {
		Signature string
	} `json:",omitempty"`
}

type pyRef struct {
	Sym     string
	File    string
	Start   int
	End     int
	Builtin bool
}

type pyDoc struct {
	Sym   string
	File  string
	Body  string
	Start int
	End   int
}
<commit_msg>use redirected-to and rev-specific url for pip (see previous commit to dep.go)<commit_after>package python

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/graph"
	"sourcegraph.com/sourcegraph/srcgraph/grapher2"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	grapher2.Register(&fauxPackage{}, grapher2.DockerGrapher{defaultPythonEnv})
}

const srcRoot = "/src"
const stdLibRepo = repo.URI("hg.python.org/cpython")

var builtinPrefixes = map[string]string{"sys": "sys", "os": "os", "path": "os/path"}

var grapherDockerfileTemplate = template.Must(template.New("").Parse(`FROM dockerfile/java
RUN apt-get update
RUN apt-get install -qy curl
RUN apt-get install -qy git
RUN apt-get install -qy {{.Python}}
RUN ln -s $(which {{.Python}}) /usr/bin/python
RUN curl https://raw.githubusercontent.com/pypa/pip/cdee19c77cf6514d42e2d1b7134f10b8ed36b63a/contrib/get-pip.py > get-pip.py
RUN python get-pip.py
RUN pip install virtualenv

# PyDep
RUN pip install git+git://github.com/sourcegraph/pydep@0.0

# Pysonar
RUN apt-get install -qy maven
RUN git clone --depth 1 --branch v0.0 https://github.com/sourcegraph/pysonar2.git /pysonar2
WORKDIR pysonar2
RUN mvn clean package
WORKDIR /

# Set up virtualenv (will contain dependencies)
RUN virtualenv /venv
`))

var grapherDockerCmdTemplate = template.Must(template.New("").Parse(`
/venv/bin/pip install {{.SrcDir}} 1>&2 || /venv/bin/pip install -r {{.SrcDir}}/requirements.txt 1>&2;

# Compute requirements
REQDATA=$(pydep-run.py {{.SrcDir}});

# Compute graph
echo 'Running graphing step...' 1>&2;
mkfifo /tmp/pysonar.err;
cat -v /tmp/pysonar.err &> /dev/null &  # bug: container hangs if we print this output
GRAPHDATA=$(java {{.JavaOpts}} -classpath /pysonar2/target/pysonar-2.0-SNAPSHOT.jar org.yinwang.pysonar.JSONDump {{.SrcDir}} '{{.IncludePaths}}' '' 2>/tmp/pysonar.err);
echo 'Graphing done.' 1>&2;

echo "{ \"graph\": $GRAPHDATA, \"reqs\": $REQDATA }";
`))

func (p *pythonEnv) grapherDockerfile() []byte {
	var buf bytes.Buffer
	grapherDockerfileTemplate.Execute(&buf, struct {
		Python string
		SrcDir string
	}{
		Python: p.PythonVersion,
		SrcDir: srcRoot,
	})
	return buf.Bytes()
}

func (p *pythonEnv) stdLibDir() string {
	return fmt.Sprintf("/usr/lib/%s", p.PythonVersion)
}

func (p *pythonEnv) sitePackagesDir() string {
	return filepath.Join("/venv", "lib", p.PythonVersion, "site-packages")
}

func (p *pythonEnv) grapherCmd() []string {
	javaOpts := os.Getenv("PYGRAPH_JAVA_OPTS")
	inclpaths := []string{srcRoot, p.stdLibDir(), p.sitePackagesDir()}

	var buf bytes.Buffer
	grapherDockerCmdTemplate.Execute(&buf, struct {
		JavaOpts     string
		SrcDir       string
		IncludePaths string
	}{
		JavaOpts:     javaOpts,
		SrcDir:       srcRoot,
		IncludePaths: strings.Join(inclpaths, ":"),
	})
	return []string{"/bin/bash", "-c", buf.String()}
}

func (p *pythonEnv) BuildGrapher(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	return &container.Command{
		Container: container.Container{
			RunOptions: []string{"-v", dir + ":" + srcRoot},
			Dockerfile: p.grapherDockerfile(),
			Cmd:        p.grapherCmd(),
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(orig []byte) ([]byte, error) {
			var o rawGraphData
			err := json.Unmarshal(orig, &o)
			if err != nil {
				return nil, err
			}

			o2 := grapher2.Output{
				Symbols: make([]*graph.Symbol, 0),
				Refs:    make([]*graph.Ref, 0),
				Docs:    make([]*graph.Doc, 0),
			}

			selfrefs := make(map[graph.Ref]struct{})
			for _, psym := range o.Graph.Syms {
				sym, selfref, err := p.convertSym(psym, c, o.Reqs)
				if err != nil {
					return nil, err
				}

				if sym != nil {
					o2.Symbols = append(o2.Symbols, sym)
				}
				if selfref != nil {
					selfrefs[*selfref] = struct{}{}
					o2.Refs = append(o2.Refs, selfref)
				}
			}
			for _, pref := range o.Graph.Refs {
				if ref, err := p.convertRef(pref, c, o.Reqs); err == nil {
					if _, exists := selfrefs[*ref]; !exists {
						o2.Refs = append(o2.Refs, ref)
					}
				} else {
					log.Printf("  (warn) unable to convert reference %+v", pref)
				}
			}
			for _, pdoc := range o.Graph.Docs {
				doc, err := p.convertDoc(pdoc, c, o.Reqs)
				if err != nil {
					return nil, err
				}
				o2.Docs = append(o2.Docs, doc)
			}

			return json.Marshal(o2)
		},
	}, nil
}

func (p *pythonEnv) convertSym(pySym *pySym, c *config.Repository, reqs []requirement) (sym *graph.Symbol, selfref *graph.Ref, err error) {
	symKey, err := p.pysonarSymPathToSymKey(pySym.Path, c, reqs)
	if err != nil {
		return
	}
	file, err := p.pysonarFilePathToFile(pySym.File)
	if err != nil {
		return
	}

	sym = &graph.Symbol{
		SymbolKey:    *symKey,
		Name:         pySym.Name,
		File:         file,
		DefStart:     pySym.DefStart,
		DefEnd:       pySym.DefEnd,
		Exported:     pySym.Exported,
		Callable:     callableSymbolKinds[pySym.Kind],
		Kind:         symbolKinds[pySym.Kind],
		SpecificKind: symbolSpecificKinds[pySym.Kind],
	}
	if pySym.Exported {
		components := strings.Split(string(sym.Path), "/")
		if len(components) == 1 {
			sym.SpecificPath = components[0]
		} else {
			// take the last 2 path components
			sym.SpecificPath = components[len(components)-2] + "." + components[len(components)-1]
		}
	} else {
		sym.SpecificPath = pySym.Name
	}
	if pySym.FuncData != nil {
		sym.TypeExpr = pySym.FuncData.Signature
	}
	if pySym.Kind == "MODULE" && strings.HasSuffix(pySym.File, "__init__.py") {
		sym.SpecificKind = Package
		sym.Kind = graph.Package
	}

	if sym.File != "" && pySym.IdentStart != pySym.IdentEnd {
		var symFile string
		symFile, err = p.pysonarFilePathToFile(pySym.File)
		if err != nil {
			return
		}
		selfref = &graph.Ref{
			SymbolRepo:     symKey.Repo,
			SymbolUnitType: symKey.UnitType,
			SymbolUnit:     symKey.Unit,
			SymbolPath:     symKey.Path,
			Def:            true,

			Repo:     symKey.Repo,
			UnitType: symKey.UnitType,
			Unit:     symKey.Unit,

			File:  symFile,
			Start: pySym.IdentStart,
			End:   pySym.IdentEnd,
		}
	}

	return
}

func (p *pythonEnv) convertRef(pyRef *pyRef, c *config.Repository, reqs []requirement) (*graph.Ref, error) {
	symKey, err := p.pysonarSymPathToSymKey(pyRef.Sym, c, reqs)
	if err != nil {
		return nil, err
	}
	refRepo, refFile, err := p.pysonarFilePathToRepoAndFile(pyRef.File, c, reqs)
	if err != nil {
		return nil, err
	}

	return &graph.Ref{
		SymbolRepo:     symKey.Repo,
		SymbolUnitType: symKey.UnitType,
		SymbolUnit:     symKey.Unit,
		SymbolPath:     symKey.Path,
		Def:            false,

		Repo:     refRepo,
		UnitType: unit.Type(&fauxPackage{}),
		Unit:     (&fauxPackage{}).Name(),

		File:  refFile,
		Start: pyRef.Start,
		End:   pyRef.End,
	}, nil
}

func (p *pythonEnv) convertDoc(pyDoc *pyDoc, c *config.Repository, reqs []requirement) (*graph.Doc, error) {
	// TODO: handle null byte (\x00) in doc body?
	symKey, err := p.pysonarSymPathToSymKey(pyDoc.Sym, c, reqs)
	if err != nil {
		return nil, err
	}
	docFile, err := p.pysonarFilePathToFile(pyDoc.File)
	if err != nil {
		return nil, err
	}
	return &graph.Doc{
		SymbolKey: *symKey,
		Format:    "", // TODO
		Data:      formatDocs(pyDoc.Body),
		File:      docFile,
		Start:     pyDoc.Start,
		End:       pyDoc.End,
	}, nil
}

func (p *pythonEnv) pysonarFilePathToFile(pth string) (string, error) {
	if newpath, err := filepath.Rel(srcRoot, pth); err == nil {
		return newpath, nil
	} else if newpath, err := filepath.Rel(p.sitePackagesDir(), pth); err == nil {
		return newpath, nil
	} else if newpath, err := filepath.Rel(p.stdLibDir(), pth); err == nil {
		return newpath, nil
	} else {
		return "", fmt.Errorf("Could not relativize file path %s", pth)
	}
}

func (p *pythonEnv) pysonarFilePathToRepoAndFile(pth string, c *config.Repository, reqs []requirement) (repo.URI, string, error) {
	if relpath, err := filepath.Rel(srcRoot, pth); err == nil {
		return c.URI, relpath, nil
	} else if relpath, err := filepath.Rel(p.sitePackagesDir(), pth); err == nil {
		var foundReq *requirement
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				pkgpath := strings.Replace(pkg, ".", "/", -1)
				if _, err := filepath.Rel(pkgpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				modpath := mod + ".py"
				if _, err := filepath.Rel(modpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
		}
		if foundReq == nil {
			return "", "", fmt.Errorf("Could not resolve repo URL for file path %s", pth)
		}
		return repo.MakeURI(foundReq.RepoURL), relpath, nil
	} else if relpath, err := filepath.Rel(p.stdLibDir(), pth); err == nil {
		return stdLibRepo, relpath, nil
	} else {
		return "", "", fmt.Errorf("Could not resolve repo URL for file path %s", pth)
	}
}

func (p *pythonEnv) pysonarSymPathToSymKey(pth string, c *config.Repository, reqs []requirement) (*graph.SymbolKey, error) {
	fauxUnit := &fauxPackage{}
	if relpath, err := filepath.Rel(srcRoot, pth); err == nil {
		return &graph.SymbolKey{
			Repo:     c.URI,
			UnitType: unit.Type(fauxUnit),
			Unit:     fauxUnit.Name(),
			Path:     graph.SymbolPath(relpath),
		}, nil
	} else if relpath, err := filepath.Rel(p.sitePackagesDir(), pth); err == nil {
		var foundReq *requirement
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				pkgpath := strings.Replace(pkg, ".", "/", -1)
				if _, err := filepath.Rel(pkgpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				modpath := mod
				if _, err := filepath.Rel(modpath, relpath); err == nil {
					foundReq = &req
					break FindReq
				}
			}
		}
		if foundReq == nil {
			return nil, fmt.Errorf("Could not find requirement matching path %s", pth)
		}

		return &graph.SymbolKey{
			Repo:     repo.MakeURI(foundReq.RepoURL),
			UnitType: unit.Type(fauxUnit),
			Unit:     fauxUnit.Name(),
			Path:     graph.SymbolPath(relpath),
		}, nil
	} else if relpath, err := filepath.Rel(p.stdLibDir(), pth); err == nil {
		return &graph.SymbolKey{
			Repo:     stdLibRepo,
			UnitType: unit.Type(fauxUnit),
			Unit:     fauxUnit.Name(),
			Path:     graph.SymbolPath(relpath),
		}, nil
	} else {
		for prefix, newPrefix := range builtinPrefixes {
			if strings.HasPrefix(pth, prefix) {
				return &graph.SymbolKey{
					Repo:     stdLibRepo,
					UnitType: unit.Type(fauxUnit),
					Unit:     fauxUnit.Name(),
					Path:     graph.SymbolPath(strings.Replace(pth, prefix, newPrefix, 1)),
				}, nil
			}
		}
		return nil, fmt.Errorf("Could not find requirement matching path %s", pth)
	}
}

type rawGraphData struct {
	Graph struct {
		Syms []*pySym
		Refs []*pyRef
		Docs []*pyDoc
	}
	Reqs []requirement
}

type pySym struct {
	Path       string
	Name       string
	File       string
	IdentStart int
	IdentEnd   int
	DefStart   int
	DefEnd     int
	Exported   bool
	Kind       string
	FuncData   *struct {
		Signature string
	} `json:",omitempty"`
}

type pyRef struct {
	Sym     string
	File    string
	Start   int
	End     int
	Builtin bool
}

type pyDoc struct {
	Sym   string
	File  string
	Body  string
	Start int
	End   int
}
<|endoftext|>


<commit_before>package topgun_test

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"golang.org/x/oauth2"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"

	gclient "code.cloudfoundry.org/garden/client"
	gconn "code.cloudfoundry.org/garden/client/connection"
	sq "github.com/Masterminds/squirrel"
	"github.com/concourse/atc"
	"github.com/concourse/go-concourse/concourse"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"

	"testing"
)

var (
	deploymentName, flyTarget string
	dbIP                      string
	atcIP, atcExternalURL     string
	atcIP2, atcExternalURL2   string

	concourseReleaseVersion, gardenRuncReleaseVersion string
	stemcellVersion                                   string

	pipelineName string

	tmpHome string
	flyBin  string

	logger *lagertest.TestLogger

	boshLogs *gexec.Session
)

var psql = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)

func TestTOPGUN(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "TOPGUN Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	flyBinPath, err := gexec.Build("github.com/concourse/fly")
	Expect(err).ToNot(HaveOccurred())

	return []byte(flyBinPath)
}, func(data []byte) {
	flyBin = string(data)
})

var _ = SynchronizedAfterSuite(func() {
}, func() {
	gexec.CleanupBuildArtifacts()
})

var _ = BeforeEach(func() {
	SetDefaultEventuallyTimeout(5 * time.Minute)
	SetDefaultEventuallyPollingInterval(time.Second)
	SetDefaultConsistentlyDuration(time.Minute)
	SetDefaultConsistentlyPollingInterval(time.Second)

	logger = lagertest.NewTestLogger("test")

	concourseReleaseVersion = os.Getenv("CONCOURSE_RELEASE_VERSION")
	if concourseReleaseVersion == "" {
		concourseReleaseVersion = "latest"
	}

	gardenRuncReleaseVersion = os.Getenv("GARDEN_RUNC_RELEASE_VERSION")
	if gardenRuncReleaseVersion == "" {
		gardenRuncReleaseVersion = "latest"
	}

	stemcellVersion = os.Getenv("STEMCELL_VERSION")
	if stemcellVersion == "" {
		stemcellVersion = "latest"
	}

	deploymentName = fmt.Sprintf("concourse-topgun-%d", GinkgoParallelNode())
	flyTarget = deploymentName

	bosh("delete-deployment")

	atcIP = fmt.Sprintf("10.234.%d.2", GinkgoParallelNode())
	atcIP2 = fmt.Sprintf("10.234.%d.3", GinkgoParallelNode())
	dbIP = fmt.Sprintf("10.234.%d.4", GinkgoParallelNode())

	atcExternalURL = fmt.Sprintf("http://%s:8080", atcIP)
	atcExternalURL2 = fmt.Sprintf("http://%s:8080", atcIP2)
})

var _ = AfterEach(func() {
	boshLogs.Signal(os.Interrupt)
	<-boshLogs.Exited
	boshLogs = nil

	deleteAllContainers()

	bosh("delete-deployment")
})

func Deploy(manifest string, operations ...string) {
	opFlags := []string{}
	for _, op := range operations {
		opFlags = append(opFlags, fmt.Sprintf("-o=%s", op))
	}

	bosh(
		append([]string{
			"deploy", manifest,
			"-v", "deployment-name=" + deploymentName,
			"-v", "atc-ip=" + atcIP,
			"-v", "atc-ip-2=" + atcIP2,
			"-v", "db-ip=" + dbIP,
			"-v", "atc-external-url=" + atcExternalURL,
			"-v", "atc-external-url-2=" + atcExternalURL2,
			"-v", "concourse-release-version=" + concourseReleaseVersion,
			"-v", "garden-runc-release-version=" + gardenRuncReleaseVersion,

			// 3363.10 becomes 3363.1 as it's floating point; quotes prevent that
			"-v", "stemcell-version='" + stemcellVersion + "'",
		}, opFlags...)...,
	)

	fly("login", "-c", atcExternalURL)

	boshLogs = spawnBosh("logs", "-f")
}

func bosh(argv ...string) {
	wait(spawnBosh(argv...))
}

func spawnBosh(argv ...string) *gexec.Session {
	return spawn("bosh", append([]string{"-n", "-d", deploymentName}, argv...)...)
}

func fly(argv ...string) {
	wait(spawnFly(argv...))
}

func concourseClient() concourse.Client {
	token, err := getATCToken(atcExternalURL)
	Expect(err).NotTo(HaveOccurred())
	httpClient := oauthClient(token)
	return concourse.NewClient(atcExternalURL, httpClient)
}

func deleteAllContainers() {
	client := concourseClient()
	workers, err := client.ListWorkers()
	Expect(err).NotTo(HaveOccurred())

	containers, err := client.ListContainers(map[string]string{})
	Expect(err).NotTo(HaveOccurred())

	for _, worker := range workers {
		connection := gconn.New("tcp", worker.GardenAddr)
		gardenClient := gclient.New(connection)
		for _, container := range containers {
			if container.WorkerName == worker.Name {
				err = gardenClient.Destroy(container.ID)
				if err != nil {
					logger.Error("failed-to-delete-container", err, lager.Data{"handle": container.ID})
				}
			}
		}
	}
}

func spawnFly(argv ...string) *gexec.Session {
	return spawn(flyBin, append([]string{"-t", flyTarget}, argv...)...)
}

func spawnFlyInteractive(stdin io.Reader, argv ...string) *gexec.Session {
	return spawnInteractive(stdin, flyBin, append([]string{"-t", flyTarget}, argv...)...)
}

func run(argc string, argv ...string) {
	wait(spawn(argc, argv...))
}

func spawn(argc string, argv ...string) *gexec.Session {
	By("running: " + argc + " " + strings.Join(argv, " "))
	cmd := exec.Command(argc, argv...)
	session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())
	return session
}

func spawnInteractive(stdin io.Reader, argc string, argv ...string) *gexec.Session {
	By("interactively running: " + argc + " " + strings.Join(argv, " "))
	cmd := exec.Command(argc, argv...)
	cmd.Stdin = stdin
	session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())
	return session
}

func wait(session *gexec.Session) {
	<-session.Exited
	Expect(session.ExitCode()).To(Equal(0))
}

func getATCToken(atcURL string) (*atc.AuthToken, error) {
	response, err := http.Get(atcURL + "/api/v1/teams/main/auth/token")
	if err != nil {
		return nil, err
	}

	var token *atc.AuthToken
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(body, &token)
	if err != nil {
		return nil, err
	}

	return token, nil
}

func oauthClient(atcToken *atc.AuthToken) *http.Client {
	return &http.Client{
		Transport: &oauth2.Transport{
			Source: oauth2.StaticTokenSource(&oauth2.Token{
				TokenType:   atcToken.Type,
				AccessToken: atcToken.Value,
			}),
			Base: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			},
		},
	}
}

func waitForLandingOrLandedWorker() string {
	return waitForWorkerInState("landing", "landed")
}

func waitForRunningWorker() string {
	return waitForWorkerInState("running")
}

func waitForStalledWorker() string {
	return waitForWorkerInState("stalled")
}

func waitForWorkerInState(desiredStates ...string) string {
	var workerName string
	Eventually(func() string {

		rows := listWorkers()
		for _, row := range rows {
			if row == "" {
				continue
			}

			worker := strings.Fields(row)

			name := worker[0]
			state := worker[len(worker)-1]

			anyMatched := false
			for _, desiredState := range desiredStates {
				if state == desiredState {
					anyMatched = true
				}
			}

			if !anyMatched {
				continue
			}

			if workerName != "" {
				Fail("multiple workers in states: " + strings.Join(desiredStates, ", "))
			}

			workerName = name
		}

		return workerName
	}).ShouldNot(BeEmpty())

	return workerName
}

func flyTable(argv ...string) []map[string]string {
	session := spawnFly(append([]string{"--print-table-headers"}, argv...)...)
	<-session.Exited
	Expect(session.ExitCode()).To(Equal(0))

	result := []map[string]string{}
	var headers []string

	rows := strings.Split(string(session.Out.Contents()), "\n")
	for i, row := range rows {
		if i == 0 {
			headers = splitFlyColumns(row)
			continue
		}
		if row == "" {
			continue
		}

		result = append(result, map[string]string{})
		columns := splitFlyColumns(row)

		Expect(columns).To(HaveLen(len(headers)))

		for j, header := range headers {
			if header == "" || columns[j] == "" {
				continue
			}

			result[i-1][header] = columns[j]
		}
	}

	return result
}

func splitFlyColumns(row string) []string {
	return regexp.MustCompile(`\s{2,}`).Split(strings.TrimSpace(row), -1)
}

func listWorkers() []string {
	session := spawnFly("workers")
	<-session.Exited

	return strings.Split(string(session.Out.Contents()), "\n")
}

func waitForWorkersToBeRunning() {
	Eventually(func() bool {

		rows := listWorkers()
		anyNotRunning := false
		for _, row := range rows {
			if row == "" {
				continue
			}

			worker := strings.Fields(row)

			state := worker[len(worker)-1]

			if state != "running" {
				anyNotRunning = true
			}
		}

		return anyNotRunning
	}).Should(BeFalse())
}

func workersWithContainers() []string {
	client := concourseClient()
	containers, err := client.ListContainers(map[string]string{})
	Expect(err).NotTo(HaveOccurred())

	usedWorkers := map[string]struct{}{}

	for _, container := range containers {
		usedWorkers[container.WorkerName] = struct{}{}
	}

	var workerNames []string
	for worker, _ := range usedWorkers {
		workerNames = append(workerNames, worker)
	}

	return workerNames
}
<commit_msg>Randomizing the topgun deployment name<commit_after>package topgun_test

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"golang.org/x/oauth2"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"

	gclient "code.cloudfoundry.org/garden/client"
	gconn "code.cloudfoundry.org/garden/client/connection"
	sq "github.com/Masterminds/squirrel"
	"github.com/concourse/atc"
	"github.com/concourse/go-concourse/concourse"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"

	"testing"
)

var (
	deploymentName, flyTarget string
	dbIP                      string
	atcIP, atcExternalURL     string
	atcIP2, atcExternalURL2   string

	concourseReleaseVersion, gardenRuncReleaseVersion string
	stemcellVersion                                   string

	pipelineName string

	tmpHome string
	flyBin  string

	logger *lagertest.TestLogger

	boshLogs *gexec.Session
)

var psql = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)

func TestTOPGUN(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "TOPGUN Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	flyBinPath, err := gexec.Build("github.com/concourse/fly")
	Expect(err).ToNot(HaveOccurred())

	return []byte(flyBinPath)
}, func(data []byte) {
	flyBin = string(data)
})

var _ = SynchronizedAfterSuite(func() {
}, func() {
	gexec.CleanupBuildArtifacts()
})

var _ = BeforeEach(func() {
	SetDefaultEventuallyTimeout(5 * time.Minute)
	SetDefaultEventuallyPollingInterval(time.Second)
	SetDefaultConsistentlyDuration(time.Minute)
	SetDefaultConsistentlyPollingInterval(time.Second)

	logger = lagertest.NewTestLogger("test")

	concourseReleaseVersion = os.Getenv("CONCOURSE_RELEASE_VERSION")
	if concourseReleaseVersion == "" {
		concourseReleaseVersion = "latest"
	}

	gardenRuncReleaseVersion = os.Getenv("GARDEN_RUNC_RELEASE_VERSION")
	if gardenRuncReleaseVersion == "" {
		gardenRuncReleaseVersion = "latest"
	}

	stemcellVersion = os.Getenv("STEMCELL_VERSION")
	if stemcellVersion == "" {
		stemcellVersion = "latest"
	}

	s1 := rand.NewSource(time.Now().UnixNano())
	r1 := rand.New(s1)

	randomizer := r1.Intn(100000)

	deploymentName = fmt.Sprintf("concourse-topgun-%d-%d", GinkgoParallelNode(), randomizer)
	flyTarget = deploymentName

	bosh("delete-deployment")

	atcIP = fmt.Sprintf("10.234.%d.2", GinkgoParallelNode())
	atcIP2 = fmt.Sprintf("10.234.%d.3", GinkgoParallelNode())
	dbIP = fmt.Sprintf("10.234.%d.4", GinkgoParallelNode())

	atcExternalURL = fmt.Sprintf("http://%s:8080", atcIP)
	atcExternalURL2 = fmt.Sprintf("http://%s:8080", atcIP2)
})

var _ = AfterEach(func() {
	boshLogs.Signal(os.Interrupt)
	<-boshLogs.Exited
	boshLogs = nil

	deleteAllContainers()

	bosh("delete-deployment")
})

func Deploy(manifest string, operations ...string) {
	opFlags := []string{}
	for _, op := range operations {
		opFlags = append(opFlags, fmt.Sprintf("-o=%s", op))
	}

	bosh(
		append([]string{
			"deploy", manifest,
			"-v", "deployment-name=" + deploymentName,
			"-v", "atc-ip=" + atcIP,
			"-v", "atc-ip-2=" + atcIP2,
			"-v", "db-ip=" + dbIP,
			"-v", "atc-external-url=" + atcExternalURL,
			"-v", "atc-external-url-2=" + atcExternalURL2,
			"-v", "concourse-release-version=" + concourseReleaseVersion,
			"-v", "garden-runc-release-version=" + gardenRuncReleaseVersion,

			// 3363.10 becomes 3363.1 as it's floating point; quotes prevent that
			"-v", "stemcell-version='" + stemcellVersion + "'",
		}, opFlags...)...,
	)

	fly("login", "-c", atcExternalURL)

	boshLogs = spawnBosh("logs", "-f")
}

func bosh(argv ...string) {
	wait(spawnBosh(argv...))
}

func spawnBosh(argv ...string) *gexec.Session {
	return spawn("bosh", append([]string{"-n", "-d", deploymentName}, argv...)...)
}

func fly(argv ...string) {
	wait(spawnFly(argv...))
}

func concourseClient() concourse.Client {
	token, err := getATCToken(atcExternalURL)
	Expect(err).NotTo(HaveOccurred())
	httpClient := oauthClient(token)
	return concourse.NewClient(atcExternalURL, httpClient)
}

func deleteAllContainers() {
	client := concourseClient()
	workers, err := client.ListWorkers()
	Expect(err).NotTo(HaveOccurred())

	containers, err := client.ListContainers(map[string]string{})
	Expect(err).NotTo(HaveOccurred())

	for _, worker := range workers {
		connection := gconn.New("tcp", worker.GardenAddr)
		gardenClient := gclient.New(connection)
		for _, container := range containers {
			if container.WorkerName == worker.Name {
				err = gardenClient.Destroy(container.ID)
				if err != nil {
					logger.Error("failed-to-delete-container", err, lager.Data{"handle": container.ID})
				}
			}
		}
	}
}

func spawnFly(argv ...string) *gexec.Session {
	return spawn(flyBin, append([]string{"-t", flyTarget}, argv...)...)
}

func spawnFlyInteractive(stdin io.Reader, argv ...string) *gexec.Session {
	return spawnInteractive(stdin, flyBin, append([]string{"-t", flyTarget}, argv...)...)
}

func run(argc string, argv ...string) {
	wait(spawn(argc, argv...))
}

func spawn(argc string, argv ...string) *gexec.Session {
	By("running: " + argc + " " + strings.Join(argv, " "))
	cmd := exec.Command(argc, argv...)
	session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())
	return session
}

func spawnInteractive(stdin io.Reader, argc string, argv ...string) *gexec.Session {
	By("interactively running: " + argc + " " + strings.Join(argv, " "))
	cmd := exec.Command(argc, argv...)
	cmd.Stdin = stdin
	session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())
	return session
}

func wait(session *gexec.Session) {
	<-session.Exited
	Expect(session.ExitCode()).To(Equal(0))
}

func getATCToken(atcURL string) (*atc.AuthToken, error) {
	response, err := http.Get(atcURL + "/api/v1/teams/main/auth/token")
	if err != nil {
		return nil, err
	}

	var token *atc.AuthToken
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(body, &token)
	if err != nil {
		return nil, err
	}

	return token, nil
}

func oauthClient(atcToken *atc.AuthToken) *http.Client {
	return &http.Client{
		Transport: &oauth2.Transport{
			Source: oauth2.StaticTokenSource(&oauth2.Token{
				TokenType:   atcToken.Type,
				AccessToken: atcToken.Value,
			}),
			Base: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			},
		},
	}
}

func waitForLandingOrLandedWorker() string {
	return waitForWorkerInState("landing", "landed")
}

func waitForRunningWorker() string {
	return waitForWorkerInState("running")
}

func waitForStalledWorker() string {
	return waitForWorkerInState("stalled")
}

func waitForWorkerInState(desiredStates ...string) string {
	var workerName string
	Eventually(func() string {

		rows := listWorkers()
		for _, row := range rows {
			if row == "" {
				continue
			}

			worker := strings.Fields(row)

			name := worker[0]
			state := worker[len(worker)-1]

			anyMatched := false
			for _, desiredState := range desiredStates {
				if state == desiredState {
					anyMatched = true
				}
			}

			if !anyMatched {
				continue
			}

			if workerName != "" {
				Fail("multiple workers in states: " + strings.Join(desiredStates, ", "))
			}

			workerName = name
		}

		return workerName
	}).ShouldNot(BeEmpty())

	return workerName
}

func flyTable(argv ...string) []map[string]string {
	session := spawnFly(append([]string{"--print-table-headers"}, argv...)...)
	<-session.Exited
	Expect(session.ExitCode()).To(Equal(0))

	result := []map[string]string{}
	var headers []string

	rows := strings.Split(string(session.Out.Contents()), "\n")
	for i, row := range rows {
		if i == 0 {
			headers = splitFlyColumns(row)
			continue
		}
		if row == "" {
			continue
		}

		result = append(result, map[string]string{})
		columns := splitFlyColumns(row)

		Expect(columns).To(HaveLen(len(headers)))

		for j, header := range headers {
			if header == "" || columns[j] == "" {
				continue
			}

			result[i-1][header] = columns[j]
		}
	}

	return result
}

func splitFlyColumns(row string) []string {
	return regexp.MustCompile(`\s{2,}`).Split(strings.TrimSpace(row), -1)
}

func listWorkers() []string {
	session := spawnFly("workers")
	<-session.Exited

	return strings.Split(string(session.Out.Contents()), "\n")
}

func waitForWorkersToBeRunning() {
	Eventually(func() bool {

		rows := listWorkers()
		anyNotRunning := false
		for _, row := range rows {
			if row == "" {
				continue
			}

			worker := strings.Fields(row)

			state := worker[len(worker)-1]

			if state != "running" {
				anyNotRunning = true
			}
		}

		return anyNotRunning
	}).Should(BeFalse())
}

func workersWithContainers() []string {
	client := concourseClient()
	containers, err := client.ListContainers(map[string]string{})
	Expect(err).NotTo(HaveOccurred())

	usedWorkers := map[string]struct{}{}

	for _, container := range containers {
		usedWorkers[container.WorkerName] = struct{}{}
	}

	var workerNames []string
	for worker, _ := range usedWorkers {
		workerNames = append(workerNames, worker)
	}

	return workerNames
}
<|endoftext|>


<commit_before>package tracks

func isPossibleChainLift(val *Segment) bool {
	// XXX, check other pieces that can be chain lifts. in theory you can also
	// place chain lifts on straight pieces of track but this seems silly
	// (maybe a super in-future optimization will allow this)
	return val.InputDegree == TRACK_UP_25 &&
		(val.OutputDegree == TRACK_UP_25 || val.OutputDegree == TRACK_NONE) &&
		val.StartingBank == TRACK_BANK_NONE &&
		val.EndingBank == TRACK_BANK_NONE
}

func (e *Element) diagonal() bool {
	// 8d to ab are straight diagonal pieces, probably a better way to do that
	// check
	return e.Segment.DirectionDelta == DIR_DIAGONAL_RIGHT ||
		e.Segment.DirectionDelta == DIR_DIAGONAL_LEFT ||
		(e.Segment.Type < 0xAC && e.Segment.Type > 0x8C)
}

func (e *Element) valid() bool {
	// Heartline and mini golf
	if e.Segment.Type >= 0xc5 && e.Segment.Type <= 0xd0 {
		return false
	}
	return true
}

// Possibilities computes all of the possible track pieces that can be built
func (s *Element) Possibilities() (o []Element) {
	for _, val := range TS_MAP {
		// XXX, figure out how to handle diagonal
		if val.InputDegree == s.Segment.OutputDegree && val.StartingBank == s.Segment.EndingBank && !s.diagonal() && s.valid() {
			o = append(o, Element{Segment: val, ChainLift: false})
			if isPossibleChainLift(val) {
				o = append(o, Element{Segment: val, ChainLift: true})
			}
			// XXX, cache possibilities, since they're not going to change, or
			// just build this once
		}
	}
	return o
}

// Compatible returns true if the end of the first element is compatible with
// the beginning of the second element.
func Compatible(first Element, second Element) bool {
	return first.Segment.OutputDegree == second.Segment.InputDegree &&
		first.Segment.EndingBank == second.Segment.StartingBank
}
<commit_msg>tracks: fix diagonal function to check diagonals. panic on non-diagonal<commit_after>package tracks

import "log"

func isPossibleChainLift(val *Segment) bool {
	// XXX, check other pieces that can be chain lifts. in theory you can also
	// place chain lifts on straight pieces of track but this seems silly
	// (maybe a super in-future optimization will allow this)
	return val.InputDegree == TRACK_UP_25 &&
		(val.OutputDegree == TRACK_UP_25 || val.OutputDegree == TRACK_NONE) &&
		val.StartingBank == TRACK_BANK_NONE &&
		val.EndingBank == TRACK_BANK_NONE
}

func (e *Element) diagonal() bool {
	// 8d to ab are straight diagonal pieces, probably a better way to do that
	// check
	return e.Segment.DirectionDelta == DIR_DIAGONAL_RIGHT ||
		e.Segment.DirectionDelta == DIR_DIAGONAL_LEFT ||
		(e.Segment.Type < 0xAC && e.Segment.Type > 0x8C)
}

func (s *Segment) diagonal() bool {
	return s.DirectionDelta == DIR_DIAGONAL_RIGHT ||
		s.DirectionDelta == DIR_DIAGONAL_LEFT ||
		(s.Type < 0xAC && s.Type > 0x8C)
}

func (e *Element) valid() bool {
	// Heartline and mini golf
	if e.Segment.Type >= 0xc5 && e.Segment.Type <= 0xd0 {
		return false
	}
	return true
}

// Possibilities computes all of the possible track pieces that can be built
func (s *Element) Possibilities() (o []Element) {
	for _, val := range TS_MAP {
		// XXX, figure out how to handle diagonal
		if val.InputDegree == s.Segment.OutputDegree && val.StartingBank == s.Segment.EndingBank && !s.diagonal() && !val.diagonal() && s.valid() {
			if val.DirectionDelta != 0 && val.DirectionDelta != 90 && val.DirectionDelta != 180 && val.DirectionDelta != 270 {
				log.Panicf("%#v", val)
			}
			o = append(o, Element{Segment: val, ChainLift: false})
			if isPossibleChainLift(val) {
				o = append(o, Element{Segment: val, ChainLift: true})
			}
			// XXX, cache possibilities, since they're not going to change, or
			// just build this once
		}
	}
	return o
}

// Compatible returns true if the end of the first element is compatible with
// the beginning of the second element.
func Compatible(first Element, second Element) bool {
	return first.Segment.OutputDegree == second.Segment.InputDegree &&
		first.Segment.EndingBank == second.Segment.StartingBank
}
<|endoftext|>


<commit_before>package timer

import (
	"container/heap"
	"fmt"
	"sync"
	"time"
)

const (
	IDLE int = iota
	STARTED
	STOPPED
)

type schedIdx struct {
	sync.RWMutex
	i map[int64]int
}

type Timer struct {
	begin int64

	inn    chan *ticket
	sync   chan *ticket
	pq     priority
	halt   chan struct{}
	end    chan struct{}
	state  int
	lookup *schedIdx
}

func (t *Timer) pop(iden int64) (ack bool) {
	t.lookup.Lock()
	defer t.lookup.Unlock()
	if _, ack = t.lookup.i[iden]; ack {
		delete(t.lookup.i, iden)
	}
	return
}

func (t *Timer) place(job *ticket) {
	t.lookup.Lock()
	defer t.lookup.Unlock()
	t.lookup.i[job.iden] = job.pos
}

func (t *Timer) dispatch() {
	now := time.Now()
	for ok := true; ok; {
		if len(t.pq) > 0 && now.After(t.pq[0].a) {
			job := heap.Pop(&t.pq).(*ticket)
			if t.pop(job.iden) {
				go job.h.Done() // fire the worker on timer
			}
		} else {
			ok = false
		}
	}
}

func (t *Timer) Tic() {
	if t.state != IDLE {
		panic(fmt.Errorf("timer must be in IDLE to Tic"))
	} else {
		t.state = STARTED
	}

	t.halt, t.end = make(chan struct{}), make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		defer wg.Done()
		for yay := true; yay; {
			// determine the closest future about to happen
			var alarm <-chan time.Time
			if len(t.pq) == 0 {
				alarm = time.After(1 * time.Hour)
			} else {
				diff := t.pq[0].a.Sub(time.Now())
				if diff < 0 {
					t.dispatch()
					continue
				}
				alarm = time.Tick(t.pq[0].a.Sub(time.Now()))
			}

			select {
			case <-t.halt:
				yay = false // someone closed the light, quit

			case job := <-t.inn:
				heap.Push(&t.pq, job)
				t.place(job)

			case old_job := <-t.sync:
				heap.Fix(&t.pq, old_job.pos)
				t.place(old_job)

			case <-alarm:
				t.dispatch()
			}
		}
	}()

	go func() { // wait for timer runner and collect it
		wg.Wait()
		close(t.end)
	}()
}

func (t *Timer) Toc() {
	close(t.halt)
	for _ = range t.end {
	}
	t.state = STOPPED
}

func (t *Timer) SchedFunc(c time.Time, handle func()) (iden int64) {
	t.begin, iden = t.begin+1, t.begin
	t.inn <- &ticket{a: c, h: HandlerFunc(handle), iden: iden}
	return
}

func (t *Timer) Sched(c time.Time, handle Handler) (iden int64) {
	t.begin, iden = t.begin+1, t.begin
	t.inn <- &ticket{a: c, h: handle, iden: iden}
	return
}

func (t *Timer) Update(iden int64, c time.Time) {
	t.lookup.RLock()
	defer t.lookup.RUnlock()
	if pos, ok := t.lookup.i[iden]; ok {
		job := t.pq[pos]
		job.a = c
		t.sync <- job
	}
}

func (t *Timer) Cancel(iden int64) {
	t.pop(iden)
}

func NewTimer() (t *Timer) {
	pri := make(priority, 0)
	heap.Init(&pri)
	return &Timer{
		state:  IDLE,
		inn:    make(chan *ticket, 1),
		sync:   make(chan *ticket, 1),
		pq:     pri,
		halt:   nil,
		end:    nil,
		lookup: &schedIdx{i: make(map[int64]int)},
	}
}
<commit_msg>FIX: avoid computing diff time twice<commit_after>package timer

import (
	"container/heap"
	"fmt"
	"sync"
	"time"
)

const (
	IDLE int = iota
	STARTED
	STOPPED
)

type schedIdx struct {
	sync.RWMutex
	i map[int64]int
}

type Timer struct {
	begin int64

	inn    chan *ticket
	sync   chan *ticket
	pq     priority
	halt   chan struct{}
	end    chan struct{}
	state  int
	lookup *schedIdx
}

func (t *Timer) pop(iden int64) (ack bool) {
	t.lookup.Lock()
	defer t.lookup.Unlock()
	if _, ack = t.lookup.i[iden]; ack {
		delete(t.lookup.i, iden)
	}
	return
}

func (t *Timer) place(job *ticket) {
	t.lookup.Lock()
	defer t.lookup.Unlock()
	t.lookup.i[job.iden] = job.pos
}

func (t *Timer) dispatch() {
	now := time.Now()
	for ok := true; ok; {
		if len(t.pq) > 0 && now.After(t.pq[0].a) {
			job := heap.Pop(&t.pq).(*ticket)
			if t.pop(job.iden) {
				go job.h.Done() // fire the worker on timer
			}
		} else {
			ok = false
		}
	}
}

func (t *Timer) Tic() {
	if t.state != IDLE {
		panic(fmt.Errorf("timer must be in IDLE to Tic"))
	} else {
		t.state = STARTED
	}

	t.halt, t.end = make(chan struct{}), make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		defer wg.Done()
		var alarm <-chan time.Time
		for yay := true; yay; {
			// determine the closest future about to happen
			if len(t.pq) == 0 {
				alarm = time.After(1 * time.Hour)
			} else {
				diff := t.pq[0].a.Sub(time.Now())
				if diff < 0 {
					t.dispatch()
					continue
				}
				alarm = time.After(diff)
			}

			select {
			case <-t.halt:
				yay = false // someone closed the light, quit

			case job := <-t.inn:
				heap.Push(&t.pq, job)
				t.place(job)

			case old_job := <-t.sync:
				heap.Fix(&t.pq, old_job.pos)
				t.place(old_job)

			case <-alarm:
				t.dispatch()
			}
		}
	}()

	go func() { // wait for timer runner and collect it
		wg.Wait()
		close(t.end)
	}()
}

func (t *Timer) Toc() {
	close(t.halt)
	for _ = range t.end {
	}
	t.state = STOPPED
}

func (t *Timer) SchedFunc(c time.Time, handle func()) (iden int64) {
	t.begin, iden = t.begin+1, t.begin
	t.inn <- &ticket{a: c, h: HandlerFunc(handle), iden: iden}
	return
}

func (t *Timer) Sched(c time.Time, handle Handler) (iden int64) {
	t.begin, iden = t.begin+1, t.begin
	t.inn <- &ticket{a: c, h: handle, iden: iden}
	return
}

func (t *Timer) Update(iden int64, c time.Time) {
	t.lookup.RLock()
	defer t.lookup.RUnlock()
	if pos, ok := t.lookup.i[iden]; ok {
		job := t.pq[pos]
		job.a = c
		t.sync <- job
	}
}

func (t *Timer) Cancel(iden int64) {
	t.pop(iden)
}

func NewTimer() (t *Timer) {
	pri := make(priority, 0)
	heap.Init(&pri)
	return &Timer{
		state:  IDLE,
		inn:    make(chan *ticket, 1),
		sync:   make(chan *ticket, 1),
		pq:     pri,
		halt:   nil,
		end:    nil,
		lookup: &schedIdx{i: make(map[int64]int)},
	}
}
<|endoftext|>


<commit_before>package cruncy

import (
  "time"
  "log"
)

type TimerData struct {
  StartTimeRun   time.Time
  StartTimeBatch time.Time
  EndTimeRun      time.Time

  BatchSize      int64
  PrevRows       int64
  Index          int64
}

func NewTimer() *TimerData {
  timer := &TimerData{}
  timer.StartTimeRun = time.Now()
  timer.StartTimeBatch = timer.StartTimeRun
  timer.PrevRows = 0
  return timer
}

func (timer TimerData) BatchDuractionSeconds() int64 {
  t1 := time.Now()
  var duration time.Duration = t1.Sub(timer.StartTimeBatch)
  return int64(duration.Seconds())
}

func (timer TimerData) TotalDuractionSeconds() int64 {
  t1 := time.Now()
  var duration time.Duration = t1.Sub(timer.StartTimeRun)
  return int64(duration.Seconds())
}

func (timer TimerData) TotalDuration() time.Duration {
  t1 := time.Now()
  return t1.Sub(timer.StartTimeRun)
}

func (timer TimerData) ShowTotalDuration() {
  duration := timer.TotalDuration()
  duration_sec := timer.TotalDuractionSeconds()
  if duration_sec > 0 {
    log.Printf("Total duration:, %v rows =%d row time=%d rows/sec ", duration, timer.Index, timer.Index / duration_sec )
  } else {
    log.Printf("Total duration:, %v rows =%d  SUPER FAST", duration, timer.Index)

  }
}

func (timer *TimerData) ShowBatchTime() {
  diff := timer.Index - timer.PrevRows

  t1 := time.Now()
  var duration time.Duration = t1.Sub(timer.StartTimeBatch)
  var d2 time.Duration = timer.TotalDuration()

  var ds int64 = int64(d2.Seconds())
  if (ds > 0) {
    log.Printf("%d rows avg flow %d/s - batch time %v batch size %d batch_flow %d \n", timer.Index, timer.Index / ds, duration,diff,diff/ds)
  } else {
    log.Printf("%d rows - batch time %v \n", timer.Index,  duration)
  }
  timer.PrevRows = timer.Index
  timer.StartTimeBatch = time.Now()

}

func (timer *TimerData) Tick() {
  timer.Index++

  if timer.Index % 100000 == 0 {
    timer.ShowBatchTime()
  }
}

func (timer *TimerData) Stop() time.Time {
  timer.EndTimeRun = time.Now()
  return timer.EndTimeRun
}

<commit_msg>Added timing log fields<commit_after>package cruncy

import (
  "time"
  log "github.com/Sirupsen/logrus"
  "fmt"
  "github.com/satori/go.uuid"
)

type TimerData struct {
  Title          string
  Uuid           string
  StartTimeRun   time.Time
  StartTimeBatch time.Time
  EndTimeRun     time.Time

  BatchSize      int64
  PrevRows       int64
  Index          int64
}

func NewTimer(title string) *TimerData {
  timer := &TimerData{}
  timer.Title = title
  timer.Uuid = uuid.NewV4().String()
  timer.StartTimeRun = time.Now()
  timer.StartTimeBatch = timer.StartTimeRun
  timer.PrevRows = 0
  return timer
}

func (timer TimerData) BatchDuractionSeconds() int64 {
  t1 := time.Now()
  var duration time.Duration = t1.Sub(timer.StartTimeBatch)
  return int64(duration.Seconds())
}

func (timer TimerData) TotalDuractionSeconds() int64 {
  t1 := time.Now()
  var duration time.Duration = t1.Sub(timer.StartTimeRun)
  return int64(duration.Seconds())
}

func (timer TimerData) TotalDuration() time.Duration {
  t1 := time.Now()
  return t1.Sub(timer.StartTimeRun)
}

func (timer TimerData) ShowTotalDuration() {
  duration := timer.TotalDuration()
  ds := timer.TotalDuractionSeconds()
  if ds > 0 {
    msg := fmt.Sprintf("Total duration:, %v rows =%d row time=%d rows/sec ", duration, timer.Index, timer.Index / duration_sec)
    log.WithFields(log.Fields{
      "uuid": timer.Uuid,
      "title": timer.Title,
      "index": timer.Index,
      "total_flow":   timer.Index / ds,
      "State":   "stopped",
    }).Info(msg)
  } else {
    log.Printf("Total duration:, %v rows =%d  SUPER FAST", duration, timer.Index)

  }
}

func (timer *TimerData) ShowBatchTime() {
  diff := timer.Index - timer.PrevRows

  t1 := time.Now()
  var duration time.Duration = t1.Sub(timer.StartTimeBatch)
  var d2 time.Duration = timer.TotalDuration()

  ds := int64(d2.Seconds())
  ds_batch := int64(duration.Seconds())

  if (ds > 0) {
    msg := fmt.Sprintf("%d rows avg flow %d/s - batch time %v batch size %d batch_flow %d \n", timer.Index, timer.Index / ds, duration, diff, diff / ds_batch)
    log.WithFields(log.Fields{
      "uuid":         timer.Uuid,
      "title":        timer.Title,
      "index":        timer.Index,
      "total_flow":   timer.Index / ds,
      "batch_time":   duration,
      "batch_size":   diff,
      "batch_flow":   diff / ds_batch,
      "State":        "in_batch",
    }).Info(msg)
  } else {
    log.Printf("%d rows - batch time %v \n", timer.Index, duration)
  }
  timer.PrevRows = timer.Index
  timer.StartTimeBatch = time.Now()

}

func (timer *TimerData) Tick() {
  timer.Index++

  if timer.Index % 100000 == 0 {
    timer.ShowBatchTime()
  }
}

func (timer *TimerData) Stop() time.Time {
  timer.EndTimeRun = time.Now()
  return timer.EndTimeRun
}

<|endoftext|>


<commit_before>// Copyright (c) 2013-2019 by Michael Dvorkin and contributors. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package mop

import (
	`bytes`
	`fmt`
	`reflect`
	`regexp`
	`strings`
	`text/template`
	`time`
)

// Column describes formatting rules for individual column within the list
// of stock quotes.
type Column struct {
	width     int                 // Column width.
	name      string              // The name of the field in the Stock struct.
	title     string              // Column title to display in the header.
	formatter func(string) string // Optional function to format the contents of the column.
}

// Layout is used to format and display all the collected data, i.e. market
// updates and the list of stock quotes.
type Layout struct {
	columns        []Column           // List of stock quotes columns.
	sorter         *Sorter            // Pointer to sorting receiver.
	regex          *regexp.Regexp     // Pointer to regular expression to align decimal points.
	marketTemplate *template.Template // Pointer to template to format market data.
	quotesTemplate *template.Template // Pointer to template to format the list of stock quotes.
}

// Creates the layout and assigns the default values that stay unchanged.
func NewLayout() *Layout {
	layout := &Layout{}
	layout.columns = []Column{
		{-7, `Ticker`, `Ticker`, nil},
		{10, `LastTrade`, `Last`, currency},
		{10, `Change`, `Change`, currency},
		{10, `ChangePct`, `Change%`, last},
		{10, `Open`, `Open`, currency},
		{10, `Low`, `Low`, currency},
		{10, `High`, `High`, currency},
		{10, `Low52`, `52w Low`, currency},
		{10, `High52`, `52w High`, currency},
		{11, `Volume`, `Volume`, nil},
		{11, `AvgVolume`, `AvgVolume`, nil},
		{9, `PeRatio`, `P/E`, blank},
		{9, `Dividend`, `Dividend`, zero},
		{9, `Yield`, `Yield`, percent},
		{11, `MarketCap`, `MktCap`, currency},
	}
	layout.regex = regexp.MustCompile(`(\.\d+)[BMK]?$`)
	layout.marketTemplate = buildMarketTemplate()
	layout.quotesTemplate = buildQuotesTemplate()

	return layout
}

// Market merges given market data structure with the market template and
// returns formatted string that includes highlighting markup.
func (layout *Layout) Market(market *Market) string {
	if ok, err := market.Ok(); !ok { // If there was an error fetching market data...
		return err // then simply return the error string.
	}

	highlight(market.Dow, market.Sp500, market.Nasdaq,
		market.Tokyo, market.HongKong, market.London, market.Frankfurt,
		market.Yield, market.Oil, market.Euro, market.Gold)
	buffer := new(bytes.Buffer)
	layout.marketTemplate.Execute(buffer, market)

	return buffer.String()
}

// Quotes uses quotes template to format timestamp, stock quotes header,
// and the list of given stock quotes. It returns formatted string with
// all the necessary markup.
func (layout *Layout) Quotes(quotes *Quotes) string {
	if ok, err := quotes.Ok(); !ok { // If there was an error fetching stock quotes...
		return err // then simply return the error string.
	}

	vars := struct {
		Now    string  // Current timestamp.
		Header string  // Formatted header line.
		Stocks []Stock // List of formatted stock quotes.
	}{
		time.Now().Format(`3:04:05pm PST`),
		layout.Header(quotes.profile),
		layout.prettify(quotes),
	}

	buffer := new(bytes.Buffer)
	layout.quotesTemplate.Execute(buffer, vars)

	return buffer.String()
}

// Header iterates over column titles and formats the header line. The
// formatting includes placing an arrow next to the sorted column title.
// When the column editor is active it knows how to highlight currently
// selected column title.
func (layout *Layout) Header(profile *Profile) string {
	str, selectedColumn := ``, profile.selectedColumn

	for i, col := range layout.columns {
		arrow := arrowFor(i, profile)
		if i != selectedColumn {
			str += fmt.Sprintf(`%*s`, col.width, arrow+col.title)
		} else {
			str += fmt.Sprintf(`<r>%*s</r>`, col.width, arrow+col.title)
		}
	}

	return `<u>` + str + `</u>`
}

// TotalColumns is the utility method for the column editor that returns
// total number of columns.
func (layout *Layout) TotalColumns() int {
	return len(layout.columns)
}

//-----------------------------------------------------------------------------
func (layout *Layout) prettify(quotes *Quotes) []Stock {
	pretty := make([]Stock, len(quotes.stocks))
	//
	// Iterate over the list of stocks and properly format all its columns.
	//
	for i, stock := range quotes.stocks {
		pretty[i].Advancing = stock.Advancing
		//
		// Iterate over the list of stock columns. For each column name:
		// - Get current column value.
		// - If the column has the formatter method then call it.
		// - Set the column value padding it to the given width.
		//
		for _, column := range layout.columns {
			// ex. value = stock.Change
			value := reflect.ValueOf(&stock).Elem().FieldByName(column.name).String()
			if column.formatter != nil {
				// ex. value = currency(value)
				value = column.formatter(value)
			}
			// ex. pretty[i].Change = layout.pad(value, 10)
			reflect.ValueOf(&pretty[i]).Elem().FieldByName(column.name).SetString(layout.pad(value, column.width))
		}
	}

	profile := quotes.profile
	if layout.sorter == nil { // Initialize sorter on first invocation.
		layout.sorter = NewSorter(profile)
	}
	layout.sorter.SortByCurrentColumn(pretty)
	//
	// Group stocks by advancing/declining unless sorted by Chanage or Change%
	// in which case the grouping has been done already.
	//
	if profile.Grouped && (profile.SortColumn < 2 || profile.SortColumn > 3) {
		pretty = group(pretty)
	}

	return pretty
}

//-----------------------------------------------------------------------------
func (layout *Layout) pad(str string, width int) string {
	match := layout.regex.FindStringSubmatch(str)
	if len(match) > 0 {
		switch len(match[1]) {
		case 2:
			str = strings.Replace(str, match[1], match[1]+`0`, 1)
		case 4, 5:
			str = strings.Replace(str, match[1], match[1][0:3], 1)
		}
	}

	return fmt.Sprintf(`%*s`, width, str)
}

//-----------------------------------------------------------------------------
func buildMarketTemplate() *template.Template {
	markup := `<yellow>Dow</> {{.Dow.change}} ({{.Dow.percent}}) at {{.Dow.latest}} <yellow>S&P 500</> {{.Sp500.change}} ({{.Sp500.percent}}) at {{.Sp500.latest}} <yellow>NASDAQ</> {{.Nasdaq.change}} ({{.Nasdaq.percent}}) at {{.Nasdaq.latest}}
<yellow>Tokyo</> {{.Tokyo.change}} ({{.Tokyo.percent}}) at {{.Tokyo.latest}} <yellow>HK</> {{.HongKong.change}} ({{.HongKong.percent}}) at {{.HongKong.latest}} <yellow>London</> {{.London.change}} ({{.London.percent}}) at {{.London.latest}} <yellow>Frankfurt</> {{.Frankfurt.change}} ({{.Frankfurt.percent}}) at {{.Frankfurt.latest}} {{if .IsClosed}}<right>U.S. markets closed</right>{{end}}
<yellow>10-Year Yield</> {{.Yield.latest}}% ({{.Yield.change}}) <yellow>Euro</> ${{.Euro.latest}} ({{.Euro.change}}%) <yellow>Yen</> ¥{{.Yen.latest}} ({{.Yen.change}}%) <yellow>Oil</> ${{.Oil.latest}} ({{.Oil.change}}%) <yellow>Gold</> ${{.Gold.latest}} ({{.Gold.change}}%)`

	return template.Must(template.New(`market`).Parse(markup))
}

//-----------------------------------------------------------------------------
func buildQuotesTemplate() *template.Template {
	markup := `<right><white>{{.Now}}</></right>

{{.Header}}
{{range.Stocks}}{{if .Advancing}}<green>{{end}}{{.Ticker}}{{.LastTrade}}{{.Change}}{{.ChangePct}}{{.Open}}{{.Low}}{{.High}}{{.Low52}}{{.High52}}{{.Volume}}{{.AvgVolume}}{{.PeRatio}}{{.Dividend}}{{.Yield}}{{.MarketCap}}</>
{{end}}`

	return template.Must(template.New(`quotes`).Parse(markup))
}

//-----------------------------------------------------------------------------
func highlight(collections ...map[string]string) {
	for _, collection := range collections {
		if collection[`change`][0:1] != `-` {
			collection[`change`] = `<green>` + collection[`change`] + `</>`
		}
	}
}

//-----------------------------------------------------------------------------
func group(stocks []Stock) []Stock {
	grouped := make([]Stock, len(stocks))
	current := 0

	for _, stock := range stocks {
		if stock.Advancing {
			grouped[current] = stock
			current++
		}
	}
	for _, stock := range stocks {
		if !stock.Advancing {
			grouped[current] = stock
			current++
		}
	}

	return grouped
}

//-----------------------------------------------------------------------------
func arrowFor(column int, profile *Profile) string {
	if column == profile.SortColumn {
		if profile.Ascending {
			return string('\U00002191')
		}
		return string('\U00002193')
	}
	return ``
}

//-----------------------------------------------------------------------------
func blank(str string) string {
	if len(str) == 3 && str[0:3] == `N/A` {
		return `-`
	}

	return str
}

//-----------------------------------------------------------------------------
func zero(str string) string {
	if str == `0.00` {
		return `-`
	}

	return currency(str)
}

//-----------------------------------------------------------------------------
func last(str string) string {
	if len(str) >= 6 && str[0:6] == `N/A - ` {
		return str[6:]
	}

	return percent(str)
}

//-----------------------------------------------------------------------------
func currency(str string) string {
	if str == `N/A` || len(str) == 0 {
		return `-`
	}
	if sign := str[0:1]; sign == `+` || sign == `-` {
		return sign + `$` + str[1:]
	}

	return `$` + str
}

// Returns percent value truncated at 2 decimal points.
//-----------------------------------------------------------------------------
func percent(str string) string {
	if str == `N/A` || len(str) == 0 {
		return `-`
	}

	split := strings.Split(str, ".")
	if len(split) == 2 {
		digits := len(split[1])
		if digits > 2 {
			digits = 2
		}
		str = split[0] + "." + split[1][0:digits]
	}
	if str[len(str)-1] != '%' {
		str += `%`
	}
	return str
}
<commit_msg>adjust ticker column width, to make room for longer ticker codes<commit_after>// Copyright (c) 2013-2019 by Michael Dvorkin and contributors. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package mop

import (
	`bytes`
	`fmt`
	`reflect`
	`regexp`
	`strings`
	`text/template`
	`time`
)

// Column describes formatting rules for individual column within the list
// of stock quotes.
type Column struct {
	width     int                 // Column width.
	name      string              // The name of the field in the Stock struct.
	title     string              // Column title to display in the header.
	formatter func(string) string // Optional function to format the contents of the column.
}

// Layout is used to format and display all the collected data, i.e. market
// updates and the list of stock quotes.
type Layout struct {
	columns        []Column           // List of stock quotes columns.
	sorter         *Sorter            // Pointer to sorting receiver.
	regex          *regexp.Regexp     // Pointer to regular expression to align decimal points.
	marketTemplate *template.Template // Pointer to template to format market data.
	quotesTemplate *template.Template // Pointer to template to format the list of stock quotes.
}

// Creates the layout and assigns the default values that stay unchanged.
func NewLayout() *Layout {
	layout := &Layout{}
	layout.columns = []Column{
		{-10, `Ticker`, `Ticker`, nil},
		{10, `LastTrade`, `Last`, currency},
		{10, `Change`, `Change`, currency},
		{10, `ChangePct`, `Change%`, last},
		{10, `Open`, `Open`, currency},
		{10, `Low`, `Low`, currency},
		{10, `High`, `High`, currency},
		{10, `Low52`, `52w Low`, currency},
		{10, `High52`, `52w High`, currency},
		{11, `Volume`, `Volume`, nil},
		{11, `AvgVolume`, `AvgVolume`, nil},
		{9, `PeRatio`, `P/E`, blank},
		{9, `Dividend`, `Dividend`, zero},
		{9, `Yield`, `Yield`, percent},
		{11, `MarketCap`, `MktCap`, currency},
	}
	layout.regex = regexp.MustCompile(`(\.\d+)[BMK]?$`)
	layout.marketTemplate = buildMarketTemplate()
	layout.quotesTemplate = buildQuotesTemplate()

	return layout
}

// Market merges given market data structure with the market template and
// returns formatted string that includes highlighting markup.
func (layout *Layout) Market(market *Market) string {
	if ok, err := market.Ok(); !ok { // If there was an error fetching market data...
		return err // then simply return the error string.
	}

	highlight(market.Dow, market.Sp500, market.Nasdaq,
		market.Tokyo, market.HongKong, market.London, market.Frankfurt,
		market.Yield, market.Oil, market.Euro, market.Gold)
	buffer := new(bytes.Buffer)
	layout.marketTemplate.Execute(buffer, market)

	return buffer.String()
}

// Quotes uses quotes template to format timestamp, stock quotes header,
// and the list of given stock quotes. It returns formatted string with
// all the necessary markup.
func (layout *Layout) Quotes(quotes *Quotes) string {
	if ok, err := quotes.Ok(); !ok { // If there was an error fetching stock quotes...
		return err // then simply return the error string.
	}

	vars := struct {
		Now    string  // Current timestamp.
		Header string  // Formatted header line.
		Stocks []Stock // List of formatted stock quotes.
	}{
		time.Now().Format(`3:04:05pm PST`),
		layout.Header(quotes.profile),
		layout.prettify(quotes),
	}

	buffer := new(bytes.Buffer)
	layout.quotesTemplate.Execute(buffer, vars)

	return buffer.String()
}

// Header iterates over column titles and formats the header line. The
// formatting includes placing an arrow next to the sorted column title.
// When the column editor is active it knows how to highlight currently
// selected column title.
func (layout *Layout) Header(profile *Profile) string {
	str, selectedColumn := ``, profile.selectedColumn

	for i, col := range layout.columns {
		arrow := arrowFor(i, profile)
		if i != selectedColumn {
			str += fmt.Sprintf(`%*s`, col.width, arrow+col.title)
		} else {
			str += fmt.Sprintf(`<r>%*s</r>`, col.width, arrow+col.title)
		}
	}

	return `<u>` + str + `</u>`
}

// TotalColumns is the utility method for the column editor that returns
// total number of columns.
func (layout *Layout) TotalColumns() int {
	return len(layout.columns)
}

//-----------------------------------------------------------------------------
func (layout *Layout) prettify(quotes *Quotes) []Stock {
	pretty := make([]Stock, len(quotes.stocks))
	//
	// Iterate over the list of stocks and properly format all its columns.
	//
	for i, stock := range quotes.stocks {
		pretty[i].Advancing = stock.Advancing
		//
		// Iterate over the list of stock columns. For each column name:
		// - Get current column value.
		// - If the column has the formatter method then call it.
		// - Set the column value padding it to the given width.
		//
		for _, column := range layout.columns {
			// ex. value = stock.Change
			value := reflect.ValueOf(&stock).Elem().FieldByName(column.name).String()
			if column.formatter != nil {
				// ex. value = currency(value)
				value = column.formatter(value)
			}
			// ex. pretty[i].Change = layout.pad(value, 10)
			reflect.ValueOf(&pretty[i]).Elem().FieldByName(column.name).SetString(layout.pad(value, column.width))
		}
	}

	profile := quotes.profile
	if layout.sorter == nil { // Initialize sorter on first invocation.
		layout.sorter = NewSorter(profile)
	}
	layout.sorter.SortByCurrentColumn(pretty)
	//
	// Group stocks by advancing/declining unless sorted by Chanage or Change%
	// in which case the grouping has been done already.
	//
	if profile.Grouped && (profile.SortColumn < 2 || profile.SortColumn > 3) {
		pretty = group(pretty)
	}

	return pretty
}

//-----------------------------------------------------------------------------
func (layout *Layout) pad(str string, width int) string {
	match := layout.regex.FindStringSubmatch(str)
	if len(match) > 0 {
		switch len(match[1]) {
		case 2:
			str = strings.Replace(str, match[1], match[1]+`0`, 1)
		case 4, 5:
			str = strings.Replace(str, match[1], match[1][0:3], 1)
		}
	}

	return fmt.Sprintf(`%*s`, width, str)
}

//-----------------------------------------------------------------------------
func buildMarketTemplate() *template.Template {
	markup := `<yellow>Dow</> {{.Dow.change}} ({{.Dow.percent}}) at {{.Dow.latest}} <yellow>S&P 500</> {{.Sp500.change}} ({{.Sp500.percent}}) at {{.Sp500.latest}} <yellow>NASDAQ</> {{.Nasdaq.change}} ({{.Nasdaq.percent}}) at {{.Nasdaq.latest}}
<yellow>Tokyo</> {{.Tokyo.change}} ({{.Tokyo.percent}}) at {{.Tokyo.latest}} <yellow>HK</> {{.HongKong.change}} ({{.HongKong.percent}}) at {{.HongKong.latest}} <yellow>London</> {{.London.change}} ({{.London.percent}}) at {{.London.latest}} <yellow>Frankfurt</> {{.Frankfurt.change}} ({{.Frankfurt.percent}}) at {{.Frankfurt.latest}} {{if .IsClosed}}<right>U.S. markets closed</right>{{end}}
<yellow>10-Year Yield</> {{.Yield.latest}}% ({{.Yield.change}}) <yellow>Euro</> ${{.Euro.latest}} ({{.Euro.change}}%) <yellow>Yen</> ¥{{.Yen.latest}} ({{.Yen.change}}%) <yellow>Oil</> ${{.Oil.latest}} ({{.Oil.change}}%) <yellow>Gold</> ${{.Gold.latest}} ({{.Gold.change}}%)`

	return template.Must(template.New(`market`).Parse(markup))
}

//-----------------------------------------------------------------------------
func buildQuotesTemplate() *template.Template {
	markup := `<right><white>{{.Now}}</></right>

{{.Header}}
{{range.Stocks}}{{if .Advancing}}<green>{{end}}{{.Ticker}}{{.LastTrade}}{{.Change}}{{.ChangePct}}{{.Open}}{{.Low}}{{.High}}{{.Low52}}{{.High52}}{{.Volume}}{{.AvgVolume}}{{.PeRatio}}{{.Dividend}}{{.Yield}}{{.MarketCap}}</>
{{end}}`

	return template.Must(template.New(`quotes`).Parse(markup))
}

//-----------------------------------------------------------------------------
func highlight(collections ...map[string]string) {
	for _, collection := range collections {
		if collection[`change`][0:1] != `-` {
			collection[`change`] = `<green>` + collection[`change`] + `</>`
		}
	}
}

//-----------------------------------------------------------------------------
func group(stocks []Stock) []Stock {
	grouped := make([]Stock, len(stocks))
	current := 0

	for _, stock := range stocks {
		if stock.Advancing {
			grouped[current] = stock
			current++
		}
	}
	for _, stock := range stocks {
		if !stock.Advancing {
			grouped[current] = stock
			current++
		}
	}

	return grouped
}

//-----------------------------------------------------------------------------
func arrowFor(column int, profile *Profile) string {
	if column == profile.SortColumn {
		if profile.Ascending {
			return string('\U00002191')
		}
		return string('\U00002193')
	}
	return ``
}

//-----------------------------------------------------------------------------
func blank(str string) string {
	if len(str) == 3 && str[0:3] == `N/A` {
		return `-`
	}

	return str
}

//-----------------------------------------------------------------------------
func zero(str string) string {
	if str == `0.00` {
		return `-`
	}

	return currency(str)
}

//-----------------------------------------------------------------------------
func last(str string) string {
	if len(str) >= 6 && str[0:6] == `N/A - ` {
		return str[6:]
	}

	return percent(str)
}

//-----------------------------------------------------------------------------
func currency(str string) string {
	if str == `N/A` || len(str) == 0 {
		return `-`
	}
	if sign := str[0:1]; sign == `+` || sign == `-` {
		return sign + `$` + str[1:]
	}

	return `$` + str
}

// Returns percent value truncated at 2 decimal points.
//-----------------------------------------------------------------------------
func percent(str string) string {
	if str == `N/A` || len(str) == 0 {
		return `-`
	}

	split := strings.Split(str, ".")
	if len(split) == 2 {
		digits := len(split[1])
		if digits > 2 {
			digits = 2
		}
		str = split[0] + "." + split[1][0:digits]
	}
	if str[len(str)-1] != '%' {
		str += `%`
	}
	return str
}
<|endoftext|>


<commit_before>package levlog

import (
	"log"
	"time"
)

type debugLevel struct {
	name		string
	id			int
}

var (
    DEBUG  debugLevel = debugLevel{"DEBUG",1}
    ERROR debugLevel =  debugLevel{"ERROR",2}
    FATAL debugLevel =  debugLevel{"FATAL",3}
)

var writer *RotateWriter

func SetOutput(filename string, timeRotation time.Duration){
	var err error
	writer,err = NewRotateWrite(filename)
	if err != nil{
		panic(err)
	}
	log.SetOutput(writer)
	TimeRotating(timeRotation)
}

var DEBUG_LEVEL debugLevel = DEBUG

func rotate()(error){
	if writer != nil{
		err := writer.Rotate()
		return err
	}
	return nil
}

func TimeRotating(dur time.Duration){
	go func (){
		for{
			select{
				case <-time.After(dur):
					err := rotate()
					if err != nil{
						panic(err)
					}
			}
		}
	}()
}

func D(v ...interface{}){
	if DEBUG_LEVEL.id <= DEBUG.id{
		log.Printf("(DEBUG): %s\n",v)
	}
}

func E(v ...interface{}){
	if DEBUG_LEVEL.id <= ERROR.id{
		log.Printf("(ERROR): %s\n",v)
	}
}

func F(v ...interface{}){
	if DEBUG_LEVEL.id <= FATAL.id{
		log.Printf("(FATAL): %s\n",v)
	}
}

func DF(format string,v ...interface{}){
	if DEBUG_LEVEL.id <= DEBUG.id{
		log.Printf("(DEBUG): " + format+"\n",v)
	}
}

func EF(format string,v ...interface{}){
	if DEBUG_LEVEL.id <= ERROR.id{
		log.Printf("(ERROR): " + format+"\n",v)
	}
}

func FF(format string,v ...interface{}){
	if DEBUG_LEVEL.id <= FATAL.id{
		log.Fatalf("(FATAL): " + format+"\n",v)
	}
}<commit_msg>add shutdown channel<commit_after>package levlog

import (
	"log"
	"time"
	"os"
	"path/filepath"
)

type debugLevel struct {
	name		string
	id			int
}

var (
    DEBUG  debugLevel = debugLevel{"DEBUG",1}
    ERROR debugLevel =  debugLevel{"ERROR",2}
    FATAL debugLevel =  debugLevel{"FATAL",3}
)

var writer *RotateWriter

func SetOutput(filename string, timeRotation time.Duration,shutdownChannel chan bool){
	var err error
	writer,err = NewRotateWrite(filename)
	if err != nil{
		panic(err)
	}
	log.SetOutput(writer)
	TimeRotating(timeRotation,shutdownChannel)
}

var DEBUG_LEVEL debugLevel = DEBUG

func rotate()(error){
	if writer != nil{
		err := writer.Rotate()
		return err
	}
	return nil
}

func TimeRotating(dur time.Duration,shutdownChannel chan bool){
	go func (shutdown chan bool){
		for{
			select{
				case <-time.After(dur):
					err := rotate()
					if err != nil{
						panic(err)
					}
				case <-shutdown:
					return
			}
		}
	}(shutdownChannel)
}

func PanicLogInFile(v ...interface{}){
	panicFileName := "." + string(filepath.Separator) + "logs" + string(filepath.Separator) + "panic.log"
	f, err := os.OpenFile(panicFileName, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
	if err != nil {
	    E("Error when open ",panicFileName,"for panic logging")
	    return
	}
	defer f.Close()

	log.SetOutput(f)
	log.Fatal(v)
}

func D(v ...interface{}){
	if DEBUG_LEVEL.id <= DEBUG.id{
		log.Printf("(DEBUG): %s\n",v)
	}
}

func E(v ...interface{}){
	if DEBUG_LEVEL.id <= ERROR.id{
		log.Printf("(ERROR): %s\n",v)
	}
}

func F(v ...interface{}){
	if DEBUG_LEVEL.id <= FATAL.id{
		log.Printf("(FATAL): %s\n",v)
	}
}

func DF(format string,v ...interface{}){
	if DEBUG_LEVEL.id <= DEBUG.id{
		log.Printf("(DEBUG): " + format+"\n",v)
	}
}

func EF(format string,v ...interface{}){
	if DEBUG_LEVEL.id <= ERROR.id{
		log.Printf("(ERROR): " + format+"\n",v)
	}
}

func FF(format string,v ...interface{}){
	if DEBUG_LEVEL.id <= FATAL.id{
		log.Fatalf("(FATAL): " + format+"\n",v)
	}
}<|endoftext|>


<commit_before>// Copyright (C) 2014 Miquel Sabaté Solà
// This file is licensed under the MIT license.
// See the LICENSE file.

package lib

import (
	"database/sql"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"github.com/coopernurse/gorp"
	_ "github.com/lib/pq"
)

// Public: get the configuration of the database for the current environment.
//
// file - The name of the JSON file describing the database configuration.
//
// Returns a string that it's supposed to be passed as the second argument
// of the sql.Open function.
func DBConfig(file string) string {
	// Read the contents and unmarshal the thing.
	root := FindRoot("vendrell", ".")
	contents, _ := ioutil.ReadFile(root + "/db/" + file)
	m := map[string]map[string]string{}
	json.Unmarshal(contents, &m)

	// Put it in a fancy string.
	current := m[Env()]
	size := len(current)
	cfg := ""
	i := 0
	for key, value := range current {
		if value != "" {
			cfg += key + "=" + value
			if i != size-1 {
				cfg += " "
			}
		}
		i++
	}
	return strings.TrimSpace(cfg)
}

// Returns a config URL for PostgreSQL when using Heroku.
func herokuUrl(env string) string {
	// Black magic to get the PostgreSQL config.
	rg := "(?i)^postgres://(?:([^:@]+):([^@]*)@)?([^@/:]+):(\\d+)/(.*)$"
	regex := regexp.MustCompile(rg)
	matches := regex.FindStringSubmatch(os.Getenv("DATABASE_URL"))
	if matches == nil {
		log.Fatalf("Wrong URL format!")
	}

	// And now we can build a proper url for PostgreSQL.
	fmt := "user=%s password=%s host=%s port=%s dbname=%s sslmode=%s"
	spec := fmt.Sprintf(fmt, matches[1], matches[2], matches[3], matches[4],
		matches[5], "disable")
	return spec
}

// Public: initializes the global Db variable.
//
// file - The name of the JSON file describing the database configuration.
//
// Returns a new DB instance.
func InitDB(file string) gorp.DbMap {
	// First of all get the URL (depends whether it's on Heroku or not).
	url := ""
	if h := os.Getenv("DATABASE_URL"); h != "" {
		url = herokuUrl(h)
	} else {
		url = DBConfig(file)
	}

	// Connect and return the connection.
	d, err := sql.Open("postgres", url)
	if err != nil {
		panic(err)
	}
	return gorp.DbMap{Db: d, Dialect: gorp.PostgresDialect{}}
}
<commit_msg>Fixed build error.<commit_after>// Copyright (C) 2014 Miquel Sabaté Solà
// This file is licensed under the MIT license.
// See the LICENSE file.

package lib

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"github.com/coopernurse/gorp"
	_ "github.com/lib/pq"
)

// Public: get the configuration of the database for the current environment.
//
// file - The name of the JSON file describing the database configuration.
//
// Returns a string that it's supposed to be passed as the second argument
// of the sql.Open function.
func DBConfig(file string) string {
	// Read the contents and unmarshal the thing.
	root := FindRoot("vendrell", ".")
	contents, _ := ioutil.ReadFile(root + "/db/" + file)
	m := map[string]map[string]string{}
	json.Unmarshal(contents, &m)

	// Put it in a fancy string.
	current := m[Env()]
	size := len(current)
	cfg := ""
	i := 0
	for key, value := range current {
		if value != "" {
			cfg += key + "=" + value
			if i != size-1 {
				cfg += " "
			}
		}
		i++
	}
	return strings.TrimSpace(cfg)
}

// Returns a config URL for PostgreSQL when using Heroku.
func herokuUrl(env string) string {
	// Black magic to get the PostgreSQL config.
	rg := "(?i)^postgres://(?:([^:@]+):([^@]*)@)?([^@/:]+):(\\d+)/(.*)$"
	regex := regexp.MustCompile(rg)
	matches := regex.FindStringSubmatch(os.Getenv("DATABASE_URL"))
	if matches == nil {
		log.Fatalf("Wrong URL format!")
	}

	// And now we can build a proper url for PostgreSQL.
	s := "user=%s password=%s host=%s port=%s dbname=%s sslmode=%s"
	spec := fmt.Sprintf(s, matches[1], matches[2], matches[3], matches[4],
		matches[5], "disable")
	return spec
}

// Public: initializes the global Db variable.
//
// file - The name of the JSON file describing the database configuration.
//
// Returns a new DB instance.
func InitDB(file string) gorp.DbMap {
	// First of all get the URL (depends whether it's on Heroku or not).
	url := ""
	if h := os.Getenv("DATABASE_URL"); h != "" {
		url = herokuUrl(h)
	} else {
		url = DBConfig(file)
	}

	// Connect and return the connection.
	d, err := sql.Open("postgres", url)
	if err != nil {
		panic(err)
	}
	return gorp.DbMap{Db: d, Dialect: gorp.PostgresDialect{}}
}
<|endoftext|>


<commit_before>package sanntid

import (
	"fmt"
	"github.com/franela/goreq"
)

type sanntidMonitoredCall struct {
	ExpectedArrivalTime string
	DeparturePlatformName string
	DestinationDisplay string
}

type sanntidMonitoredVehicleJourney struct {
	DestinationName string
	MonitoredCall sanntidMonitoredCall
	PublishedLineName string
	VehicleMode int
}

type sanntidArrivalData struct {
	MonitoredVehicleJourney sanntidMonitoredVehicleJourney
}

func requestArrivalData(locationId int) ([]sanntidArrivalData, error) {
	var data []sanntidArrivalData

	url := fmt.Sprintf("http://reisapi.ruter.no/stopvisit/getdepartures/%d", locationId)
	res, err := goreq.Request{ Uri: url }.Do()

	if err == nil {
		res.Body.FromJsonTo(&data)
	}

	return data, err
}
<commit_msg>Use built in packages to fetch data<commit_after>package sanntid

import (
	"fmt"
	"encoding/json"
	"io/ioutil"
	"net/http"
)

type sanntidMonitoredCall struct {
	ExpectedArrivalTime string
	DeparturePlatformName string
	DestinationDisplay string
}

type sanntidMonitoredVehicleJourney struct {
	DestinationName string
	MonitoredCall sanntidMonitoredCall
	PublishedLineName string
	VehicleMode int
}

type sanntidArrivalData struct {
	MonitoredVehicleJourney sanntidMonitoredVehicleJourney
}

func requestArrivalData(locationId int) ([]sanntidArrivalData, error) {
	var data []sanntidArrivalData

	url := fmt.Sprintf("http://reisapi.ruter.no/stopvisit/getdepartures/%d", locationId)

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	json.Unmarshal(body, &data)

	return data, err
}
<|endoftext|>


<commit_before>package spf

import (
	"net"

	"github.com/zaccone/goSPF/dns"
)

type SPFResult int

const (
	Illegal SPFResult = iota

	None
	Neutral
	Pass
	Fail
	Softfail
	Temperror
	Permerror

	// end of SPF
	SPFEnd
)

func (spf *SPFResult) String() string {
	switch *spf {
	case None:
		return "None"
	case Neutral:
		return "Neutral"
	case Pass:
		return "Pass"
	case Fail:
		return "Fail"
	case Temperror:
		return "Temperror"
	case Permerror:
		return "Permerror"
	default:
		return "Permerror"
	}
}

// CheckHost is a main entrypoint function evaluating e-mail with regard to SPF
// As per RFC 7208 it will accept 3 parameters (strings):
// <ip> - IP{4,6} address of the connected client
// <domain> - domain portion of the MAIL FROM or HELO identity
// <sender> - MAIL FROM or HELO identity
// All the parameters should be parsed and dereferenced from real email fields.
// This means domain should already be extracted from MAIL FROM field so this
// function can focus on the core part.
func checkHost(ip, domain, sender string) (SPFResult, error) {

	// TODO(zaccone) s/_/spfRecord/
	_, dnsErr := dns.LookupSPF(domain)

	if dnsErr != nil {
		switch dnsErr.(type) {
		// as per RFC7208 section 4.4, and DNS query errors result in Temperror
		//result immediately
		case *net.DNSError:
			return Temperror, dnsErr
		default:
			return Permerror, dnsErr

		}
	}
	query := "v=spf1 a -all"
	ipAddr := net.ParseIP(ip)
	parser := NewParser(sender, domain, ipAddr, query)

	var result SPFResult = Neutral
	var err error

	if result, err = parser.Parse(); err != nil {
		// handle error, something went wrong.
		// Let's set PermError for now.
		return Permerror, nil
	}

	// check if there is redirect and recursively call check_host with redirect
	// as domain
	if parser.Redirect != nil {
		result, err = checkHost(ip, domain, sender)
	}

	// return SPF evaluation result
	return result, nil
}
<commit_msg>Change checkHost signature.<commit_after>package spf

import (
	"net"

	"github.com/zaccone/goSPF/dns"
)

type SPFResult int

const (
	Illegal SPFResult = iota

	None
	Neutral
	Pass
	Fail
	Softfail
	Temperror
	Permerror

	// end of SPF
	SPFEnd
)

func (spf *SPFResult) String() string {
	switch *spf {
	case None:
		return "None"
	case Neutral:
		return "Neutral"
	case Pass:
		return "Pass"
	case Fail:
		return "Fail"
	case Temperror:
		return "Temperror"
	case Permerror:
		return "Permerror"
	default:
		return "Permerror"
	}
}

// CheckHost is a main entrypoint function evaluating e-mail with regard to SPF
// As per RFC 7208 it will accept 3 parameters (strings):
// <ip> - IP{4,6} address of the connected client
// <domain> - domain portion of the MAIL FROM or HELO identity
// <sender> - MAIL FROM or HELO identity
// All the parameters should be parsed and dereferenced from real email fields.
// This means domain should already be extracted from MAIL FROM field so this
// function can focus on the core part.
func checkHost(ip net.IP, domain, sender string) (SPFResult, error) {

	// TODO(zaccone) s/_/spfRecord/
	_, dnsErr := dns.LookupSPF(domain)

	if dnsErr != nil {
		switch dnsErr.(type) {
		// as per RFC7208 section 4.4, and DNS query errors result in Temperror
		//result immediately
		case *net.DNSError:
			return Temperror, dnsErr
		default:
			return Permerror, dnsErr

		}
	}
	query := "v=spf1 a -all"
	parser := NewParser(sender, domain, ip, query)

	var result SPFResult = Neutral
	var err error

	if result, err = parser.Parse(); err != nil {
		// handle error, something went wrong.
		// Let's set PermError for now.
		return Permerror, nil
	}

	// check if there is redirect and recursively call check_host with redirect
	// as domain
	if parser.Redirect != nil {
		result, err = checkHost(ip, domain, sender)
	}

	// return SPF evaluation result
	return result, nil
}
<|endoftext|>