<commit_before>package main

import "log"

func saveFile(name int, hash string) {
	log.Println(buldFileURL(name), hash, " downloaded!")
	// 	out, err := os.Create("output.txt")
	// defer out.Close()
	// ...
	// resp, err := http.Get("http://example.com/")
	// defer resp.Body.Close()
	// ...
	// n, err := io.Copy(out, resp.Body)
}

func createFolder() {
	log.Println("test")
}
<commit_msg>wip, refact<commit_after>package main

import (
	"log"
	"os"
	"time"
)

func saveFile(name int, hash string) {
	// 	out, err := os.Create("output.txt")
	// defer out.Close()
	// ...
	// resp, err := http.Get("http://example.com/")
	// defer resp.Body.Close()
	// ...
	// n, err := io.Copy(out, resp.Body)

	if checkUniq(hash) {
		createFolder()
		log.Println(buldFileURL(name), hash, " downloaded!")
		return
	}

}

func createFolder() string {
	t := time.Now()
	path := "files/" + t.Format("2006-01-02") + "/"
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.Mkdir(path, 0755)
		t = t.Add(-24 * time.Hour)
	}
	return path
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "os"
import "log"
import "flag"
import "io"
import md5 "crypto/md5"
import hex "encoding/hex"
import "path"

func walkDirectory(root string) <-chan string {
	out := make(chan string)
	var queue []string
	queue = append(queue, root)

	go func() {
		for len(queue) > 0 {
			d := queue[0]
			queue = queue[:1]

			f,err := os.Open(d)
			if err != nil {
				log.Print("Unable to open: %s\n", d)
				continue
			}

			for {
				infos,err := f.Readdir(100)
				if err == io.EOF {
					log.Print("EOF on readdir")
					break;
				}
				if err != nil {
					log.Fatal("Readdir failed")
				}

				for _,stat := range infos {
					full_path := path.Join(d, stat.Name())
					if stat.IsDir() {
						queue = append(queue, full_path)
					} else {
						out <- full_path
					}
				}
			}

		}
		close(out)
	}()

	return out
}

func hashFiles(files <-chan string) {
	for path := range files {
		f,err := os.Open(path)
		if err != nil {
			log.Printf("Couldn't open %s. Skipping it.", path)
		}
		for {
			b := make([]byte, 1<<20)
			n,err := f.Read(b)
			if n == 0 {
				log.Printf("EOF on %s", f.Name())
				break
			} else if err != nil {
				log.Fatal("Non EOF error on ", f.Name())
			}
			csum := md5.Sum(b[:n])
			fmt.Printf("%s: %s\n", f.Name(), hex.EncodeToString(csum[:]))
		}
	}
}

func main() {
	root := flag.String("directory", "", "Directory to scan")
	flag.Parse()

	files := walkDirectory(*root)
	hashFiles(files)
}
<commit_msg>define a chunk struct and return a channel of them<commit_after>package main

import "fmt"
import "os"
import "log"
import "flag"
import "io"
import md5 "crypto/md5"
import hex "encoding/hex"
import "path"

func walkDirectory(root string) <-chan string {
	out := make(chan string)
	var queue []string
	queue = append(queue, root)

	go func() {
		for len(queue) > 0 {
			d := queue[0]
			queue = queue[:1]

			f,err := os.Open(d)
			if err != nil {
				log.Print("Unable to open: %s\n", d)
				continue
			}

			for {
				infos,err := f.Readdir(100)
				if err == io.EOF {
					log.Print("EOF on readdir")
					break;
				}
				if err != nil {
					log.Fatal("Readdir failed")
				}

				for _,stat := range infos {
					full_path := path.Join(d, stat.Name())
					if stat.IsDir() {
						queue = append(queue, full_path)
					} else {
						out <- full_path
					}
				}
			}

		}
		close(out)
	}()

	return out
}

type Chunk struct {
	filename string
	offset   int64
	md5sum   string
}

func hashFiles(files <-chan string) <-chan Chunk {
	out := make(chan Chunk)

	go func() { for path := range files {
		f,err := os.Open(path)
		if err != nil {
			log.Printf("Couldn't open %s. Skipping it.", path)
		}
		var i int64
		i = 0
		for {
			b := make([]byte, 1<<20)
			n,err := f.Read(b)
			if n == 0 {
				log.Printf("EOF on %s", f.Name())
				break
			} else if err != nil {
				log.Fatal("Non EOF error on ", f.Name())
			}
			csum := md5.Sum(b[:n])
			c := Chunk{filename: path, offset: i, md5sum: hex.EncodeToString(csum[:])}
			out <- c
			i += int64(n)
		}
	}
		close(out)
	}()
	return out
}

func main() {
	root := flag.String("directory", "", "Directory to scan")
	flag.Parse()

	files := walkDirectory(*root)
	for c := range hashFiles(files) {
		fmt.Printf("%s (%d): %s\n", c.filename, c.offset, c.md5sum)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(config.ImportPathRoot), filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// Make go1.5 style vendored dep unit names (package import paths)
	// relative to vendored dir, not to top-level dir.
	for _, u := range units {
		pkg := u.Data.(*build.Package)
		i, ok := findVendor(pkg.Dir)
		if !ok {
			continue
		}
		relDir := pkg.Dir[i+len("vendor"):]
		if strings.HasPrefix(relDir, "/src/") || !strings.HasPrefix(relDir, "/") {
			continue
		}
		relImport := relDir[1:]
		u.Name = relImport
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.ToSlash(filepath.Join(pkgSubdir, f))
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	// Find vendored units to build a list of vendor directories
	vendorDirs := map[string]struct{}{}
	for _, u := range units {
		i, ok := findVendor(u.Dir)
		// Don't include old style vendor dirs
		if !ok || strings.HasPrefix(u.Dir[i:], "vendor/src/") {
			continue
		}
		vendorDirs[u.Dir[:i+len("vendor")]] = struct{}{}
	}

	for _, u := range units {
		if u.Config == nil {
			u.Config = map[string]interface{}{}
		}

		unitDir := u.Dir + string(filepath.Separator)
		var dirs vendorDirSlice
		for dir := range vendorDirs {
			// Must be a child of baseDir to use the vendor dir
			baseDir := filepath.Dir(dir) + string(filepath.Separator)
			if filepath.Clean(baseDir) == "." || strings.HasPrefix(unitDir, baseDir) {
				dirs = append(dirs, dir)
			}
		}
		sort.Sort(dirs)
		u.Config["VendorDirs"] = dirs
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

// findVendor from golang/go/cmd/go/pkg.go
func findVendor(path string) (index int, ok bool) {
	// Two cases, depending on internal at start of string or not.
	// The order matters: we must return the index of the final element,
	// because the final one is where the effective import path starts.
	switch {
	case strings.Contains(path, "/vendor/"):
		return strings.LastIndex(path, "/vendor/") + 1, true
	case strings.HasPrefix(path, "vendor/"):
		return 0, true
	}
	return 0, false
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		pkg.Dir, err = filepath.Rel(scanDir, pkg.Dir)
		if err != nil {
			return nil, err
		}
		pkg.Dir = filepath.ToSlash(pkg.Dir)
		pkg.BinDir = ""
		pkg.ConflictDir = ""

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""
		pkg.SrcRoot = ""
		pkg.PkgRoot = ""

		pkg.ImportPos = nil
		pkg.TestImportPos = nil
		pkg.XTestImportPos = nil

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if err != nil {
		log.Printf("Error scanning %s for packages: %v. Ignoring source files in this directory.", dir, err)
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(filepath.ToSlash(fullPath), "/Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				return nil, err
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}

// StringSlice attaches the methods of sort.Interface to []string, sorting in decreasing string length
type vendorDirSlice []string

func (p vendorDirSlice) Len() int           { return len(p) }
func (p vendorDirSlice) Less(i, j int) bool { return len(p[i]) >= len(p[j]) }
func (p vendorDirSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<commit_msg>scan: Only set VendorDirs if non-empty<commit_after>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(config.ImportPathRoot), filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// Make go1.5 style vendored dep unit names (package import paths)
	// relative to vendored dir, not to top-level dir.
	for _, u := range units {
		pkg := u.Data.(*build.Package)
		i, ok := findVendor(pkg.Dir)
		if !ok {
			continue
		}
		relDir := pkg.Dir[i+len("vendor"):]
		if strings.HasPrefix(relDir, "/src/") || !strings.HasPrefix(relDir, "/") {
			continue
		}
		relImport := relDir[1:]
		u.Name = relImport
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.ToSlash(filepath.Join(pkgSubdir, f))
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	// Find vendored units to build a list of vendor directories
	vendorDirs := map[string]struct{}{}
	for _, u := range units {
		i, ok := findVendor(u.Dir)
		// Don't include old style vendor dirs
		if !ok || strings.HasPrefix(u.Dir[i:], "vendor/src/") {
			continue
		}
		vendorDirs[u.Dir[:i+len("vendor")]] = struct{}{}
	}

	for _, u := range units {
		unitDir := u.Dir + string(filepath.Separator)
		var dirs vendorDirSlice
		for dir := range vendorDirs {
			// Must be a child of baseDir to use the vendor dir
			baseDir := filepath.Dir(dir) + string(filepath.Separator)
			if filepath.Clean(baseDir) == "." || strings.HasPrefix(unitDir, baseDir) {
				dirs = append(dirs, dir)
			}
		}
		sort.Sort(dirs)
		if len(dirs) > 0 {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}
			u.Config["VendorDirs"] = dirs
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

// findVendor from golang/go/cmd/go/pkg.go
func findVendor(path string) (index int, ok bool) {
	// Two cases, depending on internal at start of string or not.
	// The order matters: we must return the index of the final element,
	// because the final one is where the effective import path starts.
	switch {
	case strings.Contains(path, "/vendor/"):
		return strings.LastIndex(path, "/vendor/") + 1, true
	case strings.HasPrefix(path, "vendor/"):
		return 0, true
	}
	return 0, false
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		pkg.Dir, err = filepath.Rel(scanDir, pkg.Dir)
		if err != nil {
			return nil, err
		}
		pkg.Dir = filepath.ToSlash(pkg.Dir)
		pkg.BinDir = ""
		pkg.ConflictDir = ""

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""
		pkg.SrcRoot = ""
		pkg.PkgRoot = ""

		pkg.ImportPos = nil
		pkg.TestImportPos = nil
		pkg.XTestImportPos = nil

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if err != nil {
		log.Printf("Error scanning %s for packages: %v. Ignoring source files in this directory.", dir, err)
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(filepath.ToSlash(fullPath), "/Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				return nil, err
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}

// StringSlice attaches the methods of sort.Interface to []string, sorting in decreasing string length
type vendorDirSlice []string

func (p vendorDirSlice) Len() int           { return len(p) }
func (p vendorDirSlice) Less(i, j int) bool { return len(p[i]) >= len(p[j]) }
func (p vendorDirSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/crackcomm/go-clitable"
	"github.com/fatih/structs"
	"github.com/maliceio/go-plugin-utils/database/elasticsearch"
	"github.com/maliceio/go-plugin-utils/utils"
	"github.com/parnurzeal/gorequest"
	"github.com/urfave/cli"
)

// Version stores the plugin's version
var Version string

// BuildTime stores the plugin's build time
var BuildTime string

const (
	name     = "fileinfo"
	category = "metadata"
)

type pluginResults struct {
	ID       string   `structs:"id"`
	FileInfo FileInfo `structs:"fileinfo"`
}

// FileInfo json object
type FileInfo struct {
	SSDeep   string            `json:"ssdeep" structs:"ssdeep"`
	TRiD     []string          `json:"trid" structs:"trid"`
	Exiftool map[string]string `json:"exiftool" structs:"exiftool"`
}

// ParseExiftoolOutput convert exiftool output into JSON
func ParseExiftoolOutput(exifout string) map[string]string {

	var ignoreTags = []string{
		"Directory",
		"File Name",
		"File Permissions",
		"File Modification Date/Time",
	}

	lines := strings.Split(exifout, "\n")
	if !(len(lines) > 1) {
		return ""
	}
	datas := make(map[string]string, len(lines))

	for _, line := range lines {
		keyvalue := strings.Split(line, ":")
		if len(keyvalue) != 2 {
			continue
		}
		if !utils.StringInSlice(strings.TrimSpace(keyvalue[0]), ignoreTags) {
			datas[strings.TrimSpace(utils.CamelCase(keyvalue[0]))] = strings.TrimSpace(keyvalue[1])
		}
	}

	return datas
}

// ParseSsdeepOutput convert ssdeep output into JSON
func ParseSsdeepOutput(ssdout string) string {

	// Break output into lines
	lines := strings.Split(ssdout, "\n")
	if !(len(lines) > 1) {
		return ""
	}
	// Break second line into hash and path
	hashAndPath := strings.Split(lines[1], ",")

	return strings.TrimSpace(hashAndPath[0])
}

// ParseTRiDOutput convert trid output into JSON
func ParseTRiDOutput(tridout string) []string {

	keepLines := []string{}

	lines := strings.Split(tridout, "\n")
	if !(len(lines) > 1) {
		return ""
	}
	lines = lines[6:]
	// fmt.Println(lines)

	for _, line := range lines {
		if len(strings.TrimSpace(line)) != 0 {
			keepLines = append(keepLines, strings.TrimSpace(line))
		}
	}

	return keepLines
}

func printStatus(resp gorequest.Response, body string, errs []error) {
	fmt.Println(resp.Status)
}

func printMarkDownTable(finfo FileInfo) {

	// print ssdeep
	fmt.Println("#### SSDeep")
	fmt.Println(finfo.SSDeep)
	fmt.Println()
	// print trid
	fmt.Println("#### TRiD")
	table := clitable.New([]string{"TRiD", ""})
	for _, trd := range finfo.TRiD {
		fmt.Println(" - ", trd)
	}
	fmt.Println()
	// print exiftool
	fmt.Println("#### Exiftool")
	table = clitable.New([]string{"Field", "Value"})
	for key, value := range finfo.Exiftool {
		table.AddRow(map[string]interface{}{"Field": key, "Value": value})
	}
	table.Markdown = true
	table.Print()
}

var appHelpTemplate = `Usage: {{.Name}} {{if .Flags}}[OPTIONS] {{end}}COMMAND [arg...]

{{.Usage}}

Version: {{.Version}}{{if or .Author .Email}}

Author:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}
{{if .Flags}}
Options:
  {{range .Flags}}{{.}}
  {{end}}{{end}}
Commands:
  {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
  {{end}}
Run '{{.Name}} COMMAND --help' for more information on a command.
`

func main() {
	cli.AppHelpTemplate = appHelpTemplate
	app := cli.NewApp()
	app.Name = "fileinfo"
	app.Author = "blacktop"
	app.Email = "https://github.com/blacktop"
	app.Version = Version + ", BuildTime: " + BuildTime
	app.Compiled, _ = time.Parse("20060102", BuildTime)
	app.Usage = "Malice File Info Plugin - ssdeep/exiftool/TRiD"
	var elasitcsearch string
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, V",
			Usage: "verbose output",
		},
		cli.BoolFlag{
			Name:  "table, t",
			Usage: "output as Markdown table",
		},
		cli.BoolFlag{
			Name:   "post, p",
			Usage:  "POST results to Malice webhook",
			EnvVar: "MALICE_ENDPOINT",
		},
		cli.BoolFlag{
			Name:   "proxy, x",
			Usage:  "proxy settings for Malice webhook endpoint",
			EnvVar: "MALICE_PROXY",
		},
		cli.StringFlag{
			Name:        "elasitcsearch",
			Value:       "",
			Usage:       "elasitcsearch address for Malice to store results",
			EnvVar:      "MALICE_ELASTICSEARCH",
			Destination: &elasitcsearch,
		},
	}
	app.Action = func(c *cli.Context) error {
		path := c.Args().First()

		if _, err := os.Stat(path); os.IsNotExist(err) {
			utils.Assert(err)
		}

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		fileInfo := FileInfo{
			SSDeep:   ParseSsdeepOutput(utils.RunCommand("ssdeep", path)),
			TRiD:     ParseTRiDOutput(utils.RunCommand("trid", path)),
			Exiftool: ParseExiftoolOutput(utils.RunCommand("exiftool", path)),
		}

		// upsert into Database
		elasticsearch.InitElasticSearch()
		elasticsearch.WritePluginResultsToDatabase(elasticsearch.PluginResults{
			ID:       utils.Getopt("MALICE_SCANID", utils.GetSHA256(path)),
			Name:     name,
			Category: category,
			Data:     structs.Map(fileInfo),
		})

		if c.Bool("table") {
			printMarkDownTable(fileInfo)
		} else {
			fileInfoJSON, err := json.Marshal(fileInfo)
			utils.Assert(err)
			if c.Bool("post") {
				request := gorequest.New()
				if c.Bool("proxy") {
					request = gorequest.New().Proxy(os.Getenv("MALICE_PROXY"))
				}
				request.Post(os.Getenv("MALICE_ENDPOINT")).
					Set("Task", path).
					Send(fileInfoJSON).
					End(printStatus)
			}
			// write to stdout
			fmt.Println(string(fileInfoJSON))
		}
		return nil
	}

	err := app.Run(os.Args)
	utils.Assert(err)
}
<commit_msg>fixes<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/crackcomm/go-clitable"
	"github.com/fatih/structs"
	"github.com/maliceio/go-plugin-utils/database/elasticsearch"
	"github.com/maliceio/go-plugin-utils/utils"
	"github.com/parnurzeal/gorequest"
	"github.com/urfave/cli"
)

// Version stores the plugin's version
var Version string

// BuildTime stores the plugin's build time
var BuildTime string

const (
	name     = "fileinfo"
	category = "metadata"
)

type pluginResults struct {
	ID       string   `structs:"id"`
	FileInfo FileInfo `structs:"fileinfo"`
}

// FileInfo json object
type FileInfo struct {
	SSDeep   string            `json:"ssdeep" structs:"ssdeep"`
	TRiD     []string          `json:"trid" structs:"trid"`
	Exiftool map[string]string `json:"exiftool" structs:"exiftool"`
}

// ParseExiftoolOutput convert exiftool output into JSON
func ParseExiftoolOutput(exifout string) map[string]string {

	var ignoreTags = []string{
		"Directory",
		"File Name",
		"File Permissions",
		"File Modification Date/Time",
	}

	lines := strings.Split(exifout, "\n")
	if !(len(lines) > 1) {
		return nil
	}
	datas := make(map[string]string, len(lines))

	for _, line := range lines {
		keyvalue := strings.Split(line, ":")
		if len(keyvalue) != 2 {
			continue
		}
		if !utils.StringInSlice(strings.TrimSpace(keyvalue[0]), ignoreTags) {
			datas[strings.TrimSpace(utils.CamelCase(keyvalue[0]))] = strings.TrimSpace(keyvalue[1])
		}
	}

	return datas
}

// ParseSsdeepOutput convert ssdeep output into JSON
func ParseSsdeepOutput(ssdout string) string {

	// Break output into lines
	lines := strings.Split(ssdout, "\n")
	if !(len(lines) > 1) {
		return ""
	}
	// Break second line into hash and path
	hashAndPath := strings.Split(lines[1], ",")

	return strings.TrimSpace(hashAndPath[0])
}

// ParseTRiDOutput convert trid output into JSON
func ParseTRiDOutput(tridout string) []string {

	keepLines := []string{}

	lines := strings.Split(tridout, "\n")
	if !(len(lines) > 1) {
		return nil
	}
	lines = lines[6:]
	// fmt.Println(lines)

	for _, line := range lines {
		if len(strings.TrimSpace(line)) != 0 {
			keepLines = append(keepLines, strings.TrimSpace(line))
		}
	}

	return keepLines
}

func printStatus(resp gorequest.Response, body string, errs []error) {
	fmt.Println(resp.Status)
}

func printMarkDownTable(finfo FileInfo) {

	// print ssdeep
	fmt.Println("#### SSDeep")
	fmt.Println(finfo.SSDeep)
	fmt.Println()
	// print trid
	fmt.Println("#### TRiD")
	table := clitable.New([]string{"TRiD", ""})
	for _, trd := range finfo.TRiD {
		fmt.Println(" - ", trd)
	}
	fmt.Println()
	// print exiftool
	fmt.Println("#### Exiftool")
	table = clitable.New([]string{"Field", "Value"})
	for key, value := range finfo.Exiftool {
		table.AddRow(map[string]interface{}{"Field": key, "Value": value})
	}
	table.Markdown = true
	table.Print()
}

var appHelpTemplate = `Usage: {{.Name}} {{if .Flags}}[OPTIONS] {{end}}COMMAND [arg...]

{{.Usage}}

Version: {{.Version}}{{if or .Author .Email}}

Author:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}
{{if .Flags}}
Options:
  {{range .Flags}}{{.}}
  {{end}}{{end}}
Commands:
  {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
  {{end}}
Run '{{.Name}} COMMAND --help' for more information on a command.
`

func main() {
	cli.AppHelpTemplate = appHelpTemplate
	app := cli.NewApp()
	app.Name = "fileinfo"
	app.Author = "blacktop"
	app.Email = "https://github.com/blacktop"
	app.Version = Version + ", BuildTime: " + BuildTime
	app.Compiled, _ = time.Parse("20060102", BuildTime)
	app.Usage = "Malice File Info Plugin - ssdeep/exiftool/TRiD"
	var elasitcsearch string
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, V",
			Usage: "verbose output",
		},
		cli.BoolFlag{
			Name:  "table, t",
			Usage: "output as Markdown table",
		},
		cli.BoolFlag{
			Name:   "post, p",
			Usage:  "POST results to Malice webhook",
			EnvVar: "MALICE_ENDPOINT",
		},
		cli.BoolFlag{
			Name:   "proxy, x",
			Usage:  "proxy settings for Malice webhook endpoint",
			EnvVar: "MALICE_PROXY",
		},
		cli.StringFlag{
			Name:        "elasitcsearch",
			Value:       "",
			Usage:       "elasitcsearch address for Malice to store results",
			EnvVar:      "MALICE_ELASTICSEARCH",
			Destination: &elasitcsearch,
		},
	}
	app.Action = func(c *cli.Context) error {
		path := c.Args().First()

		if _, err := os.Stat(path); os.IsNotExist(err) {
			utils.Assert(err)
		}

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		fileInfo := FileInfo{
			SSDeep:   ParseSsdeepOutput(utils.RunCommand("ssdeep", path)),
			TRiD:     ParseTRiDOutput(utils.RunCommand("trid", path)),
			Exiftool: ParseExiftoolOutput(utils.RunCommand("exiftool", path)),
		}

		// upsert into Database
		elasticsearch.InitElasticSearch()
		elasticsearch.WritePluginResultsToDatabase(elasticsearch.PluginResults{
			ID:       utils.Getopt("MALICE_SCANID", utils.GetSHA256(path)),
			Name:     name,
			Category: category,
			Data:     structs.Map(fileInfo),
		})

		if c.Bool("table") {
			printMarkDownTable(fileInfo)
		} else {
			fileInfoJSON, err := json.Marshal(fileInfo)
			utils.Assert(err)
			if c.Bool("post") {
				request := gorequest.New()
				if c.Bool("proxy") {
					request = gorequest.New().Proxy(os.Getenv("MALICE_PROXY"))
				}
				request.Post(os.Getenv("MALICE_ENDPOINT")).
					Set("Task", path).
					Send(fileInfoJSON).
					End(printStatus)
			}
			// write to stdout
			fmt.Println(string(fileInfoJSON))
		}
		return nil
	}

	err := app.Run(os.Args)
	utils.Assert(err)
}
<|endoftext|>


<commit_before>package segygo

import (
	"bytes"
	"encoding/binary"
	"io/ioutil"
	//"log"
	"os"
	"unsafe"
	"reflect"
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("segygo")
var format = logging.MustStringFormatter(
	`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
)

const Version = "0.1"
const SEGY_DESC_HDR_LEN = 3200
const SEGY_BIN_HDR_LEN = 400
const SEGY_TRACE_HDR_LEN = 240

type BinHeader struct {
	Jobid, Lino, Reno                                                                                int32
	Ntrpr, Nart                                                                                int16
	Hdt, Dto, Hns, Nso                                                                                uint16
	Format, Fold, Tsort, Vscode, Hsfs, Hsfe, Hslen, Hstyp, Schn, Hstas, Hstae, Htatyp, Hcorr, Bgrcv, Rcvm, Mfeet, Polyt, Vgpol int16
	Hunass                                                                                [170]int16 // unassigned
}

type TraceHeader struct {
	Tracel                                                               int32
	Tracer                                                               int32
	Fldr                                                                 int32
	Tracf                                                                int32
	Ep                                                                   int32
	CDP                                                                  int32
	CDPT                                                                 int32
	Trid                                                                 int16
	Nvs                                                                  int16
	Nhs                                                                  int16
	Duse                                                                 int16
	Offset                                                               int32
	Gelev                                                                int32
	Selev                                                                int32
	Sdepth                                                               int32
	Gdel                                                                 int32
	Sdel                                                                 int32
	SwDep                                                                int32
	GwDep                                                                int32
	Scalel                                                               int16
	Scalco                                                               int16
	Sx                                                                   int32
	Sy                                                                   int32
	Gx                                                                   int32
	Gy                                                                   int32
	CoUnit                                                               int16
	WeVel                                                                int16
	SweVel                                                               int16
	Sut, Gut, Sstat, Gstat, Tstat, Laga, Lagb, Delrt, Muts, Mute         int16
	Ns, Dt                                                               uint16
	Gain, Igc, Igi, Corr, Sfs, Sfe, Slen, Styp, Stas, Stae, Tatyp        int16
	Afilf, Afils, NoFilf, NoFils, Lcf, Hcf, Lcs, Hcs, Year, Day          int16
	Hour, Minute, Sec, Timbas, Trwf, Grnors, Grnofr, Grnlof, Gaps, Otrav int16
	D1, F1, D2, F2, Ungpow, Unscale                                      float32
	Ntr                                                                  int32
	Mark, Shortpad                                                       int16
	Unass                                                                [14]int16 // unassigned short array
}

type Trace struct {
	TraceHeader
	Data []float32
}

type SegyFile struct {
	Filename string
	Header   BinHeader
	NrTraces int64
	file     *os.File
	Position int64
	LogLevel logging.Level
}

func CreateFile(filename string) (SegyFile, error) {
	var s SegyFile
	var binHdr BinHeader
	f, err := os.Create(filename)
	defer f.Close()

	if err != nil {
		return s, err
	}

	s.LogLevel = logging.WARNING

	// Setup proper logging
	backend1 := logging.NewLogBackend(os.Stderr, "", 0)
	backend1Formatter := logging.NewBackendFormatter(backend1, format)
	logging.SetBackend(backend1Formatter)
	logging.SetLevel(s.LogLevel, "")

	log.Debugf("Creating SEG-Y file: %s", s.Filename)

	s.Filename = filename
	s.Header = binHdr
	s.NrTraces = 0
	s.file = f
	s.Position = 0

	accum := make([]byte, 3200)
	//r := bytes.NewWriter(accum)
	//binary.Write()
	buff := bytes.NewBuffer(accum)
	if err = binary.Write(buff, binary.BigEndian, &s.Header); err != nil {
		log.Errorf("Error creating buffer to hold binary header for segy file: %s. Msg: %s", s.Filename, err)
		return s, err
	}

	n, err := f.Write(buff.Bytes())
	if err != nil {
		log.Errorf("Error writing binary header to segy file: %s. Msg: %s", s.Filename, err)
		return s, err
	}
	log.Debugf("Wrote %d bytes to file: %s", n, s.Filename)

	return s, err

}

func OpenFile(filename string) (SegyFile, error) {
	var s SegyFile
	var binHdr BinHeader

	s.Filename = filename
	s.LogLevel = logging.WARNING

	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return s, err
	}

	// Setup proper logging
	backend1 := logging.NewLogBackend(os.Stderr, "", 0)
	backend1Formatter := logging.NewBackendFormatter(backend1, format)
	logging.SetBackend(backend1Formatter)
	logging.SetLevel(s.LogLevel, "")

	accum := []byte{}
	accum = append(accum, b...)

	accum2 := accum[3200:]
	r := bytes.NewReader(accum2)
	log.Debugf("Number of bytes: %d", r.Len())

	if err = binary.Read(r, binary.BigEndian, &binHdr); err != nil {
		log.Errorf("Error reading segy file (bigendian). %s", err)
		return s, err
	}

	// Open and store the os.File object in our struct
	file, err := os.Open(s.Filename)
	s.file = file
	defer file.Close()

	s.Header = binHdr
	s.NrTraces = s.GetNrTraces()

	return s, err
}

func (s *SegyFile) SetVerbose(verbose bool) {

	if verbose {
		s.LogLevel = logging.DEBUG
		logging.SetLevel(s.LogLevel, "")
	} else {
		s.LogLevel = logging.WARNING
		logging.SetLevel(s.LogLevel, "")
	}

}

func (s *SegyFile) GetNrTraces() int64 {
	fi, err := s.file.Stat()
	if err != nil {
		log.Warning("unable to get Stat()")
		log.Fatal(err)
	}
	size := fi.Size()
	nSamples := s.Header.Hns
	txtAndBinarySize := int64(SEGY_DESC_HDR_LEN + SEGY_BIN_HDR_LEN)
	nTraces := ((size - txtAndBinarySize) / (int64(SEGY_TRACE_HDR_LEN) + int64(nSamples)*int64(unsafe.Sizeof(float32(1)))))

	return nTraces
}

func (s *SegyFile) GetNrSamples() int32 {
	return int32(s.Header.Hns)
}

func (s *SegyFile) GetHeader() map[string]interface{} {
	m := make(map[string]interface{})
	v := reflect.ValueOf(s.Header)
	for i := 0; i < v.NumField(); i++ {
		key := v.Type().Field(i).Name
		val := v.Field(i).Interface()
		log.Debugf("name = %s, value = %d", key, val)
		m[key] = val
	}

	return m
}

func (s *SegyFile) ReadTrace() (Trace, error) {
	trace := Trace{}
	traceBuff := make([]float32, s.GetNrSamples())
	byteBuff := make([]byte, s.GetNrSamples()*4)
	trace.Data = traceBuff

	trcHdrBuff := make([]byte, SEGY_TRACE_HDR_LEN)
	bytesRead, err := s.file.Read(trcHdrBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	trcHdrReader := bytes.NewReader(trcHdrBuff)
	err = binary.Read(trcHdrReader, binary.BigEndian, &trace.TraceHeader)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	bytesRead, err = s.file.Read(byteBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	if bytesRead == 0 {
		log.Infof("No bytes read for trace #", s.Position)
	}

	for i := range trace.Data {
		trace.Data[i] = float32(binary.BigEndian.Uint32(byteBuff[i*4 : (i+1)*4]))
	}

	// Then figure out the size of the data, and read it
	return trace, nil
}

//func (s *SegyFile)
<commit_msg>import rearranged<commit_after>package segygo

import (
	"bytes"
	"encoding/binary"
	"io/ioutil"
	//"log"
	"github.com/op/go-logging"
	"os"
	"reflect"
	"unsafe"
)

var log = logging.MustGetLogger("segygo")
var format = logging.MustStringFormatter(
	`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
)

const Version = "0.1"
const SEGY_DESC_HDR_LEN = 3200
const SEGY_BIN_HDR_LEN = 400
const SEGY_TRACE_HDR_LEN = 240

type BinHeader struct {
	Jobid, Lino, Reno                                                                                int32
	Ntrpr, Nart                                                                                int16
	Hdt, Dto, Hns, Nso                                                                                uint16
	Format, Fold, Tsort, Vscode, Hsfs, Hsfe, Hslen, Hstyp, Schn, Hstas, Hstae, Htatyp, Hcorr, Bgrcv, Rcvm, Mfeet, Polyt, Vgpol int16
	Hunass                                                                                [170]int16 // unassigned
}

type TraceHeader struct {
	Tracel                                                               int32
	Tracer                                                               int32
	Fldr                                                                 int32
	Tracf                                                                int32
	Ep                                                                   int32
	CDP                                                                  int32
	CDPT                                                                 int32
	Trid                                                                 int16
	Nvs                                                                  int16
	Nhs                                                                  int16
	Duse                                                                 int16
	Offset                                                               int32
	Gelev                                                                int32
	Selev                                                                int32
	Sdepth                                                               int32
	Gdel                                                                 int32
	Sdel                                                                 int32
	SwDep                                                                int32
	GwDep                                                                int32
	Scalel                                                               int16
	Scalco                                                               int16
	Sx                                                                   int32
	Sy                                                                   int32
	Gx                                                                   int32
	Gy                                                                   int32
	CoUnit                                                               int16
	WeVel                                                                int16
	SweVel                                                               int16
	Sut, Gut, Sstat, Gstat, Tstat, Laga, Lagb, Delrt, Muts, Mute         int16
	Ns, Dt                                                               uint16
	Gain, Igc, Igi, Corr, Sfs, Sfe, Slen, Styp, Stas, Stae, Tatyp        int16
	Afilf, Afils, NoFilf, NoFils, Lcf, Hcf, Lcs, Hcs, Year, Day          int16
	Hour, Minute, Sec, Timbas, Trwf, Grnors, Grnofr, Grnlof, Gaps, Otrav int16
	D1, F1, D2, F2, Ungpow, Unscale                                      float32
	Ntr                                                                  int32
	Mark, Shortpad                                                       int16
	Unass                                                                [14]int16 // unassigned short array
}

type Trace struct {
	TraceHeader
	Data []float32
}

type SegyFile struct {
	Filename string
	Header   BinHeader
	NrTraces int64
	file     *os.File
	Position int64
	LogLevel logging.Level
}

func CreateFile(filename string) (SegyFile, error) {
	var s SegyFile
	var binHdr BinHeader
	f, err := os.Create(filename)
	defer f.Close()

	if err != nil {
		return s, err
	}

	s.LogLevel = logging.WARNING

	// Setup proper logging
	backend1 := logging.NewLogBackend(os.Stderr, "", 0)
	backend1Formatter := logging.NewBackendFormatter(backend1, format)
	logging.SetBackend(backend1Formatter)
	logging.SetLevel(s.LogLevel, "")

	log.Debugf("Creating SEG-Y file: %s", s.Filename)

	s.Filename = filename
	s.Header = binHdr
	s.NrTraces = 0
	s.file = f
	s.Position = 0

	accum := make([]byte, 3200)
	//r := bytes.NewWriter(accum)
	//binary.Write()
	buff := bytes.NewBuffer(accum)
	if err = binary.Write(buff, binary.BigEndian, &s.Header); err != nil {
		log.Errorf("Error creating buffer to hold binary header for segy file: %s. Msg: %s", s.Filename, err)
		return s, err
	}

	n, err := f.Write(buff.Bytes())
	if err != nil {
		log.Errorf("Error writing binary header to segy file: %s. Msg: %s", s.Filename, err)
		return s, err
	}
	log.Debugf("Wrote %d bytes to file: %s", n, s.Filename)

	return s, err

}

func OpenFile(filename string) (SegyFile, error) {
	var s SegyFile
	var binHdr BinHeader

	s.Filename = filename
	s.LogLevel = logging.WARNING

	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return s, err
	}

	// Setup proper logging
	backend1 := logging.NewLogBackend(os.Stderr, "", 0)
	backend1Formatter := logging.NewBackendFormatter(backend1, format)
	logging.SetBackend(backend1Formatter)
	logging.SetLevel(s.LogLevel, "")

	accum := []byte{}
	accum = append(accum, b...)

	accum2 := accum[3200:]
	r := bytes.NewReader(accum2)
	log.Debugf("Number of bytes: %d", r.Len())

	if err = binary.Read(r, binary.BigEndian, &binHdr); err != nil {
		log.Errorf("Error reading segy file (bigendian). %s", err)
		return s, err
	}

	// Open and store the os.File object in our struct
	file, err := os.Open(s.Filename)
	s.file = file
	defer file.Close()

	s.Header = binHdr
	s.NrTraces = s.GetNrTraces()

	return s, err
}

func (s *SegyFile) SetVerbose(verbose bool) {

	if verbose {
		s.LogLevel = logging.DEBUG
		logging.SetLevel(s.LogLevel, "")
	} else {
		s.LogLevel = logging.WARNING
		logging.SetLevel(s.LogLevel, "")
	}

}

func (s *SegyFile) GetNrTraces() int64 {
	fi, err := s.file.Stat()
	if err != nil {
		log.Warning("unable to get Stat()")
		log.Fatal(err)
	}
	size := fi.Size()
	nSamples := s.Header.Hns
	txtAndBinarySize := int64(SEGY_DESC_HDR_LEN + SEGY_BIN_HDR_LEN)
	nTraces := ((size - txtAndBinarySize) / (int64(SEGY_TRACE_HDR_LEN) + int64(nSamples)*int64(unsafe.Sizeof(float32(1)))))

	return nTraces
}

func (s *SegyFile) GetNrSamples() int32 {
	return int32(s.Header.Hns)
}

func (s *SegyFile) GetHeader() map[string]interface{} {
	m := make(map[string]interface{})
	v := reflect.ValueOf(s.Header)
	for i := 0; i < v.NumField(); i++ {
		key := v.Type().Field(i).Name
		val := v.Field(i).Interface()
		log.Debugf("name = %s, value = %d", key, val)
		m[key] = val
	}

	return m
}

func (s *SegyFile) ReadTrace() (Trace, error) {
	trace := Trace{}
	traceBuff := make([]float32, s.GetNrSamples())
	byteBuff := make([]byte, s.GetNrSamples()*4)
	trace.Data = traceBuff

	trcHdrBuff := make([]byte, SEGY_TRACE_HDR_LEN)
	bytesRead, err := s.file.Read(trcHdrBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	trcHdrReader := bytes.NewReader(trcHdrBuff)
	err = binary.Read(trcHdrReader, binary.BigEndian, &trace.TraceHeader)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	bytesRead, err = s.file.Read(byteBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	if bytesRead == 0 {
		log.Infof("No bytes read for trace #", s.Position)
	}

	for i := range trace.Data {
		trace.Data[i] = float32(binary.BigEndian.Uint32(byteBuff[i*4 : (i+1)*4]))
	}

	// Then figure out the size of the data, and read it
	return trace, nil
}

//func (s *SegyFile)
<|endoftext|>


<commit_before>package devices

// stuff in here is a hack to be able to switch between embd and some other library...

import (
	"errors"
	"fmt"
	"os"
	"time"

	"github.com/kidoman/embd"
)

type SPI interface {
	//fmt.Stringer
	//io.Writer
	Tx(w, r []byte) error
	Speed(hz int64) error
	Configure(mode int, bits int) error
	Close() error
}

const (
	SPIMode0 = 0x0 // CPOL=0, CPHA=0
	SPIMode1 = 0x1 // CPOL=0, CPHA=1
	SPIMode2 = 0x2 // CPOL=1, CPHA=0
	SPIMode3 = 0x3 // CPOL=1, CPHA=1
)

type GPIO interface {
	In(edge int) error
	Read() int
	WaitForEdge(timeout time.Duration) bool
	Out(level int)
	Number() int
	Close()
}

const (
	GpioLow        = 0
	GpioHigh       = 1
	GpioNoEdge     = 0
	GpioRisingEdge = 1
)

//===== SPI shim for embd

func NewSPI() SPI {
	return &spi{embd.NewSPIBus(embd.SPIMode0, 0, 4, 8, 0)}
}

type spi struct {
	embd.SPIBus
}

func (s *spi) Tx(w, r []byte) error {
	copy(r, w)
	return s.TransferAndReceiveData(r)
}

func (s *spi) Speed(hz int64) error {
	if hz != 4000000 {
		return errors.New("SPI: sorry, only 4Mhz supported")
	}
	return nil
}

func (s *spi) Configure(mode int, bits int) error {
	if mode != SPIMode0 {
		return errors.New("SPI: sorry, only SPI mode 0 supported")
	}
	if bits != 8 {
		return errors.New("SPI: sorry, only 8-bit mode supported")
	}
	return nil
}

//===== GPIO shim for embd

func NewGPIO(name string) GPIO {
	g, err := embd.NewDigitalPin(name)
	if err != nil {
		fmt.Fprintf(os.Stderr, "NewDigitalPin: %s\n", err)
		return nil
	}
	return &gpio{p: g, dir: embd.In, edge: make(chan struct{}, 1)}
}

type gpio struct {
	p    embd.DigitalPin
	dir  embd.Direction
	edge chan struct{}
}

func (g *gpio) In(edge int) error {
	if err := g.p.SetDirection(embd.In); err != nil {
		return err
	}
	g.dir = embd.In
	if edge != GpioNoEdge {
		e := []embd.Edge{embd.EdgeNone, embd.EdgeRising, embd.EdgeFalling, embd.EdgeBoth}[edge]
		//fmt.Fprintf(os.Stderr, "Watching pin %d\n", g.p.N())
		return g.p.Watch(e, g.edgeCB)
	}
	return nil
}

func (g *gpio) Read() int {
	v, _ := g.p.Read()
	return v
}

func (g *gpio) WaitForEdge(timeout time.Duration) bool {
	to := time.After(timeout)
	select {
	case <-g.edge:
		return true
	case <-to:
		return false
	}
}

func (g *gpio) Out(level int) {
	if g.dir != embd.Out {
		g.p.SetDirection(embd.Out)
		g.dir = embd.In
	}
	g.p.Write(level)
}

func (g *gpio) Number() int {
	return g.p.N()
}

func (g *gpio) Close() { g.p.Close() }

func (g *gpio) edgeCB(embd.DigitalPin) {
	//fmt.Fprintf(os.Stderr, "Intr %d\n", g.p.N())
	select {
	case g.edge <- struct{}{}:
	default:
	}
}
<commit_msg>convert to periph<commit_after><|endoftext|>


<commit_before>
// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spec

import (
	"encoding/json"
	"fmt"
	"net/url"
	"path/filepath"

	"github.com/go-swagger/go-swagger/swag"
)

//go:generate go-bindata -pkg=spec -prefix=./schemas -ignore=.*\.md ./schemas/...
//go:generate perl -pi -e s,Json,JSON,g bindata.go

const (
	// SwaggerSchemaURL the url for the swagger 2.0 schema to validate specs
	SwaggerSchemaURL = "http://swagger.io/v2/schema.json#"
	// JSONSchemaURL the url for the json schema schema
	JSONSchemaURL = "http://json-schema.org/draft-04/schema#"
)

var (
	jsonSchema    = MustLoadJSONSchemaDraft04()
	swaggerSchema = MustLoadSwagger20Schema()
)

// DocLoader represents a doc loader type
type DocLoader func(string) (json.RawMessage, error)

// JSONSpec loads a spec from a json document
func JSONSpec(path string) (*Document, error) {
	data, err := swag.JSONDoc(path)
	if err != nil {
		return nil, err
	}
	// convert to json
	return New(json.RawMessage(data), "")
}

// YAMLSpec loads a swagger spec document
func YAMLSpec(path string) (*Document, error) {
	data, err := swag.YAMLDoc(path)
	if err != nil {
		return nil, err
	}

	return New(data, "")
}

// MustLoadJSONSchemaDraft04 panics when Swagger20Schema returns an error
func MustLoadJSONSchemaDraft04() *Schema {
	d, e := JSONSchemaDraft04()
	if e != nil {
		panic(e)
	}
	return d
}

// JSONSchemaDraft04 loads the json schema document for json shema draft04
func JSONSchemaDraft04() (*Schema, error) {
	b, err := Asset("jsonschema-draft-04.json")
	if err != nil {
		return nil, err
	}

	schema := new(Schema)
	if err := json.Unmarshal(b, schema); err != nil {
		return nil, err
	}
	return schema, nil
}

// MustLoadSwagger20Schema panics when Swagger20Schema returns an error
func MustLoadSwagger20Schema() *Schema {
	d, e := Swagger20Schema()
	if e != nil {
		panic(e)
	}
	return d
}

// Swagger20Schema loads the swagger 2.0 schema from the embedded assets
func Swagger20Schema() (*Schema, error) {

	b, err := Asset("v2/schema.json")
	if err != nil {
		return nil, err
	}

	schema := new(Schema)
	if err := json.Unmarshal(b, schema); err != nil {
		return nil, err
	}
	return schema, nil
}

// Document represents a swagger spec document
type Document struct {
	specAnalyzer
	spec *Swagger
	raw  json.RawMessage
}

// Load loads a new spec document
func Load(path string) (*Document, error) {
	specURL, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	ext := filepath.Ext(specURL.Path)
	if ext == ".yaml" || ext == ".yml" {
		return YAMLSpec(path)
	}

	return JSONSpec(path)
}

// New creates a new shema document
func New(data json.RawMessage, version string) (*Document, error) {
	if version == "" {
		version = "2.0"
	}
	if version != "2.0" {
		return nil, fmt.Errorf("spec version %q is not supported", version)
	}

	spec := new(Swagger)
	if err := json.Unmarshal(data, spec); err != nil {
		return nil, err
	}

	d := &Document{
		specAnalyzer: specAnalyzer{
			spec:        spec,
			consumes:    make(map[string]struct{}),
			produces:    make(map[string]struct{}),
			authSchemes: make(map[string]struct{}),
			operations:  make(map[string]map[string]*Operation),
		},
		spec: spec,
		raw:  data,
	}
	d.initialize()
	return d, nil
}

// Expanded expands the ref fields in the spec document and returns a new spec document
func (d *Document) Expanded() (*Document, error) {
	spec := new(Swagger)
	if err := json.Unmarshal(d.raw, spec); err != nil {
		return nil, err
	}
	if err := expandSpec(spec); err != nil {
		return nil, err
	}

	dd := &Document{
		specAnalyzer: specAnalyzer{
			spec:        spec,
			consumes:    make(map[string]struct{}),
			produces:    make(map[string]struct{}),
			authSchemes: make(map[string]struct{}),
			operations:  make(map[string]map[string]*Operation),
		},
		spec: spec,
		raw:  d.raw,
	}
	dd.initialize()
	return dd, nil
}

// BasePath the base path for this spec
func (d *Document) BasePath() string {
	return d.spec.BasePath
}

// Version returns the version of this spec
func (d *Document) Version() string {
	return d.spec.Swagger
}

// Schema returns the swagger 2.0 schema
func (d *Document) Schema() *Schema {
	return swaggerSchema
}

// Spec returns the swagger spec object model
func (d *Document) Spec() *Swagger {
	return d.spec
}

// Host returns the host for the API
func (d *Document) Host() string {
	return d.spec.Host
}

// Raw returns the raw swagger spec as json bytes
func (d *Document) Raw() json.RawMessage {
	return d.raw
}

// Reload reanalyzes the spec
func (d *Document) Reload() *Document {
	d.specAnalyzer = specAnalyzer{
		spec:        d.spec,
		consumes:    make(map[string]struct{}),
		produces:    make(map[string]struct{}),
		authSchemes: make(map[string]struct{}),
		operations:  make(map[string]map[string]*Operation),
	}
	d.initialize()
	return d
}
<commit_msg>ensure basepath starts with '/', don't render unset body<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spec

import (
	"encoding/json"
	"fmt"
	"net/url"
	"path/filepath"

	"github.com/go-swagger/go-swagger/swag"
)

//go:generate go-bindata -pkg=spec -prefix=./schemas -ignore=.*\.md ./schemas/...
//go:generate perl -pi -e s,Json,JSON,g bindata.go

const (
	// SwaggerSchemaURL the url for the swagger 2.0 schema to validate specs
	SwaggerSchemaURL = "http://swagger.io/v2/schema.json#"
	// JSONSchemaURL the url for the json schema schema
	JSONSchemaURL = "http://json-schema.org/draft-04/schema#"
)

var (
	jsonSchema    = MustLoadJSONSchemaDraft04()
	swaggerSchema = MustLoadSwagger20Schema()
)

// DocLoader represents a doc loader type
type DocLoader func(string) (json.RawMessage, error)

// JSONSpec loads a spec from a json document
func JSONSpec(path string) (*Document, error) {
	data, err := swag.JSONDoc(path)
	if err != nil {
		return nil, err
	}
	// convert to json
	return New(json.RawMessage(data), "")
}

// YAMLSpec loads a swagger spec document
func YAMLSpec(path string) (*Document, error) {
	data, err := swag.YAMLDoc(path)
	if err != nil {
		return nil, err
	}

	return New(data, "")
}

// MustLoadJSONSchemaDraft04 panics when Swagger20Schema returns an error
func MustLoadJSONSchemaDraft04() *Schema {
	d, e := JSONSchemaDraft04()
	if e != nil {
		panic(e)
	}
	return d
}

// JSONSchemaDraft04 loads the json schema document for json shema draft04
func JSONSchemaDraft04() (*Schema, error) {
	b, err := Asset("jsonschema-draft-04.json")
	if err != nil {
		return nil, err
	}

	schema := new(Schema)
	if err := json.Unmarshal(b, schema); err != nil {
		return nil, err
	}
	return schema, nil
}

// MustLoadSwagger20Schema panics when Swagger20Schema returns an error
func MustLoadSwagger20Schema() *Schema {
	d, e := Swagger20Schema()
	if e != nil {
		panic(e)
	}
	return d
}

// Swagger20Schema loads the swagger 2.0 schema from the embedded assets
func Swagger20Schema() (*Schema, error) {

	b, err := Asset("v2/schema.json")
	if err != nil {
		return nil, err
	}

	schema := new(Schema)
	if err := json.Unmarshal(b, schema); err != nil {
		return nil, err
	}
	return schema, nil
}

// Document represents a swagger spec document
type Document struct {
	specAnalyzer
	spec *Swagger
	raw  json.RawMessage
}

// Load loads a new spec document
func Load(path string) (*Document, error) {
	specURL, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	ext := filepath.Ext(specURL.Path)
	if ext == ".yaml" || ext == ".yml" {
		return YAMLSpec(path)
	}

	return JSONSpec(path)
}

// New creates a new shema document
func New(data json.RawMessage, version string) (*Document, error) {
	if version == "" {
		version = "2.0"
	}
	if version != "2.0" {
		return nil, fmt.Errorf("spec version %q is not supported", version)
	}

	spec := new(Swagger)
	if err := json.Unmarshal(data, spec); err != nil {
		return nil, err
	}

	d := &Document{
		specAnalyzer: specAnalyzer{
			spec:        spec,
			consumes:    make(map[string]struct{}),
			produces:    make(map[string]struct{}),
			authSchemes: make(map[string]struct{}),
			operations:  make(map[string]map[string]*Operation),
		},
		spec: spec,
		raw:  data,
	}
	d.initialize()
	return d, nil
}

// Expanded expands the ref fields in the spec document and returns a new spec document
func (d *Document) Expanded() (*Document, error) {
	spec := new(Swagger)
	if err := json.Unmarshal(d.raw, spec); err != nil {
		return nil, err
	}
	if err := expandSpec(spec); err != nil {
		return nil, err
	}

	dd := &Document{
		specAnalyzer: specAnalyzer{
			spec:        spec,
			consumes:    make(map[string]struct{}),
			produces:    make(map[string]struct{}),
			authSchemes: make(map[string]struct{}),
			operations:  make(map[string]map[string]*Operation),
		},
		spec: spec,
		raw:  d.raw,
	}
	dd.initialize()
	return dd, nil
}

// BasePath the base path for this spec
func (d *Document) BasePath() string {
	return d.spec.BasePath
}

// Version returns the version of this spec
func (d *Document) Version() string {
	return d.spec.Swagger
}

// Schema returns the swagger 2.0 schema
func (d *Document) Schema() *Schema {
	return swaggerSchema
}

// Spec returns the swagger spec object model
func (d *Document) Spec() *Swagger {
	return d.spec
}

// Host returns the host for the API
func (d *Document) Host() string {
	return d.spec.Host
}

// Raw returns the raw swagger spec as json bytes
func (d *Document) Raw() json.RawMessage {
	return d.raw
}

// Reload reanalyzes the spec
func (d *Document) Reload() *Document {
	d.specAnalyzer = specAnalyzer{
		spec:        d.spec,
		consumes:    make(map[string]struct{}),
		produces:    make(map[string]struct{}),
		authSchemes: make(map[string]struct{}),
		operations:  make(map[string]map[string]*Operation),
	}
	d.initialize()
	return d
}
<|endoftext|>


<commit_before>package sqlz

/************
统计CREATE，SELECT，INSERT，DELETE，UPDATE 等的数量，并且统计操作的表的数量。
************/
import (
	///"github.com/golang/glog"
	"sync"
	"time"

	"github.com/toontong/sqlz/sqlparser"
)

var (
	z_open       bool
	z_running    chan bool
	z_result     *StatusResult
	z_stackQuery chan string
)

type SQL_Type string

const (
	ERROR_SQL SQL_Type = "ERROR_SQL"
	UNKNOW    SQL_Type = "UNKNOW_SQL"
	SELECT    SQL_Type = "SELECT"
	INSERT    SQL_Type = "INSERT"
	UPDATE    SQL_Type = "UPDATE"
	DELETE    SQL_Type = "DELETE"
	SHOW      SQL_Type = "SHOW"
	CREATE    SQL_Type = "CREATE"
	RENAME    SQL_Type = "RENAME"
	ALTER     SQL_Type = "ALTER"
	DROP      SQL_Type = "DROP"
)

type Count struct {
	// Table      string
	// Count      int64
	TableCount map[string]int64 //key is table name.
}

func NewCount() *Count {
	c := &Count{
		TableCount: make(map[string]int64),
	}
	return c
}

func (count *Count) add(tableName string) int64 {
	if cnt, ok := count.TableCount[tableName]; ok {
		count.TableCount[tableName]++
		return cnt + 1
	} else {
		count.TableCount[tableName] = 1
		return 1
	}
}

type StatusResult struct {
	Opration map[SQL_Type]*Count
	Error    int
	Success  int
	Waiting  int
	Start    time.Time
	End      time.Time
	lock     *sync.RWMutex
}

func newStatusResult() *StatusResult {
	status := &StatusResult{
		Opration: make(map[SQL_Type]*Count),
		Start:    time.Now(),
		Error:    0,
		Success:  0,
		Waiting:  0,
		lock:     new(sync.RWMutex),
	}
	return status
}

func (res *StatusResult) addOpration(typ SQL_Type, tableName string) {
	z_result.lock.Lock()
	defer z_result.lock.Unlock()

	if count, ok := z_result.Opration[typ]; ok {
		count.add(tableName)
	} else {
		count = NewCount()
		count.add(tableName)
		z_result.Opration[typ] = count
	}
}

// -----------------------------
// 开启统计功能,可随时开启
func StartZ() {
	if z_open {
		return
	}
	z_open = true

	cleanStatus()
	z_stackQuery = make(chan string, 4096)
	go statistics()
}

func Z(query string) bool {
	if !z_open {
		return false
	}
	select {
	case z_stackQuery <- query:
		return true
	default:
		// glog.Error("query stack was full or empty.")
		return false
	}
}

func StopZ() {
	if !z_open {
		return
	}
	z_open = false

	// wait statistics thread exit
	<-z_running

	z_stackQuery = nil
}

func Status() StatusResult {
	if z_result == nil {
		cleanStatus()
	}
	z_result.lock.Lock()
	defer z_result.lock.Unlock()
	z_result.Waiting = len(z_stackQuery)
	z_result.End = time.Now()
	return *z_result
}

// -----------------------------
func cleanStatus() {
	z_result = newStatusResult()
}

func init() {
	cleanStatus()
	z_running = make(chan bool, 1)
}

func statistics() {
	var query string
	for z_open {

		select {
		case query = <-z_stackQuery:
		default:
			continue
		}

		stmt, err := sqlparser.Parse(query)
		if err != nil {
			z_result.Error++
			continue
		}
		if !z_open {
			break
		}
		z_result.analyze(stmt)
		z_result.Success++
	}
	z_running <- false
}

func (res *StatusResult) analyze(stmt sqlparser.Statement) {

	switch sql := stmt.(type) {
	case *sqlparser.Select:
		for _, tableExpr := range sql.From {
			node, ok := tableExpr.(*sqlparser.AliasedTableExpr)
			if !ok {
				res.addOpration(SELECT, "")
			} else {
				res.addOpration(SELECT, sqlparser.GetTableName(node.Expr))
			}
		}
	case *sqlparser.Insert:
		res.addOpration(INSERT, sqlparser.GetTableName(sql.Table))
	case *sqlparser.Update:
		res.addOpration(UPDATE, sqlparser.GetTableName(sql.Table))
	case *sqlparser.Delete:
		res.addOpration(DELETE, sqlparser.GetTableName(sql.Table))
	case *sqlparser.Show:
		res.addOpration(SHOW, sql.Section)
	case *sqlparser.DDL:
		var typ SQL_Type
		var tableName []byte
		switch sql.Action {
		case sqlparser.AST_CREATE:
			typ = CREATE
			tableName = sql.NewName
		case sqlparser.AST_RENAME:
			typ = RENAME
			tableName = sql.Table
		case sqlparser.AST_DROP:
			typ = DROP
			tableName = sql.Table
		case sqlparser.AST_ALTER:
			typ = ALTER
			tableName = sql.Table
		default:
			typ = UNKNOW
		}
		res.addOpration(typ, string(tableName))
	case nil:
		res.addOpration(ERROR_SQL, "nil")
	default:
		res.addOpration(UNKNOW, "nil")
	}
}
<commit_msg>导出 ParseQuery<commit_after>package sqlz

/************
统计CREATE，SELECT，INSERT，DELETE，UPDATE 等的数量，并且统计操作的表的数量。
************/
import (
	///"github.com/golang/glog"
	"sync"
	"time"

	"github.com/toontong/sqlz/sqlparser"
)

var (
	z_open       bool
	z_running    chan bool
	z_result     *StatusResult
	z_stackQuery chan string
)

type SQL_Type string

const (
	ERROR_SQL SQL_Type = "ERROR_SQL"
	UNKNOW    SQL_Type = "UNKNOW_SQL"
	SELECT    SQL_Type = "SELECT"
	INSERT    SQL_Type = "INSERT"
	UPDATE    SQL_Type = "UPDATE"
	DELETE    SQL_Type = "DELETE"
	SHOW      SQL_Type = "SHOW"
	CREATE    SQL_Type = "CREATE"
	RENAME    SQL_Type = "RENAME"
	ALTER     SQL_Type = "ALTER"
	DROP      SQL_Type = "DROP"
)

type Count struct {
	// Table      string
	// Count      int64
	TableCount map[string]int64 //key is table name.
}

func NewCount() *Count {
	c := &Count{
		TableCount: make(map[string]int64),
	}
	return c
}

func (count *Count) add(tableName string) int64 {
	if cnt, ok := count.TableCount[tableName]; ok {
		count.TableCount[tableName]++
		return cnt + 1
	} else {
		count.TableCount[tableName] = 1
		return 1
	}
}

type StatusResult struct {
	Opration map[SQL_Type]*Count
	Error    int
	Success  int
	Waiting  int
	Start    time.Time
	End      time.Time
	lock     *sync.RWMutex
}

func newStatusResult() *StatusResult {
	status := &StatusResult{
		Opration: make(map[SQL_Type]*Count),
		Start:    time.Now(),
		Error:    0,
		Success:  0,
		Waiting:  0,
		lock:     new(sync.RWMutex),
	}
	return status
}

func (res *StatusResult) addOpration(typ SQL_Type, tableName string) {
	z_result.lock.Lock()
	defer z_result.lock.Unlock()

	if count, ok := z_result.Opration[typ]; ok {
		count.add(tableName)
	} else {
		count = NewCount()
		count.add(tableName)
		z_result.Opration[typ] = count
	}
}

// -----------------------------
// 开启统计功能,可随时开启
func StartZ() {
	if z_open {
		return
	}
	z_open = true

	cleanStatus()
	z_stackQuery = make(chan string, 4096)
	go statistics()
}

func Z(query string) bool {
	if !z_open {
		return false
	}
	select {
	case z_stackQuery <- query:
		return true
	default:
		// glog.Error("query stack was full or empty.")
		return false
	}
}

func StopZ() {
	if !z_open {
		return
	}
	z_open = false

	// wait statistics thread exit
	<-z_running

	z_stackQuery = nil
}

func Status() StatusResult {
	if z_result == nil {
		cleanStatus()
	}
	z_result.lock.Lock()
	defer z_result.lock.Unlock()
	z_result.Waiting = len(z_stackQuery)
	z_result.End = time.Now()
	return *z_result
}

// -----------------------------
func cleanStatus() {
	z_result = newStatusResult()
}

func init() {
	cleanStatus()
	z_running = make(chan bool, 1)
}

func statistics() {
	var query string
	for z_open {

		select {
		case query = <-z_stackQuery:
		default:
			continue
		}

		typ, table := ParseQuery(query)
		if !z_open {
			break
		}

		if typ == ERROR_SQL {
			z_result.Error++
			continue
		}
		z_result.addOpration(typ, table)
		z_result.Success++
	}
	z_running <- false
}

func ParseQuery(query string) (action SQL_Type, table string) {
	// stmt sqlparser.Statement
	stmt, err := sqlparser.Parse(query)
	if err != nil {
		return ERROR_SQL, string(UNKNOW)
	}

	switch sql := stmt.(type) {
	case *sqlparser.Select:
		action = SELECT
		for _, tableExpr := range sql.From {
			node, ok := tableExpr.(*sqlparser.AliasedTableExpr)
			if !ok {
				table = string(UNKNOW)
			} else {
				table = sqlparser.GetTableName(node.Expr)
			}
		}
	case *sqlparser.Insert:
		action, table = INSERT, sqlparser.GetTableName(sql.Table)
	case *sqlparser.Update:
		action, table = UPDATE, sqlparser.GetTableName(sql.Table)
	case *sqlparser.Delete:
		action, table = DELETE, sqlparser.GetTableName(sql.Table)
	case *sqlparser.Show:
		action, table = SHOW, sql.Section
	case *sqlparser.DDL:

		var tableName []byte
		switch sql.Action {
		case sqlparser.AST_CREATE:
			action = CREATE
			tableName = sql.NewName
		case sqlparser.AST_RENAME:
			action = RENAME
			tableName = sql.Table
		case sqlparser.AST_DROP:
			action = DROP
			tableName = sql.Table
		case sqlparser.AST_ALTER:
			action = ALTER
			tableName = sql.Table
		default:
			action = UNKNOW
		}
		table = string(tableName)
	case nil:
		action, table = ERROR_SQL, "nil"
	default:
		action, table = UNKNOW, "nil"
	}
	return action, table
}
<|endoftext|>


<commit_before>package stom

import (
	"database/sql/driver"
	"errors"
	"fmt"
	"reflect"
)

type Policy uint8

const (
	PolicyUseDefault Policy = iota
	PolicyExclude

	defaultTag = "db"
)

type ToMappable interface {
	ToMap() (map[string]interface{}, error)
}

var defaultStom = stom{defaultTag, PolicyUseDefault, nil}

type stom struct {
	tag          string
	policy       Policy
	defaultValue interface{}
}

func (this *stom) SetTag(tag string) {
	this.tag = tag
}

func (this *stom) SetDefault(defaultValue interface{}) {
	this.defaultValue = defaultValue
}

func (this *stom) SetPolicy(policy Policy) {
	this.policy = policy
}

func (this *stom) ToMap(s interface{}) (map[string]interface{}, error) {
	typ := reflect.TypeOf(s)
	// if a pointer to a struct is passed, get the type of the dereferenced object
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}

	if typ.Kind() != reflect.Struct {
		return nil, errors.New(fmt.Sprintf("expected struct, got %v", typ.Kind()))
	}

	result := map[string]interface{}{}

	val := reflect.ValueOf(s)
	// loop through the struct's fields and set the map
	for i := 0; i < typ.NumField(); i++ {
		field := typ.Field(i)
		if field.PkgPath != "" { // unexported
			continue
		}

		// TODO: try indices instead of values
		// TODO: parallelize it
		// TODO: cache should work as so:
		// 1. Create a map 'field name' => index
		// 2. Get a value by index
		// 3. Convert value to appropriate value
		// 4. DONE

		if t := field.Tag.Get(this.tag); t != "" && t != "-" {
			vField := val.Field(i)
			var v interface{}
			if vField.Kind() == reflect.Ptr {
				if vField.Elem().IsValid() {
					v = vField.Elem().Interface()
				}
			} else {
				v = vField.Interface()
			}

			v = convertValue(v)

			if v != nil {
				result[t] = v
			} else if this.policy == PolicyUseDefault {
				result[t] = this.defaultValue
			}
		}

	}

	return result, nil

	// TODO:
	// 1. Check if it's a struct. If not - return error
	// 2. Scan through struct's fields and get all the tags
	// 3. If tag is in this.tags - use tag name as a key and field value as value
	// and put them into map
}

func SetDefault(defaultValue interface{})                 { defaultStom.SetDefault(defaultValue) }
func SetTag(tag string)                                   { defaultStom.SetTag(tag) }
func SetPolicy(policy Policy)                             { defaultStom.SetPolicy(policy) }
func ToMap(s interface{}) (map[string]interface{}, error) { return defaultStom.ToMap(s) }

func convertValue(input interface{}) (output interface{}) {
	// TODO: check if input is a structure
	output = input
	switch t := input.(type) {
	case driver.Valuer:
		if converted, err := t.Value(); converted == nil || err != nil {
			output = nil
		}
	}

	return

}
<commit_msg>- implemented by-struct stom objects. They speed parsing up - global ToMap is still working<commit_after>package stom

import (
	"database/sql/driver"
	"errors"
	"fmt"
	"reflect"
)

type Policy uint8

const (
	PolicyUseDefault Policy = iota
	PolicyExclude
)

// package settings
var (
	tag          string      = "db"
	policy       Policy      = PolicyUseDefault
	defaultValue interface{} = nil
)

type ToMappable interface {
	ToMap() (map[string]interface{}, error)
}

type ToMapper interface {
	ToMap(s interface{}) (map[string]interface{}, error)
}

type stom struct {
	defaultValue interface{}
	policy       Policy
	tag          string

	typ   reflect.Type
	cache map[string]int
}

func MustNewStom(s interface{}) *stom {
	typ, err := getStructType(s)
	if err != nil {
		panic(err.Error())
	}

	stom := &stom{
		typ:          typ,
		defaultValue: defaultValue,
		policy:       policy,
	}
	return stom.SetTag(tag)
}

func (this *stom) SetTag(tag string) *stom {
	this.tag = tag
	this.cache = extractTagValues(this.typ, this.tag)

	return this
}
func (this *stom) SetDefault(defaultValue interface{}) { this.defaultValue = defaultValue }
func (this *stom) SetPolicy(policy Policy)             { this.policy = policy }

func (this *stom) ToMap(s interface{}) (map[string]interface{}, error) {
	val := reflect.ValueOf(s)
	if val.Kind() == reflect.Ptr {
		s = val.Elem().Interface()
	}

	typ := reflect.TypeOf(s)
	if typ != this.typ {
		return nil, errors.New(fmt.Sprintf("stom is set up to work with type %s, but %s given", this.typ, typ))
	}

	return toMap(s, this.cache)
}

func SetTag(t string)           { tag = t }
func SetDefault(dv interface{}) { defaultValue = dv }
func SetPolicy(p Policy)        { policy = p }

func ToMap(s interface{}) (map[string]interface{}, error) {
	if tomappable, ok := s.(ToMappable); ok {
		return tomappable.ToMap()
	}

	typ := reflect.TypeOf(s)

	if typ.Kind() != reflect.Struct {
		return nil, errors.New(fmt.Sprintf("expected struct, got %v", typ.Kind()))
	}

	tagmap := extractTagValues(typ, tag)

	return toMap(s, tagmap)
}

func getStructType(s interface{}) (t reflect.Type, err error) {
	t = reflect.TypeOf(s)

	if t.Kind() == reflect.Invalid {
		err = errors.New(fmt.Sprintf("value is invalid:\n %v", s))
		return
	}

	if t.Kind() != reflect.Struct {
		err = errors.New(fmt.Sprintf("provided value is not a struct!\n%v", s))
	}

	return
}

func extractTagValues(typ reflect.Type, tag string) map[string]int {
	tagValues := map[string]int{}

	for i := 0; i < typ.NumField(); i++ {
		field := typ.Field(i)
		if field.PkgPath != "" { // unexported
			continue
		}

		if tagValue := field.Tag.Get(tag); tagValue != "" && tagValue != "-" {
			tagValues[tagValue] = i
		}
	}

	return tagValues
}

func toMap(s interface{}, tagmap map[string]int) (map[string]interface{}, error) {
	val := reflect.ValueOf(s)

	result := map[string]interface{}{}

	for tag, index := range tagmap {
		vField := val.Field(index)

		v, err := convertValue(vField)
		if err != nil {
			return result, err
		}

		if v != nil {
			result[tag] = v
		} else if policy == PolicyUseDefault {
			result[tag] = defaultValue
		}

	}

	return result, nil
}

func convertValue(vField reflect.Value) (v interface{}, err error) {
	kind := vField.Kind()
	if kind == reflect.Ptr {
		if vField.Elem().IsValid() {
			v = vField.Elem().Interface()
		}
	} else {
		v = vField.Interface()
	}

	switch t := v.(type) {
	case driver.Valuer: // support for NullTypes like sql.NullString and so on
		if converted, convErr := t.Value(); convErr != nil || converted == nil {
			v = nil
		}
		return

	case ToMappable:
		v, err = t.ToMap()
		return
	}

	return v, nil

}
<|endoftext|>


<commit_before>package turn

import (
  "bytes"
  "encoding/binary"
  "errors"
)

const (
  magicCookie uint32 = 0x2112A442
)

type StunClass uint16
const (
  StunRequest StunClass = iota
  StunIndication
  StunResponse
  StunError
)

type StunType uint16
const (
  StunBinding StunType = 1 + iota
)

type StunHeader struct {
  Class   StunClass
  Type    StunType
  Length  uint16
  Id      []byte
}

func (h *StunHeader) Encode() ([]byte, error) {
  var classEnc uint16 = 0
  buf := new(bytes.Buffer)

  hType := uint16(h.Type)
  hClass := uint16(h.Class)

  //bits 0-3 are low bits of type
  classEnc |= hType & 15
  //bit 4 is low bit of class
  classEnc |= (hClass & 1) << 4
  //bits 5-7 are bits 4-6 of type
  classEnc |= ((hType >> 4) & 7) << 5
  //bit 8 is high bit of class
  classEnc |= (hClass & 2) << 7
  //bits 9-13 are high bits of type
  classEnc |= ((hType >> 7) & 31) << 9

  err := binary.Write(buf, binary.BigEndian, classEnc)
  err = binary.Write(buf, binary.BigEndian, h.Length)
  err = binary.Write(buf, binary.BigEndian, magicCookie)
  err = binary.Write(buf, binary.BigEndian, h.Id)

  if len(h.Id) != 12 {
    return nil, errors.New("Unsupported Transaction ID Length")
  }

  if err != nil {
    return nil, err
  }
  return buf.Bytes(), nil
}

func (h *StunHeader) Decode(data []byte) (error) {
  if len(data) < 20 {
    return errors.New("Header Length Too Short")
  }

  classEnc := binary.BigEndian.Uint16(data)
  stunClass := StunClass(((classEnc & 4) >> 3) + ((classEnc & 8) >> 6))
  stunType := StunType(classEnc & 15 + ((classEnc >> 5) & 7) << 4 + ((classEnc >> 9) & 31) << 7)

  if classEnc >> 14 != 0 {
    return errors.New("First 2 bits are not 0")
  }

  if binary.BigEndian.Uint32(data[4:]) != magicCookie {
    return errors.New("Bad Magic Cookie")
  }

  if binary.BigEndian.Uint16(data[2:]) & 3 != 0 {
    return errors.New("Message Length is not a multiple of 4")
  }

  h.Type = stunType
  h.Class = stunClass
  h.Length = binary.BigEndian.Uint16(data[2:])
  h.Id = data[8:20]

  return nil
}
<commit_msg>add types for stun attributes<commit_after>package turn

import (
  "bytes"
  "encoding/binary"
  "errors"
)

const (
  magicCookie uint32 = 0x2112A442
)

type StunClass uint16
const (
  StunRequest StunClass = iota
  StunIndication
  StunResponse
  StunError
)

type StunType uint16
const (
  StunBinding StunType = 1 + iota
)

type StunHeader struct {
  Class   StunClass
  Type    StunType
  Length  uint16
  Id      []byte
}

type StunAttributeType uint16
const (
  MappedAddress StunAttributeType = 0x1
  Username = 0x6
  MessageIntegrity = 0x8
  ErrorCode = 0x9
  UnknownAttributes = 0xA
  Realm = 0x14
  Nonce = 0x15
  XorMappedAddress = 0x20

  // comprehension-optional attributes
  Software = 0x8022
  AlternateServer = 0x8023
  Fingerprint = 0x8028
)

type StunAttribute struct {
  Type    StunAttributeType
  Length  uint16
  Value   []byte
}

func (h *StunHeader) Encode() ([]byte, error) {
  var classEnc uint16 = 0
  buf := new(bytes.Buffer)

  hType := uint16(h.Type)
  hClass := uint16(h.Class)

  //bits 0-3 are low bits of type
  classEnc |= hType & 15
  //bit 4 is low bit of class
  classEnc |= (hClass & 1) << 4
  //bits 5-7 are bits 4-6 of type
  classEnc |= ((hType >> 4) & 7) << 5
  //bit 8 is high bit of class
  classEnc |= (hClass & 2) << 7
  //bits 9-13 are high bits of type
  classEnc |= ((hType >> 7) & 31) << 9

  err := binary.Write(buf, binary.BigEndian, classEnc)
  err = binary.Write(buf, binary.BigEndian, h.Length)
  err = binary.Write(buf, binary.BigEndian, magicCookie)
  err = binary.Write(buf, binary.BigEndian, h.Id)

  if len(h.Id) != 12 {
    return nil, errors.New("Unsupported Transaction ID Length")
  }

  if err != nil {
    return nil, err
  }
  return buf.Bytes(), nil
}

func (h *StunHeader) Decode(data []byte) (error) {
  if len(data) < 20 {
    return errors.New("Header Length Too Short")
  }

  classEnc := binary.BigEndian.Uint16(data)
  stunClass := StunClass(((classEnc & 4) >> 3) + ((classEnc & 8) >> 6))
  stunType := StunType(classEnc & 15 + ((classEnc >> 5) & 7) << 4 + ((classEnc >> 9) & 31) << 7)

  if classEnc >> 14 != 0 {
    return errors.New("First 2 bits are not 0")
  }

  if binary.BigEndian.Uint32(data[4:]) != magicCookie {
    return errors.New("Bad Magic Cookie")
  }

  if binary.BigEndian.Uint16(data[2:]) & 3 != 0 {
    return errors.New("Message Length is not a multiple of 4")
  }

  h.Type = stunType
  h.Class = stunClass
  h.Length = binary.BigEndian.Uint16(data[2:])
  h.Id = data[8:20]

  return nil
}
<|endoftext|>