<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

//-------------------------------------------------------------------------
// []package_import
//-------------------------------------------------------------------------

type package_import struct {
	alias string
	path  string
}

// Parses import declarations until the first non-import declaration and fills
// `packages` array with import information.
func collect_package_imports(filename string, decls []ast.Decl, context build.Context) []package_import {
	pi := make([]package_import, 0, 16)
	for _, decl := range decls {
		if gd, ok := decl.(*ast.GenDecl); ok && gd.Tok == token.IMPORT {
			for _, spec := range gd.Specs {
				imp := spec.(*ast.ImportSpec)
				path, alias := path_and_alias(imp)
				path, ok := abs_path_for_package(filename, path, context)
				if ok && alias != "_" {
					pi = append(pi, package_import{alias, path})
				}
			}
		} else {
			break
		}
	}
	return pi
}

//-------------------------------------------------------------------------
// decl_file_cache
//
// Contains cache for top-level declarations of a file as well as its
// contents, AST and import information.
//-------------------------------------------------------------------------

type decl_file_cache struct {
	name  string // file name
	mtime int64  // last modification time

	decls     map[string]*decl // top-level declarations
	error     error            // last error
	packages  []package_import // import information
	filescope *scope

	fset    *token.FileSet
	context build.Context
}

func new_decl_file_cache(name string, context build.Context) *decl_file_cache {
	return &decl_file_cache{
		name:    name,
		context: context,
	}
}

func (f *decl_file_cache) update() {
	stat, err := os.Stat(f.name)
	if err != nil {
		f.decls = nil
		f.error = err
		f.fset = nil
		return
	}

	statmtime := stat.ModTime().UnixNano()
	if f.mtime == statmtime {
		return
	}

	f.mtime = statmtime
	f.read_file()
}

func (f *decl_file_cache) read_file() {
	var data []byte
	data, f.error = file_reader.read_file(f.name)
	if f.error != nil {
		return
	}
	data, _ = filter_out_shebang(data)

	f.process_data(data)
}

func (f *decl_file_cache) process_data(data []byte) {
	var file *ast.File
	f.fset = token.NewFileSet()
	file, f.error = parser.ParseFile(f.fset, "", data, 0)
	f.filescope = new_scope(nil)
	for _, d := range file.Decls {
		anonymify_ast(d, 0, f.filescope)
	}
	f.packages = collect_package_imports(f.name, file.Decls, f.context)
	f.decls = make(map[string]*decl, len(file.Decls))
	for _, decl := range file.Decls {
		append_to_top_decls(f.decls, decl, f.filescope)
	}
}

func append_to_top_decls(decls map[string]*decl, decl ast.Decl, scope *scope) {
	foreach_decl(decl, func(data *foreach_decl_struct) {
		class := ast_decl_class(data.decl)
		for i, name := range data.names {
			typ, v, vi := data.type_value_index(i)

			d := new_decl_full(name.Name, class, 0, typ, v, vi, scope)
			if d == nil {
				return
			}

			methodof := method_of(decl)
			if methodof != "" {
				decl, ok := decls[methodof]
				if ok {
					decl.add_child(d)
				} else {
					decl = new_decl(methodof, decl_methods_stub, scope)
					decls[methodof] = decl
					decl.add_child(d)
				}
			} else {
				decl, ok := decls[d.name]
				if ok {
					decl.expand_or_replace(d)
				} else {
					decls[d.name] = d
				}
			}
		}
	})
}

func abs_path_for_package(filename, p string, context build.Context) (string, bool) {
	dir, _ := filepath.Split(filename)
	if len(p) == 0 {
		return "", false
	}
	if p[0] == '.' {
		return fmt.Sprintf("%s.a", filepath.Join(dir, p)), true
	}
	pkg, ok := find_go_dag_package(p, dir)
	if ok {
		return pkg, true
	}
	return find_global_file(p, context)
}

func path_and_alias(imp *ast.ImportSpec) (string, string) {
	path := ""
	if imp.Path != nil {
		path = string(imp.Path.Value)
		path = path[1 : len(path)-1]
	}
	alias := ""
	if imp.Name != nil {
		alias = imp.Name.Name
	}
	return path, alias
}

func find_go_dag_package(imp, filedir string) (string, bool) {
	// Support godag directory structure
	dir, pkg := filepath.Split(imp)
	godag_pkg := filepath.Join(filedir, "..", dir, "_obj", pkg+".a")
	if file_exists(godag_pkg) {
		return godag_pkg, true
	}
	return "", false
}

// autobuild compares the mod time of the source files of the package, and if any of them is newer
// than the package object file will rebuild it.
func autobuild(p *build.Package) error {
	if p.Dir == "" {
		return fmt.Errorf("no files to build")
	}
	ps, err := os.Stat(p.PkgObj)
	if err != nil {
		// Assume package file does not exist and build for the first time.
		return build_package(p)
	}
	pt := ps.ModTime()
	fs, err := ioutil.ReadDir(p.Dir)
	for _, f := range fs {
		if f.IsDir() {
			break
		}
		if f.ModTime().After(pt) {
			// Source file is newer than package file; rebuild.
			return build_package(p)
		}
	}
	return nil
}

// build_package builds the package by calling `go install package/import`. If everything compiles
// correctly, the newly compiled package should then be in the usual place in the `$GOPATH/pkg`
// directory, and gocode will pick it up from there.
func build_package(p *build.Package) error {
	log.Printf("-------------------")
	log.Printf("rebuilding package %s", p.Name)
	log.Printf("package import: %s", p.ImportPath)
	log.Printf("package object: %s", p.PkgObj)
	log.Printf("package source dir: %s", p.Dir)
	log.Printf("package source files: %v", p.GoFiles)
	// TODO: Should read STDERR rather than STDOUT.
	out, err := exec.Command("go", "install", p.ImportPath).Output()
	if err != nil {
		return err
	}
	log.Printf("build out: %s\n", string(out))
	return nil
}

func log_found_package_maybe(imp, pkgpath string) {
	if *g_debug {
		log.Printf("Found \"%s\" at \"%s\"\n", imp, pkgpath)
	}
}

func log_build_context(context build.Context) {
	log.Printf(" GOROOT: %s\n", context.GOROOT)
	log.Printf(" GOPATH: %s\n", context.GOPATH)
	log.Printf(" GOOS: %s\n", context.GOOS)
	log.Printf(" GOARCH: %s\n", context.GOARCH)
	log.Printf(" lib-path: \"%s\"\n", g_config.LibPath)
}

// find_global_file returns the file path of the compiled package corresponding to the specified
// import, and a boolean stating whether such path is valid.
// TODO: Return only one value, possibly empty string if not found.
func find_global_file(imp string, context build.Context) (string, bool) {
	// gocode synthetically generates the builtin package
	// "unsafe", since the "unsafe.a" package doesn't really exist.
	// Thus, when the user request for the package "unsafe" we
	// would return synthetic global file that would be used
	// just as a key name to find this synthetic package
	if imp == "unsafe" {
		return "unsafe", true
	}

	p, err := context.Import(imp, "", build.AllowBinary)
	if err == nil {
		if g_config.Autobuild {
			autobuild(p)
		}
		if file_exists(p.PkgObj) {
			log_found_package_maybe(imp, p.PkgObj)
			return p.PkgObj, true
		}
	}

	pkgfile := fmt.Sprintf("%s.a", imp)

	// if lib-path is defined, use it
	if g_config.LibPath != "" {
		for _, p := range filepath.SplitList(g_config.LibPath) {
			pkg_path := filepath.Join(p, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
			// Also check the relevant pkg/OS_ARCH dir for the libpath, if provided.
			pkgdir := fmt.Sprintf("%s_%s", context.GOOS, context.GOARCH)
			pkg_path = filepath.Join(p, "pkg", pkgdir, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
		}
	}

	if *g_debug {
		log.Printf("Import path \"%s\" was not resolved\n", imp)
		log.Println("Gocode's build context is:")
		log_build_context(context)
	}
	return "", false
}

func package_name(file *ast.File) string {
	if file.Name != nil {
		return file.Name.Name
	}
	return ""
}

//-------------------------------------------------------------------------
// decl_cache
//
// Thread-safe collection of DeclFileCache entities.
//-------------------------------------------------------------------------

type decl_cache struct {
	cache   map[string]*decl_file_cache
	context build.Context
	sync.Mutex
}

func new_decl_cache(context build.Context) *decl_cache {
	return &decl_cache{
		cache:   make(map[string]*decl_file_cache),
		context: context,
	}
}

func (c *decl_cache) get(filename string) *decl_file_cache {
	c.Lock()
	defer c.Unlock()

	f, ok := c.cache[filename]
	if !ok {
		f = new_decl_file_cache(filename, c.context)
		c.cache[filename] = f
	}
	return f
}

func (c *decl_cache) get_and_update(filename string) *decl_file_cache {
	f := c.get(filename)
	f.update()
	return f
}
<commit_msg>Replace \"%s\" with %q. Handly Go's printf feature.<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

//-------------------------------------------------------------------------
// []package_import
//-------------------------------------------------------------------------

type package_import struct {
	alias string
	path  string
}

// Parses import declarations until the first non-import declaration and fills
// `packages` array with import information.
func collect_package_imports(filename string, decls []ast.Decl, context build.Context) []package_import {
	pi := make([]package_import, 0, 16)
	for _, decl := range decls {
		if gd, ok := decl.(*ast.GenDecl); ok && gd.Tok == token.IMPORT {
			for _, spec := range gd.Specs {
				imp := spec.(*ast.ImportSpec)
				path, alias := path_and_alias(imp)
				path, ok := abs_path_for_package(filename, path, context)
				if ok && alias != "_" {
					pi = append(pi, package_import{alias, path})
				}
			}
		} else {
			break
		}
	}
	return pi
}

//-------------------------------------------------------------------------
// decl_file_cache
//
// Contains cache for top-level declarations of a file as well as its
// contents, AST and import information.
//-------------------------------------------------------------------------

type decl_file_cache struct {
	name  string // file name
	mtime int64  // last modification time

	decls     map[string]*decl // top-level declarations
	error     error            // last error
	packages  []package_import // import information
	filescope *scope

	fset    *token.FileSet
	context build.Context
}

func new_decl_file_cache(name string, context build.Context) *decl_file_cache {
	return &decl_file_cache{
		name:    name,
		context: context,
	}
}

func (f *decl_file_cache) update() {
	stat, err := os.Stat(f.name)
	if err != nil {
		f.decls = nil
		f.error = err
		f.fset = nil
		return
	}

	statmtime := stat.ModTime().UnixNano()
	if f.mtime == statmtime {
		return
	}

	f.mtime = statmtime
	f.read_file()
}

func (f *decl_file_cache) read_file() {
	var data []byte
	data, f.error = file_reader.read_file(f.name)
	if f.error != nil {
		return
	}
	data, _ = filter_out_shebang(data)

	f.process_data(data)
}

func (f *decl_file_cache) process_data(data []byte) {
	var file *ast.File
	f.fset = token.NewFileSet()
	file, f.error = parser.ParseFile(f.fset, "", data, 0)
	f.filescope = new_scope(nil)
	for _, d := range file.Decls {
		anonymify_ast(d, 0, f.filescope)
	}
	f.packages = collect_package_imports(f.name, file.Decls, f.context)
	f.decls = make(map[string]*decl, len(file.Decls))
	for _, decl := range file.Decls {
		append_to_top_decls(f.decls, decl, f.filescope)
	}
}

func append_to_top_decls(decls map[string]*decl, decl ast.Decl, scope *scope) {
	foreach_decl(decl, func(data *foreach_decl_struct) {
		class := ast_decl_class(data.decl)
		for i, name := range data.names {
			typ, v, vi := data.type_value_index(i)

			d := new_decl_full(name.Name, class, 0, typ, v, vi, scope)
			if d == nil {
				return
			}

			methodof := method_of(decl)
			if methodof != "" {
				decl, ok := decls[methodof]
				if ok {
					decl.add_child(d)
				} else {
					decl = new_decl(methodof, decl_methods_stub, scope)
					decls[methodof] = decl
					decl.add_child(d)
				}
			} else {
				decl, ok := decls[d.name]
				if ok {
					decl.expand_or_replace(d)
				} else {
					decls[d.name] = d
				}
			}
		}
	})
}

func abs_path_for_package(filename, p string, context build.Context) (string, bool) {
	dir, _ := filepath.Split(filename)
	if len(p) == 0 {
		return "", false
	}
	if p[0] == '.' {
		return fmt.Sprintf("%s.a", filepath.Join(dir, p)), true
	}
	pkg, ok := find_go_dag_package(p, dir)
	if ok {
		return pkg, true
	}
	return find_global_file(p, context)
}

func path_and_alias(imp *ast.ImportSpec) (string, string) {
	path := ""
	if imp.Path != nil {
		path = string(imp.Path.Value)
		path = path[1 : len(path)-1]
	}
	alias := ""
	if imp.Name != nil {
		alias = imp.Name.Name
	}
	return path, alias
}

func find_go_dag_package(imp, filedir string) (string, bool) {
	// Support godag directory structure
	dir, pkg := filepath.Split(imp)
	godag_pkg := filepath.Join(filedir, "..", dir, "_obj", pkg+".a")
	if file_exists(godag_pkg) {
		return godag_pkg, true
	}
	return "", false
}

// autobuild compares the mod time of the source files of the package, and if any of them is newer
// than the package object file will rebuild it.
func autobuild(p *build.Package) error {
	if p.Dir == "" {
		return fmt.Errorf("no files to build")
	}
	ps, err := os.Stat(p.PkgObj)
	if err != nil {
		// Assume package file does not exist and build for the first time.
		return build_package(p)
	}
	pt := ps.ModTime()
	fs, err := ioutil.ReadDir(p.Dir)
	for _, f := range fs {
		if f.IsDir() {
			break
		}
		if f.ModTime().After(pt) {
			// Source file is newer than package file; rebuild.
			return build_package(p)
		}
	}
	return nil
}

// build_package builds the package by calling `go install package/import`. If everything compiles
// correctly, the newly compiled package should then be in the usual place in the `$GOPATH/pkg`
// directory, and gocode will pick it up from there.
func build_package(p *build.Package) error {
	log.Printf("-------------------")
	log.Printf("rebuilding package %s", p.Name)
	log.Printf("package import: %s", p.ImportPath)
	log.Printf("package object: %s", p.PkgObj)
	log.Printf("package source dir: %s", p.Dir)
	log.Printf("package source files: %v", p.GoFiles)
	// TODO: Should read STDERR rather than STDOUT.
	out, err := exec.Command("go", "install", p.ImportPath).Output()
	if err != nil {
		return err
	}
	log.Printf("build out: %s\n", string(out))
	return nil
}

func log_found_package_maybe(imp, pkgpath string) {
	if *g_debug {
		log.Printf("Found %q at %q\n", imp, pkgpath)
	}
}

func log_build_context(context build.Context) {
	log.Printf(" GOROOT: %s\n", context.GOROOT)
	log.Printf(" GOPATH: %s\n", context.GOPATH)
	log.Printf(" GOOS: %s\n", context.GOOS)
	log.Printf(" GOARCH: %s\n", context.GOARCH)
	log.Printf(" lib-path: %q\n", g_config.LibPath)
}

// find_global_file returns the file path of the compiled package corresponding to the specified
// import, and a boolean stating whether such path is valid.
// TODO: Return only one value, possibly empty string if not found.
func find_global_file(imp string, context build.Context) (string, bool) {
	// gocode synthetically generates the builtin package
	// "unsafe", since the "unsafe.a" package doesn't really exist.
	// Thus, when the user request for the package "unsafe" we
	// would return synthetic global file that would be used
	// just as a key name to find this synthetic package
	if imp == "unsafe" {
		return "unsafe", true
	}

	p, err := context.Import(imp, "", build.AllowBinary)
	if err == nil {
		if g_config.Autobuild {
			autobuild(p)
		}
		if file_exists(p.PkgObj) {
			log_found_package_maybe(imp, p.PkgObj)
			return p.PkgObj, true
		}
	}

	pkgfile := fmt.Sprintf("%s.a", imp)

	// if lib-path is defined, use it
	if g_config.LibPath != "" {
		for _, p := range filepath.SplitList(g_config.LibPath) {
			pkg_path := filepath.Join(p, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
			// Also check the relevant pkg/OS_ARCH dir for the libpath, if provided.
			pkgdir := fmt.Sprintf("%s_%s", context.GOOS, context.GOARCH)
			pkg_path = filepath.Join(p, "pkg", pkgdir, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
		}
	}

	if *g_debug {
		log.Printf("Import path %q was not resolved\n", imp)
		log.Println("Gocode's build context is:")
		log_build_context(context)
	}
	return "", false
}

func package_name(file *ast.File) string {
	if file.Name != nil {
		return file.Name.Name
	}
	return ""
}

//-------------------------------------------------------------------------
// decl_cache
//
// Thread-safe collection of DeclFileCache entities.
//-------------------------------------------------------------------------

type decl_cache struct {
	cache   map[string]*decl_file_cache
	context build.Context
	sync.Mutex
}

func new_decl_cache(context build.Context) *decl_cache {
	return &decl_cache{
		cache:   make(map[string]*decl_file_cache),
		context: context,
	}
}

func (c *decl_cache) get(filename string) *decl_file_cache {
	c.Lock()
	defer c.Unlock()

	f, ok := c.cache[filename]
	if !ok {
		f = new_decl_file_cache(filename, c.context)
		c.cache[filename] = f
	}
	return f
}

func (c *decl_cache) get_and_update(filename string) *decl_file_cache {
	f := c.get(filename)
	f.update()
	return f
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

//-------------------------------------------------------------------------
// []package_import
//-------------------------------------------------------------------------

type package_import struct {
	alias string
	path  string
}

// Parses import declarations until the first non-import declaration and fills
// `packages` array with import information.
func collect_package_imports(filename string, decls []ast.Decl, context build.Context) []package_import {
	pi := make([]package_import, 0, 16)
	for _, decl := range decls {
		if gd, ok := decl.(*ast.GenDecl); ok && gd.Tok == token.IMPORT {
			for _, spec := range gd.Specs {
				imp := spec.(*ast.ImportSpec)
				path, alias := path_and_alias(imp)
				path, ok := abs_path_for_package(filename, path, context)
				if ok && alias != "_" {
					pi = append(pi, package_import{alias, path})
				}
			}
		} else {
			break
		}
	}
	return pi
}

//-------------------------------------------------------------------------
// decl_file_cache
//
// Contains cache for top-level declarations of a file as well as its
// contents, AST and import information.
//-------------------------------------------------------------------------

type decl_file_cache struct {
	name  string // file name
	mtime int64  // last modification time

	decls     map[string]*decl // top-level declarations
	error     error            // last error
	packages  []package_import // import information
	filescope *scope

	fset    *token.FileSet
	context build.Context
}

func new_decl_file_cache(name string, context build.Context) *decl_file_cache {
	return &decl_file_cache{
		name:    name,
		context: context,
	}
}

func (f *decl_file_cache) update() {
	stat, err := os.Stat(f.name)
	if err != nil {
		f.decls = nil
		f.error = err
		f.fset = nil
		return
	}

	statmtime := stat.ModTime().UnixNano()
	if f.mtime == statmtime {
		return
	}

	f.mtime = statmtime
	f.read_file()
}

func (f *decl_file_cache) read_file() {
	var data []byte
	data, f.error = file_reader.read_file(f.name)
	if f.error != nil {
		return
	}
	data, _ = filter_out_shebang(data)

	f.process_data(data)
}

func (f *decl_file_cache) process_data(data []byte) {
	var file *ast.File
	f.fset = token.NewFileSet()
	file, f.error = parser.ParseFile(f.fset, "", data, 0)
	f.filescope = new_scope(nil)
	for _, d := range file.Decls {
		anonymify_ast(d, 0, f.filescope)
	}
	f.packages = collect_package_imports(f.name, file.Decls, f.context)
	f.decls = make(map[string]*decl, len(file.Decls))
	for _, decl := range file.Decls {
		append_to_top_decls(f.decls, decl, f.filescope)
	}
}

func append_to_top_decls(decls map[string]*decl, decl ast.Decl, scope *scope) {
	foreach_decl(decl, func(data *foreach_decl_struct) {
		class := ast_decl_class(data.decl)
		for i, name := range data.names {
			typ, v, vi := data.type_value_index(i)

			d := new_decl_full(name.Name, class, 0, typ, v, vi, scope)
			if d == nil {
				return
			}

			methodof := method_of(decl)
			if methodof != "" {
				decl, ok := decls[methodof]
				if ok {
					decl.add_child(d)
				} else {
					decl = new_decl(methodof, decl_methods_stub, scope)
					decls[methodof] = decl
					decl.add_child(d)
				}
			} else {
				decl, ok := decls[d.name]
				if ok {
					decl.expand_or_replace(d)
				} else {
					decls[d.name] = d
				}
			}
		}
	})
}

func abs_path_for_package(filename, p string, context build.Context) (string, bool) {
	dir, _ := filepath.Split(filename)
	if len(p) == 0 {
		return "", false
	}
	if p[0] == '.' {
		return fmt.Sprintf("%s.a", filepath.Join(dir, p)), true
	}
	pkg, ok := find_go_dag_package(p, dir)
	if ok {
		return pkg, true
	}
	return find_global_file(p, context)
}

func path_and_alias(imp *ast.ImportSpec) (string, string) {
	path := ""
	if imp.Path != nil {
		path = string(imp.Path.Value)
		path = path[1 : len(path)-1]
	}
	alias := ""
	if imp.Name != nil {
		alias = imp.Name.Name
	}
	return path, alias
}

func find_go_dag_package(imp, filedir string) (string, bool) {
	// Support godag directory structure
	dir, pkg := filepath.Split(imp)
	godag_pkg := filepath.Join(filedir, "..", dir, "_obj", pkg+".a")
	if file_exists(godag_pkg) {
		return godag_pkg, true
	}
	return "", false
}

// autobuild compares the mod time of the source files of the package, and if any of them is newer
// than the package object file will rebuild it.
func autobuild(p *build.Package) error {
	if p.Dir == "" {
		return fmt.Errorf("no files to build")
	}
	ps, err := os.Stat(p.PkgObj)
	if err != nil {
		// Assume package file does not exist and build for the first time.
		return build_package(p)
	}
	pt := ps.ModTime()
	fs, err := ioutil.ReadDir(p.Dir)
	for _, f := range fs {
		if f.IsDir() {
			break
		}
		if f.ModTime().After(pt) {
			// Source file is newer than package file; rebuild.
			return build_package(p)
		}
	}
	return nil
}

// build_package builds the package by calling `go install package/import`. If everything compiles
// correctly, the newly compiled package should then be in the usual place in the `$GOPATH/pkg`
// directory, and gocode will pick it up from there.
func build_package(p *build.Package) error {
	log.Printf("-------------------")
	log.Printf("rebuilding package %s", p.Name)
	log.Printf("package import: %s", p.ImportPath)
	log.Printf("package object: %s", p.PkgObj)
	log.Printf("package source dir: %s", p.Dir)
	log.Printf("package source files: %v", p.GoFiles)
	// TODO: Should read STDERR rather than STDOUT.
	out, err := exec.Command("go", "install", p.ImportPath).Output()
	if err != nil {
		return err
	}
	log.Printf("build out: %s\n", string(out))
	return nil
}

func log_found_package_maybe(imp, pkgpath string) {
	if *g_debug {
		log.Printf("Found %q at %q\n", imp, pkgpath)
	}
}

func log_build_context(context build.Context) {
	log.Printf(" GOROOT: %s\n", context.GOROOT)
	log.Printf(" GOPATH: %s\n", context.GOPATH)
	log.Printf(" GOOS: %s\n", context.GOOS)
	log.Printf(" GOARCH: %s\n", context.GOARCH)
	log.Printf(" lib-path: %q\n", g_config.LibPath)
}

// find_global_file returns the file path of the compiled package corresponding to the specified
// import, and a boolean stating whether such path is valid.
// TODO: Return only one value, possibly empty string if not found.
func find_global_file(imp string, context build.Context) (string, bool) {
	// gocode synthetically generates the builtin package
	// "unsafe", since the "unsafe.a" package doesn't really exist.
	// Thus, when the user request for the package "unsafe" we
	// would return synthetic global file that would be used
	// just as a key name to find this synthetic package
	if imp == "unsafe" {
		return "unsafe", true
	}

	p, err := context.Import(imp, "", build.AllowBinary)
	if err == nil {
		if g_config.Autobuild {
			autobuild(p)
		}
		if file_exists(p.PkgObj) {
			log_found_package_maybe(imp, p.PkgObj)
			return p.PkgObj, true
		}
	}

	pkgfile := fmt.Sprintf("%s.a", imp)

	// if lib-path is defined, use it
	if g_config.LibPath != "" {
		for _, p := range filepath.SplitList(g_config.LibPath) {
			pkg_path := filepath.Join(p, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
			// Also check the relevant pkg/OS_ARCH dir for the libpath, if provided.
			pkgdir := fmt.Sprintf("%s_%s", context.GOOS, context.GOARCH)
			pkg_path = filepath.Join(p, "pkg", pkgdir, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
		}
	}

	if *g_debug {
		log.Printf("Import path %q was not resolved\n", imp)
		log.Println("Gocode's build context is:")
		log_build_context(context)
	}
	return "", false
}

func package_name(file *ast.File) string {
	if file.Name != nil {
		return file.Name.Name
	}
	return ""
}

//-------------------------------------------------------------------------
// decl_cache
//
// Thread-safe collection of DeclFileCache entities.
//-------------------------------------------------------------------------

type decl_cache struct {
	cache   map[string]*decl_file_cache
	context build.Context
	sync.Mutex
}

func new_decl_cache(context build.Context) *decl_cache {
	return &decl_cache{
		cache:   make(map[string]*decl_file_cache),
		context: context,
	}
}

func (c *decl_cache) get(filename string) *decl_file_cache {
	c.Lock()
	defer c.Unlock()

	f, ok := c.cache[filename]
	if !ok {
		f = new_decl_file_cache(filename, c.context)
		c.cache[filename] = f
	}
	return f
}

func (c *decl_cache) get_and_update(filename string) *decl_file_cache {
	f := c.get(filename)
	f.update()
	return f
}
<commit_msg>Minor fixes to autobuild function.<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

//-------------------------------------------------------------------------
// []package_import
//-------------------------------------------------------------------------

type package_import struct {
	alias string
	path  string
}

// Parses import declarations until the first non-import declaration and fills
// `packages` array with import information.
func collect_package_imports(filename string, decls []ast.Decl, context build.Context) []package_import {
	pi := make([]package_import, 0, 16)
	for _, decl := range decls {
		if gd, ok := decl.(*ast.GenDecl); ok && gd.Tok == token.IMPORT {
			for _, spec := range gd.Specs {
				imp := spec.(*ast.ImportSpec)
				path, alias := path_and_alias(imp)
				path, ok := abs_path_for_package(filename, path, context)
				if ok && alias != "_" {
					pi = append(pi, package_import{alias, path})
				}
			}
		} else {
			break
		}
	}
	return pi
}

//-------------------------------------------------------------------------
// decl_file_cache
//
// Contains cache for top-level declarations of a file as well as its
// contents, AST and import information.
//-------------------------------------------------------------------------

type decl_file_cache struct {
	name  string // file name
	mtime int64  // last modification time

	decls     map[string]*decl // top-level declarations
	error     error            // last error
	packages  []package_import // import information
	filescope *scope

	fset    *token.FileSet
	context build.Context
}

func new_decl_file_cache(name string, context build.Context) *decl_file_cache {
	return &decl_file_cache{
		name:    name,
		context: context,
	}
}

func (f *decl_file_cache) update() {
	stat, err := os.Stat(f.name)
	if err != nil {
		f.decls = nil
		f.error = err
		f.fset = nil
		return
	}

	statmtime := stat.ModTime().UnixNano()
	if f.mtime == statmtime {
		return
	}

	f.mtime = statmtime
	f.read_file()
}

func (f *decl_file_cache) read_file() {
	var data []byte
	data, f.error = file_reader.read_file(f.name)
	if f.error != nil {
		return
	}
	data, _ = filter_out_shebang(data)

	f.process_data(data)
}

func (f *decl_file_cache) process_data(data []byte) {
	var file *ast.File
	f.fset = token.NewFileSet()
	file, f.error = parser.ParseFile(f.fset, "", data, 0)
	f.filescope = new_scope(nil)
	for _, d := range file.Decls {
		anonymify_ast(d, 0, f.filescope)
	}
	f.packages = collect_package_imports(f.name, file.Decls, f.context)
	f.decls = make(map[string]*decl, len(file.Decls))
	for _, decl := range file.Decls {
		append_to_top_decls(f.decls, decl, f.filescope)
	}
}

func append_to_top_decls(decls map[string]*decl, decl ast.Decl, scope *scope) {
	foreach_decl(decl, func(data *foreach_decl_struct) {
		class := ast_decl_class(data.decl)
		for i, name := range data.names {
			typ, v, vi := data.type_value_index(i)

			d := new_decl_full(name.Name, class, 0, typ, v, vi, scope)
			if d == nil {
				return
			}

			methodof := method_of(decl)
			if methodof != "" {
				decl, ok := decls[methodof]
				if ok {
					decl.add_child(d)
				} else {
					decl = new_decl(methodof, decl_methods_stub, scope)
					decls[methodof] = decl
					decl.add_child(d)
				}
			} else {
				decl, ok := decls[d.name]
				if ok {
					decl.expand_or_replace(d)
				} else {
					decls[d.name] = d
				}
			}
		}
	})
}

func abs_path_for_package(filename, p string, context build.Context) (string, bool) {
	dir, _ := filepath.Split(filename)
	if len(p) == 0 {
		return "", false
	}
	if p[0] == '.' {
		return fmt.Sprintf("%s.a", filepath.Join(dir, p)), true
	}
	pkg, ok := find_go_dag_package(p, dir)
	if ok {
		return pkg, true
	}
	return find_global_file(p, context)
}

func path_and_alias(imp *ast.ImportSpec) (string, string) {
	path := ""
	if imp.Path != nil {
		path = string(imp.Path.Value)
		path = path[1 : len(path)-1]
	}
	alias := ""
	if imp.Name != nil {
		alias = imp.Name.Name
	}
	return path, alias
}

func find_go_dag_package(imp, filedir string) (string, bool) {
	// Support godag directory structure
	dir, pkg := filepath.Split(imp)
	godag_pkg := filepath.Join(filedir, "..", dir, "_obj", pkg+".a")
	if file_exists(godag_pkg) {
		return godag_pkg, true
	}
	return "", false
}

// autobuild compares the mod time of the source files of the package, and if any of them is newer
// than the package object file will rebuild it.
func autobuild(p *build.Package) error {
	if p.Dir == "" {
		return fmt.Errorf("no files to build")
	}
	ps, err := os.Stat(p.PkgObj)
	if err != nil {
		// Assume package file does not exist and build for the first time.
		return build_package(p)
	}
	pt := ps.ModTime()
	fs, err := ioutil.ReadDir(p.Dir)
	if err != nil {
		return err
	}
	for _, f := range fs {
		if f.IsDir() {
			continue
		}
		if f.ModTime().After(pt) {
			// Source file is newer than package file; rebuild.
			return build_package(p)
		}
	}
	return nil
}

// build_package builds the package by calling `go install package/import`. If everything compiles
// correctly, the newly compiled package should then be in the usual place in the `$GOPATH/pkg`
// directory, and gocode will pick it up from there.
func build_package(p *build.Package) error {
	log.Printf("-------------------")
	log.Printf("rebuilding package %s", p.Name)
	log.Printf("package import: %s", p.ImportPath)
	log.Printf("package object: %s", p.PkgObj)
	log.Printf("package source dir: %s", p.Dir)
	log.Printf("package source files: %v", p.GoFiles)
	// TODO: Should read STDERR rather than STDOUT.
	out, err := exec.Command("go", "install", p.ImportPath).Output()
	if err != nil {
		return err
	}
	log.Printf("build out: %s\n", string(out))
	return nil
}

func log_found_package_maybe(imp, pkgpath string) {
	if *g_debug {
		log.Printf("Found %q at %q\n", imp, pkgpath)
	}
}

func log_build_context(context build.Context) {
	log.Printf(" GOROOT: %s\n", context.GOROOT)
	log.Printf(" GOPATH: %s\n", context.GOPATH)
	log.Printf(" GOOS: %s\n", context.GOOS)
	log.Printf(" GOARCH: %s\n", context.GOARCH)
	log.Printf(" lib-path: %q\n", g_config.LibPath)
}

// find_global_file returns the file path of the compiled package corresponding to the specified
// import, and a boolean stating whether such path is valid.
// TODO: Return only one value, possibly empty string if not found.
func find_global_file(imp string, context build.Context) (string, bool) {
	// gocode synthetically generates the builtin package
	// "unsafe", since the "unsafe.a" package doesn't really exist.
	// Thus, when the user request for the package "unsafe" we
	// would return synthetic global file that would be used
	// just as a key name to find this synthetic package
	if imp == "unsafe" {
		return "unsafe", true
	}

	p, err := context.Import(imp, "", build.AllowBinary)
	if err == nil {
		if g_config.Autobuild {
			autobuild(p)
		}
		if file_exists(p.PkgObj) {
			log_found_package_maybe(imp, p.PkgObj)
			return p.PkgObj, true
		}
	}

	pkgfile := fmt.Sprintf("%s.a", imp)

	// if lib-path is defined, use it
	if g_config.LibPath != "" {
		for _, p := range filepath.SplitList(g_config.LibPath) {
			pkg_path := filepath.Join(p, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
			// Also check the relevant pkg/OS_ARCH dir for the libpath, if provided.
			pkgdir := fmt.Sprintf("%s_%s", context.GOOS, context.GOARCH)
			pkg_path = filepath.Join(p, "pkg", pkgdir, pkgfile)
			if file_exists(pkg_path) {
				log_found_package_maybe(imp, pkg_path)
				return pkg_path, true
			}
		}
	}

	if *g_debug {
		log.Printf("Import path %q was not resolved\n", imp)
		log.Println("Gocode's build context is:")
		log_build_context(context)
	}
	return "", false
}

func package_name(file *ast.File) string {
	if file.Name != nil {
		return file.Name.Name
	}
	return ""
}

//-------------------------------------------------------------------------
// decl_cache
//
// Thread-safe collection of DeclFileCache entities.
//-------------------------------------------------------------------------

type decl_cache struct {
	cache   map[string]*decl_file_cache
	context build.Context
	sync.Mutex
}

func new_decl_cache(context build.Context) *decl_cache {
	return &decl_cache{
		cache:   make(map[string]*decl_file_cache),
		context: context,
	}
}

func (c *decl_cache) get(filename string) *decl_file_cache {
	c.Lock()
	defer c.Unlock()

	f, ok := c.cache[filename]
	if !ok {
		f = new_decl_file_cache(filename, c.context)
		c.cache[filename] = f
	}
	return f
}

func (c *decl_cache) get_and_update(filename string) *decl_file_cache {
	f := c.get(filename)
	f.update()
	return f
}
<|endoftext|>


<commit_before>package disk

import (
	"io"
	"log"
	"os"
)

// WriteAt writes len(p) bytes from p to the file at path at offset off.
// It returns the number of bytes written from p (0 <= n <= len(p)) and
// any error encountered that caused the write to stop early.
// WriteAt must return a non-nil error if it returns n < len(p).
// TODO(xiang90): []byte or io.reader?
func WriteAt(path string, p []byte, off int64) (int, error) {
	// block size
	bsize := blockSize(path)
	// payload size
	psize := bsize - crc32Len

	f, err := os.OpenFile(path, os.O_WRONLY, 0600)
	if err != nil {
		return 0, err
	}
	flen := getFileLength(f)

	// offset index that starts to write in the next time
	i := min(off, flen)
	// number of bytes that has written
	n := 0
	// index that ends writing
	end := off + int64(len(p))
	rbuf := make([]byte, psize)
	for i < end {
		pidx := i / psize
		poff := i - pidx*psize
		wbuf := make([]byte, 0)

		// read out the head of data in block
		if poff > 0 {
			rn, err := readBlock(f, rbuf, pidx, bsize)
			if err != nil && err != io.EOF {
				return n, err
			}
			if poff < rn {
				log.Panicf("unexpected insufficient read")
			}
			wbuf = append(wbuf, rbuf[:poff]...)
		}

		left := end - i
		pleft := psize - poff
		wn := min(left, pleft)
		var wrtdata []byte
		if i+wn < off {
			wrtdata = make([]byte, wn)
		} else if i > off {
			wrtdata = p[n : n+int(wn)]
		} else {
			wrtdata = append(make([]byte, off-i), p[:i+wn-off]...)
		}
		wbuf = append(wbuf, wrtdata...)

		// read out the tail of data in block
		if poff+wn < psize && end < flen {
			rn, err := readBlock(f, rbuf, pidx, bsize)
			if err != nil && err != io.EOF {
				return n, err
			}
			if poff+wn < rn {
				log.Panicf("unexpected insufficient read")
			}
			wbuf = append(wbuf, p[poff+wn:]...)
		}

		err := writeBlock(f, pidx, bsize, wbuf)
		if err != nil {
			return n, err
		}
		n += int(wn)
		i += int64(len(wbuf))
	}
	return n, nil
}

// blockSize returns the block size of the file at given path.
func blockSize(path string) int64 {
	// TODO (xiang90): implement it
	return 4096
}

func getFileLength(f *os.File) int64 {
	fi, err := f.Stat()
	if err != nil {
		return 0
	}
	return fi.Size()
}
<commit_msg>improve based on comments<commit_after>package disk

import (
	"io"
	"os"
)

// WriteAt writes len(p) bytes to the File starting at byte offset off.
// It returns the number of bytes written and an error, if any. WriteAt
// returns a non-nil error when n != len(p).
func WriteAt(path string, p []byte, off int64) (int, error) {
	// block size
	bsize := blockSize(path)
	// payload size
	psize := bsize - crc32Len

	f, err := os.OpenFile(path, os.O_WRONLY, 0600)
	if err != nil {
		return 0, err
	}

	flen := getFileLength(f)
	// start index
	st := min(off, flen)
	// data to write that starts from the offset
	data := &buffer{
		leadingZero: off - st,
		p:           p,
	}
	// index that ends writing
	end := st + data.size()
	// has trailing data in file after the end position
	hasTrailing := end < flen

	stPidx, stPoff := offToPayloadPos(st, psize)
	endPidx, endPoff := offToPayloadPos(end, psize)
	// read buffer
	rbuf := make([]byte, psize)
	// fast path for writing at one block
	if stPidx == endPidx {
		_, err := readBlock(f, rbuf, stPidx, bsize)
		if err != nil && err != io.EOF {
			return 0, err
		}
		var wbuf []byte
		if hasTrailing {
			copy(rbuf[stPoff:endPoff], data.slice(data.size()))
			wbuf = rbuf
		} else {
			wbuf = append(rbuf[:stPoff], data.slice(data.size())...)
		}
		err = writeBlock(f, stPidx, bsize, wbuf)
		if err != nil {
			return 0, err
		}
		return len(p), nil
	}

	// number of bytes that has written
	n := 0
	// head block
	if stPoff > 0 {
		_, err := readBlock(f, rbuf, stPidx, bsize)
		if err != nil && err != io.EOF {
			return n, err
		}
		wbuf := append(rbuf[:stPoff], data.slice(psize-stPoff)...)
		err = writeBlock(f, stPidx, bsize, wbuf)
		if err != nil {
			return n, err
		}
		stPidx++
		n += int(psize - stPoff)
	}
	// middle blocks
	for i := stPidx; i < endPidx; i++ {
		err := writeBlock(f, stPidx, bsize, data.slice(psize))
		if err != nil {
			return n, err
		}
		n += int(psize)
	}
	// tail block
	if endPoff > 0 {
		var wbuf []byte
		if hasTrailing {
			rn, err := readBlock(f, rbuf, endPidx, bsize)
			if err != nil && err != io.EOF {
				return n, err
			}
			copy(rbuf[:endPoff], data.slice(endPoff))
			wbuf = rbuf[:rn]
		} else {
			wbuf = data.slice(endPoff)
		}
		err := writeBlock(f, stPidx, bsize, wbuf)
		if err != nil {
			return n, err
		}
		n += int(endPoff)
	}
	return n, nil
}

// buffer represents a byte slice. It starts with leading zeros, and follows
// with the given data.
type buffer struct {
	leadingZero int64
	p           []byte

	off int64
}

func (b *buffer) size() int64 { return b.leadingZero + int64(len(b.p)) }

// slice consumes n bytes, moves the cursor, and returns it.
func (b *buffer) slice(n int64) []byte {
	lo, hi := b.off, b.off+n
	b.off += n
	if hi <= b.leadingZero {
		return make([]byte, n)
	} else if lo >= b.leadingZero {
		return b.p[lo-b.leadingZero : hi-b.leadingZero]
	} else {
		return append(make([]byte, b.leadingZero-lo), b.p[:hi-b.leadingZero]...)
	}
}

func offToPayloadPos(off, psize int64) (bidx, boff int64) {
	bidx = off / psize
	boff = off - bidx*psize
	return
}

// blockSize returns the block size of the file at given path.
func blockSize(path string) int64 {
	// TODO (xiang90): implement it
	return 4096
}

func getFileLength(f *os.File) int64 {
	fi, err := f.Stat()
	if err != nil {
		return 0
	}
	return fi.Size()
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

// Read system DNS config from /etc/resolv.conf

package dnsconfig

import (
	"net"

	"github.com/guilhem/netparse"
)

type DnsConfig struct {
	servers  []string // servers to use
	search   []string // suffixes to append to local name
	ndots    int      // number of dots in name to trigger absolute lookup
	timeout  int      // seconds before giving up on packet
	attempts int      // lost packets before giving up on server
	rotate   bool     // round robin among servers
}

// See resolv.conf(5) on a Linux machine.
// TODO(rsc): Supposed to call uname() and chop the beginning
// of the host name to get the default search domain.
func DnsReadConfig(filename string) (*DnsConfig, error) {
	file, err := netparse.Open(filename)
	if err != nil {
		return nil, &net.DNSConfigError{err}
	}
	conf := new(DnsConfig)
	conf.servers = make([]string, 0, 3) // small, but the standard limit
	conf.search = make([]string, 0)
	conf.ndots = 1
	conf.timeout = 5
	conf.attempts = 2
	conf.rotate = false
	for line, ok := file.ReadLine(); ok; line, ok = file.ReadLine() {
		f := netparse.GetFields(line)
		if len(f) < 1 {
			continue
		}
		switch f[0] {
		case "nameserver": // add one name server
			a := conf.servers
			n := len(a)
			if len(f) > 1 && n < cap(a) {
				// One more check: make sure server name is
				// just an IP address.  Otherwise we need DNS
				// to look it up.
				name := f[1]
				switch len(net.ParseIP(name)) {
				case 16:
					name = "[" + name + "]"
					fallthrough
				case 4:
					a = a[0 : n+1]
					a[n] = name
					conf.servers = a
				}
			}

		case "domain": // set search path to just this domain
			if len(f) > 1 {
				conf.search = make([]string, 1)
				conf.search[0] = f[1]
			} else {
				conf.search = make([]string, 0)
			}

		case "search": // set search path to given servers
			conf.search = make([]string, len(f)-1)
			for i := 0; i < len(conf.search); i++ {
				conf.search[i] = f[i+1]
			}

		case "options": // magic options
			for i := 1; i < len(f); i++ {
				s := f[i]
				switch {
				case len(s) >= 6 && s[0:6] == "ndots:":
					n, _, _ := netparse.Dtoi(s, 6)
					if n < 1 {
						n = 1
					}
					conf.ndots = n
				case len(s) >= 8 && s[0:8] == "timeout:":
					n, _, _ := netparse.Dtoi(s, 8)
					if n < 1 {
						n = 1
					}
					conf.timeout = n
				case len(s) >= 8 && s[0:9] == "attempts:":
					n, _, _ := netparse.Dtoi(s, 9)
					if n < 1 {
						n = 1
					}
					conf.attempts = n
				case s == "rotate":
					conf.rotate = true
				}
			}
		}
	}
	file.Close()

	return conf, nil
}
<commit_msg>Add DnsWriteConfig Rewrite a lot of part to use core pkg<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

// Read system DNS config from /etc/resolv.conf

package dnsconfig

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
)

type DnsConfig struct {
	servers  []string // servers to use
	search   []string // suffixes to append to local name
	ndots    int      // number of dots in name to trigger absolute lookup
	timeout  int      // seconds before giving up on packet
	attempts int      // lost packets before giving up on server
	rotate   bool     // round robin among servers
}

// See resolv.conf(5) on a Linux machine.
// TODO(rsc): Supposed to call uname() and chop the beginning
// of the host name to get the default search domain.
func DnsReadConfig(filename string) (*DnsConfig, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)

	conf := new(DnsConfig)
	// conf.servers = make([]string, 0, 3) // small, but the standard limit
	// conf.search = make([]string, 0)
	// conf.ndots = 1
	// conf.timeout = 5
	// conf.attempts = 2
	// conf.rotate = false

	for scanner.Scan() {
		line := scanner.Text()
		scannerLine := bufio.NewScanner(strings.NewReader(line))
		scannerLine.Split(bufio.ScanWords)
		var lineArr []string
		for scannerLine.Scan() {
			lineArr = append(lineArr, scannerLine.Text())
		}

		//empty line
		if len(lineArr) == 0 {
			continue
		}
		switch lineArr[0] {
		case "nameserver": // add one name server
			if len(lineArr) > 1 {
				conf.servers = append(conf.servers, lineArr[1])
			}

		case "domain": // set search path to just this domain
			if len(lineArr) > 1 {
				conf.search = make([]string, 1)
				conf.search[0] = lineArr[1]
			} else {
				conf.search = make([]string, 0)
			}

		case "search": // set search path to given servers
			conf.search = make([]string, len(lineArr)-1)
			for i := 0; i < len(conf.search); i++ {
				conf.search[i] = lineArr[i+1]
			}

		case "options": // magic options
			for i := 1; i < len(lineArr); i++ {
				s := lineArr[i]
				switch {
				case strings.HasPrefix(s, "ndots:"):
					v := strings.TrimPrefix(s, "ndots:")
					conf.ndots, _ = strconv.Atoi(v)
				case strings.HasPrefix(s, "timeout:"):
					v := strings.TrimPrefix(s, "timeout:")
					conf.timeout, _ = strconv.Atoi(v)
				case strings.HasPrefix(s, "attempts:"):
					v := strings.TrimPrefix(s, "attempts:")
					conf.attempts, _ = strconv.Atoi(v)
				case s == "rotate":
					conf.rotate = true
				}
			}
		}
	}
	return conf, nil
}

func DnsWriteConfig(conf *DnsConfig, filename string) (err error) {
	file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer file.Close()

	w := bufio.NewWriter(file)

	for _, server := range conf.servers {
		line := "nameserver " + server
		fmt.Fprintln(w, line)
	}
	for _, s := range conf.search {
		line := "search " + s
		fmt.Fprintln(w, line)
	}
	if conf.ndots != 0 || conf.timeout != 0 || conf.attempts != 0 || conf.rotate != false {
		line := "options"
		if conf.ndots != 0 {
			line += " ndots:" + strconv.Itoa(conf.ndots)
		}
		if conf.timeout != 0 {
			line += " timeout:" + strconv.Itoa(conf.timeout)
		}
		if conf.attempts != 0 {
			line += " attempts:" + strconv.Itoa(conf.attempts)
		}
		if conf.rotate == true {
			line += " rotate"
		}
		fmt.Fprintln(w, line)
	}
	w.Flush()

	return
}
<|endoftext|>


<commit_before>// Package demo :: chan.go
package demo

import (
	u "github.com/dockerian/go-coding/utils"
)

// MergeChannels merges inputs to one out channel
func MergeChannels(out chan float32, inputs []chan float32) {
	var a, b chan float32
	switch len(inputs) {
	case 2:
		b = inputs[1]
		fallthrough
	case 1:
		a = inputs[0]
	case 0:
	default:
		a = make(chan float32)
		b = make(chan float32)
		half := len(inputs) / 2
		go MergeChannels(a, inputs[:half])
		go MergeChannels(b, inputs[half:])
	}

	mergeChan(out, a, b)
}

func mergeChan(out chan<- float32, a, b <-chan float32) {
	for a != nil || b != nil {
		select {
		case v, ok := <-a:
			if !ok {
				a = nil
				u.Debug("channel a (%v) is closed.\n", &a)
				continue
			}
			out <- v
		case v, ok := <-b:
			if !ok {
				b = nil
				u.Debug("channel b (%v) is closed.\n", &b)
				continue
			}
			out <- v
		}
	}
	close(out)
}
<commit_msg>added func GetMergedChannels<commit_after>// Package demo :: chan.go
package demo

import (
	u "github.com/dockerian/go-coding/utils"
)

// GetMergedChannels merges two channels to one.
// See https://medium.com/justforfunc/why-are-there-nil-channels-in-go-9877cc0b2308
// Notes: in common pattern, each channel (e.g. `ch` as `chan<- int`)
//   - receives data in a lengthy producer (e.g. `ch <- v`)
//   - closes at the end of the producer
func GetMergedChannels(ch1, ch2 <-chan int) <-chan int {
	out := make(chan int)
	go func() {
		defer close(out)
		for ch1 != nil || ch2 != nil {
			select {
			case v, ok := <-ch1:
				if !ok { // checking if ch1 is closed
					ch1 = nil // prevent from busy loop on closed channel which never blocks but nil will
					// u.Debug("channel ch1 (%v) is closed.\n", &ch1)
					continue
				}
				out <- v
			case v, ok := <-ch2:
				if !ok { // checking if ch2 is closed
					ch2 = nil // prevent from busy loop on closed channel which never blocks but nil will
					// u.Debug("channel ch2 (%v) is closed.\n", &ch2)
					continue
				}
				out <- v
			}
		}
	}()
	return out
}

// MergeChannels merges inputs to one out channel
func MergeChannels(out chan float32, inputs []chan float32) {
	var a, b chan float32
	switch len(inputs) {
	case 2:
		b = inputs[1]
		fallthrough
	case 1:
		a = inputs[0]
	case 0:
	default:
		a = make(chan float32)
		b = make(chan float32)
		half := len(inputs) / 2
		go MergeChannels(a, inputs[:half])
		go MergeChannels(b, inputs[half:])
	}

	mergeChan(out, a, b)
}

func mergeChan(out chan<- float32, a, b <-chan float32) {
	for a != nil || b != nil {
		select {
		case v, ok := <-a:
			if !ok {
				a = nil
				u.Debug("channel a (%v) is closed.\n", &a)
				continue
			}
			out <- v
		case v, ok := <-b:
			if !ok {
				b = nil
				u.Debug("channel b (%v) is closed.\n", &b)
				continue
			}
			out <- v
		}
	}
	close(out)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"log"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/arthurkiller/perfM"
)

func main() {
	f, err := os.Open("./data")
	defer f.Close()
	if err != nil {
		log.Println(err)
		return
	}

	br := bufio.NewReader(f)
	wg := new(sync.WaitGroup)
	perfm := perfM.New(perfM.Config{})
	go perfm.Start()

	for {
		s, err := br.ReadString('\n')
		if err != nil {
			if err != io.EOF {
				log.Println(err)
				return
			}
			break
		}
		i, _ := strconv.Atoi(s)
		i %= 10
		wg.Add(1)
		go func() {
			t := perfm.Do()
			time.Sleep(time.Millisecond * time.Duration(i))
			t.Done(perfm)
			defer wg.Done()
		}()
	}

	wg.Wait()
	perfm.Stop()
}
<commit_msg>fast fix the shown<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/arthurkiller/perfM"
)

func main() {
	f, err := os.Open("./data")
	defer f.Close()
	if err != nil {
		log.Println(err)
		return
	}

	br := bufio.NewReader(f)
	wg := new(sync.WaitGroup)
	perfm := perfM.New(perfM.Config{})
	go perfm.Start()

	for {
		s, err := br.ReadString('\n')
		s = strings.Trim(s, "\n")
		if err != nil {
			if err != io.EOF {
				log.Println(err)
				return
			}
			break
		}

		i, err := strconv.Atoi(s)
		if err != nil {
			fmt.Println(err)
		}
		i %= 10
		wg.Add(1)
		go func() {
			t := perfm.Do()
			time.Sleep(100 * time.Millisecond * time.Duration(i))
			t.Done(perfm)
			defer wg.Done()
		}()
	}

	wg.Wait()
	perfm.Stop()
}
<|endoftext|>


<commit_before>package vizzini_test

import (
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/receptor"
	. "github.com/pivotal-cf-experimental/vizzini/matchers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("DiskLimits", func() {
	var lrp receptor.DesiredLRPCreateRequest
	BeforeEach(func() {
		lrp = DesiredLRPWithGuid(guid)
	})

	Describe("with a preloaded rootfs, the disk limit is applied to the COW layer", func() {
		Context("when the disk limit exceeds the contents to be copied in", func() {
			It("should not crash, but should start succesfully", func() {
				lrp.DiskMB = 64
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
			})
		})

		Context("when the disk limit is less than the contents to be copied in", func() {
			It("should crash", func() {
				lrp.DiskMB = 4
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPThatHasCrashed(guid, 0))
			})
		})
	})

	Describe("with a docker-image rootfs", func() {
		BeforeEach(func() {
			lrp.RootFS = "docker:///onsi/grace-busybox"
			lrp.Setup = nil //note: we copy nothing in, the docker image on its own should cause this failure
			lrp.Action = models.WrapAction(&models.RunAction{
				Path: "/grace",
				User: "root",
				Env:  []*models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
			})
			lrp.Monitor = nil
		})

		Context("when the disk limit exceeds the size of the docker image", func() {
			It("should not crash, but should start succesfully", func() {
				lrp.DiskMB = 64
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
			})
		})

		Context("when the disk limit is less than the size of the docker image", func() {
			It("should crash", func() {
				lrp.DiskMB = 4
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPThatHasCrashed(guid, 0))
			})
		})
	})
})
<commit_msg>ensure disk limits tests don't leak an actual lrp (fixes flakiness)<commit_after>package vizzini_test

import (
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/receptor"
	. "github.com/pivotal-cf-experimental/vizzini/matchers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("DiskLimits", func() {
	var lrp receptor.DesiredLRPCreateRequest
	BeforeEach(func() {
		lrp = DesiredLRPWithGuid(guid)
	})

	Describe("with a preloaded rootfs, the disk limit is applied to the COW layer", func() {
		Context("when the disk limit exceeds the contents to be copied in", func() {
			It("should not crash, but should start succesfully", func() {
				lrp.DiskMB = 64
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
			})
		})

		Context("when the disk limit is less than the contents to be copied in", func() {
			It("should crash", func() {
				lrp.DiskMB = 4
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPThatHasCrashed(guid, 0))

				//getting all the way helps ensure the tests don't spuriously fail
				//when we delete the DesiredLRP if the application is in the middle of restarting it looks like we need to wiat for a convergence
				//loop to eventually clean it up.  This is likely a bug, though it's not crticial.
				Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))
			})
		})
	})

	Describe("with a docker-image rootfs", func() {
		BeforeEach(func() {
			lrp.RootFS = "docker:///onsi/grace-busybox"
			lrp.Setup = nil //note: we copy nothing in, the docker image on its own should cause this failure
			lrp.Action = models.WrapAction(&models.RunAction{
				Path: "/grace",
				User: "root",
				Env:  []*models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
			})
			lrp.Monitor = nil
		})

		Context("when the disk limit exceeds the size of the docker image", func() {
			It("should not crash, but should start succesfully", func() {
				lrp.DiskMB = 64
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
			})
		})

		Context("when the disk limit is less than the size of the docker image", func() {
			It("should crash", func() {
				lrp.DiskMB = 4
				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPThatHasCrashed(guid, 0))

				//getting all the way helps ensure the tests don't spuriously fail
				//when we delete the DesiredLRP if the application is in the middle of restarting it looks like we need to wiat for a convergence
				//loop to eventually clean it up.  This is likely a bug, though it's not crticial.
				Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))
			})
		})
	})
})
<|endoftext|>


<commit_before>package miner

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// HeaderMemory is the number of previous calls to 'header'
	// that are remembered. Additionally, 'header' will only poll for a
	// new block every 'headerMemory / blockMemory' times it is
	// called. This reduces the amount of memory used, but comes at the cost of
	// not always having the most recent transactions.
	HeaderMemory = 10000

	// BlockMemory is the maximum number of blocks the miner will store
	// Blocks take up to 2 megabytes of memory, which is why this number is
	// limited.
	BlockMemory = 50

	// MaxSourceBlockAge is the maximum amount of time that is allowed to
	// elapse between generating source blocks.
	MaxSourceBlockAge = 60 * time.Second
)

var (
	errNilCS     = errors.New("miner cannot use a nil consensus set")
	errNilTpool  = errors.New("miner cannot use a nil transaction pool")
	errNilWallet = errors.New("miner cannot use a nil wallet")
)

// Miner struct contains all variables the miner needs
// in order to create and submit blocks.
type Miner struct {
	// Module dependencies.
	cs     modules.ConsensusSet
	tpool  modules.TransactionPool
	wallet modules.Wallet

	// BlockManager variables. Becaues blocks are large, one block is used to
	// make many headers which can be used by miners. Headers include an
	// arbitrary data transaction (appended to the block) to make the merkle
	// roots unique (preventing miners from doing redundant work). Every N
	// requests or M seconds, a new block is used to create headers.
	//
	// Only 'blocksMemory' blocks are kept in memory at a time, which
	// keeps ram usage reasonable. Miners may request many headers in parallel,
	// and thus may be working on different blocks. When they submit the solved
	// header to the block manager, the rest of the block needs to be found in
	// a lookup.
	blockMem        map[types.BlockHeader]*types.Block             // Mappings from headers to the blocks they are derived from.
	arbDataMem      map[types.BlockHeader][crypto.EntropySize]byte // Mappings from the headers to their unique arb data.
	headerMem       []types.BlockHeader                            // A circular list of headers that have been given out from the api recently.
	sourceBlock     *types.Block                                   // The block from which new headers for mining are created.
	sourceBlockTime time.Time                                      // How long headers have been using the same block (different from 'recent block').
	memProgress     int                                            // The index of the most recent header used in headerMem.

	// CPUMiner variables.
	miningOn bool  // indicates if the miner is supposed to be running
	mining   bool  // indicates if the miner is actually running
	hashRate int64 // indicates hashes per second

	// Utils
	log        *persist.Logger
	mu         sync.RWMutex
	persist    persistence
	persistDir string
}

// startupRescan will rescan the blockchain in the event that the miner
// persistance layer has become desynchronized from the consensus persistance
// layer. This might happen if a user replaces any of the folders with backups
// or deletes any of the folders.
func (m *Miner) startupRescan() error {
	// Unsubscribe the miner from the consensus set. Though typically
	// miner.consensusRescan will only be called if the miner is not yet
	// subscribed successfully to the consensus set, the function is allowed to
	// be used in other ways.
	m.cs.Unsubscribe(m)

	// Reset all of the variables that have relevance to the consensus set. The
	// operations are wrapped by an anonymous function so that the locking can
	// be handled using a defer statement.
	err := func() error {
		m.mu.Lock()
		defer m.mu.Unlock()

		m.persist.RecentChange = modules.ConsensusChangeID{}
		m.persist.Height = 0
		m.persist.Target = types.Target{}
		err := m.save()
		if err != nil {
			return err
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// ConsensusSetPerscribe is a blocking call that will not return until
	// rescanning is complete.
	return m.cs.ConsensusSetPersistentSubscribe(m, modules.ConsensusChangeID{})
}

// New returns a ready-to-go miner that is not mining.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, w modules.Wallet, persistDir string) (*Miner, error) {
	// Create the miner and its dependencies.
	if cs == nil {
		return nil, errNilCS
	}
	if tpool == nil {
		return nil, errNilTpool
	}
	if w == nil {
		return nil, errNilWallet
	}

	// Assemble the miner. The miner is assembled without an address because
	// the wallet is likely not unlocked yet. The miner will grab an address
	// after the miner is unlocked (this must be coded manually for each
	// function that potentially requires the miner to have an address.
	m := &Miner{
		cs:     cs,
		tpool:  tpool,
		wallet: w,

		blockMem:   make(map[types.BlockHeader]*types.Block),
		arbDataMem: make(map[types.BlockHeader][crypto.EntropySize]byte),
		headerMem:  make([]types.BlockHeader, HeaderMemory),

		persistDir: persistDir,
	}

	err := m.initPersist()
	if err != nil {
		return nil, errors.New("miner persistence startup failed: " + err.Error())
	}

	err = m.cs.ConsensusSetPersistentSubscribe(m, m.persist.RecentChange)
	if err == modules.ErrInvalidConsensusChangeID {
		// Perform a rescan of the consensus set if the change id is not found.
		// The id will only be not found if there has been desynchronization
		// between the miner and the consensus package.
		err = m.startupRescan()
		if err != nil {
			return nil, errors.New("miner startup failed - rescanning failed: " + err.Error())
		}
	} else if err != nil {
		return nil, errors.New("miner subscription failed: " + err.Error())
	}

	m.tpool.TransactionPoolSubscribe(m)
	return m, nil
}

// Close terminates all ongoing processes involving the miner, enabling garbage
// collection.
func (m *Miner) Close() error {
	m.cs.Unsubscribe(m)
	return m.log.Close()
}

// checkAddress checks that the miner has an address, fetching an address from
// the wallet if not.
func (m *Miner) checkAddress() error {
	if m.persist.Address != (types.UnlockHash{}) {
		return nil
	}
	uc, err := m.wallet.NextAddress()
	if err != nil {
		return err
	}
	m.persist.Address = uc.UnlockHash()
	return nil
}

// BlocksMined returns the number of good blocks and stale blocks that have
// been mined by the miner.
func (m *Miner) BlocksMined() (goodBlocks, staleBlocks int) {
	m.mu.Lock()
	defer m.mu.Unlock()

	for _, blockID := range m.persist.BlocksFound {
		if m.cs.InCurrentPath(blockID) {
			goodBlocks++
		} else {
			staleBlocks++
		}
	}
	return
}
<commit_msg>save before closing in miner<commit_after>package miner

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// HeaderMemory is the number of previous calls to 'header'
	// that are remembered. Additionally, 'header' will only poll for a
	// new block every 'headerMemory / blockMemory' times it is
	// called. This reduces the amount of memory used, but comes at the cost of
	// not always having the most recent transactions.
	HeaderMemory = 10000

	// BlockMemory is the maximum number of blocks the miner will store
	// Blocks take up to 2 megabytes of memory, which is why this number is
	// limited.
	BlockMemory = 50

	// MaxSourceBlockAge is the maximum amount of time that is allowed to
	// elapse between generating source blocks.
	MaxSourceBlockAge = 60 * time.Second
)

var (
	errNilCS     = errors.New("miner cannot use a nil consensus set")
	errNilTpool  = errors.New("miner cannot use a nil transaction pool")
	errNilWallet = errors.New("miner cannot use a nil wallet")
)

// Miner struct contains all variables the miner needs
// in order to create and submit blocks.
type Miner struct {
	// Module dependencies.
	cs     modules.ConsensusSet
	tpool  modules.TransactionPool
	wallet modules.Wallet

	// BlockManager variables. Becaues blocks are large, one block is used to
	// make many headers which can be used by miners. Headers include an
	// arbitrary data transaction (appended to the block) to make the merkle
	// roots unique (preventing miners from doing redundant work). Every N
	// requests or M seconds, a new block is used to create headers.
	//
	// Only 'blocksMemory' blocks are kept in memory at a time, which
	// keeps ram usage reasonable. Miners may request many headers in parallel,
	// and thus may be working on different blocks. When they submit the solved
	// header to the block manager, the rest of the block needs to be found in
	// a lookup.
	blockMem        map[types.BlockHeader]*types.Block             // Mappings from headers to the blocks they are derived from.
	arbDataMem      map[types.BlockHeader][crypto.EntropySize]byte // Mappings from the headers to their unique arb data.
	headerMem       []types.BlockHeader                            // A circular list of headers that have been given out from the api recently.
	sourceBlock     *types.Block                                   // The block from which new headers for mining are created.
	sourceBlockTime time.Time                                      // How long headers have been using the same block (different from 'recent block').
	memProgress     int                                            // The index of the most recent header used in headerMem.

	// CPUMiner variables.
	miningOn bool  // indicates if the miner is supposed to be running
	mining   bool  // indicates if the miner is actually running
	hashRate int64 // indicates hashes per second

	// Utils
	log        *persist.Logger
	mu         sync.RWMutex
	persist    persistence
	persistDir string
}

// startupRescan will rescan the blockchain in the event that the miner
// persistance layer has become desynchronized from the consensus persistance
// layer. This might happen if a user replaces any of the folders with backups
// or deletes any of the folders.
func (m *Miner) startupRescan() error {
	// Unsubscribe the miner from the consensus set. Though typically
	// miner.consensusRescan will only be called if the miner is not yet
	// subscribed successfully to the consensus set, the function is allowed to
	// be used in other ways.
	m.cs.Unsubscribe(m)

	// Reset all of the variables that have relevance to the consensus set. The
	// operations are wrapped by an anonymous function so that the locking can
	// be handled using a defer statement.
	err := func() error {
		m.mu.Lock()
		defer m.mu.Unlock()

		m.persist.RecentChange = modules.ConsensusChangeID{}
		m.persist.Height = 0
		m.persist.Target = types.Target{}
		err := m.save()
		if err != nil {
			return err
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// ConsensusSetPerscribe is a blocking call that will not return until
	// rescanning is complete.
	return m.cs.ConsensusSetPersistentSubscribe(m, modules.ConsensusChangeID{})
}

// New returns a ready-to-go miner that is not mining.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, w modules.Wallet, persistDir string) (*Miner, error) {
	// Create the miner and its dependencies.
	if cs == nil {
		return nil, errNilCS
	}
	if tpool == nil {
		return nil, errNilTpool
	}
	if w == nil {
		return nil, errNilWallet
	}

	// Assemble the miner. The miner is assembled without an address because
	// the wallet is likely not unlocked yet. The miner will grab an address
	// after the miner is unlocked (this must be coded manually for each
	// function that potentially requires the miner to have an address.
	m := &Miner{
		cs:     cs,
		tpool:  tpool,
		wallet: w,

		blockMem:   make(map[types.BlockHeader]*types.Block),
		arbDataMem: make(map[types.BlockHeader][crypto.EntropySize]byte),
		headerMem:  make([]types.BlockHeader, HeaderMemory),

		persistDir: persistDir,
	}

	err := m.initPersist()
	if err != nil {
		return nil, errors.New("miner persistence startup failed: " + err.Error())
	}

	err = m.cs.ConsensusSetPersistentSubscribe(m, m.persist.RecentChange)
	if err == modules.ErrInvalidConsensusChangeID {
		// Perform a rescan of the consensus set if the change id is not found.
		// The id will only be not found if there has been desynchronization
		// between the miner and the consensus package.
		err = m.startupRescan()
		if err != nil {
			return nil, errors.New("miner startup failed - rescanning failed: " + err.Error())
		}
	} else if err != nil {
		return nil, errors.New("miner subscription failed: " + err.Error())
	}

	m.tpool.TransactionPoolSubscribe(m)
	return m, nil
}

// Close terminates all ongoing processes involving the miner, enabling garbage
// collection.
func (m *Miner) Close() error {
	// Save the latest miner state.
	err := func() error {
		m.mu.Lock()
		defer m.mu.Unlock()
		return m.save()
	}()
	if err != nil {
		return err
	}

	m.cs.Unsubscribe(m)
	return m.log.Close()
}

// checkAddress checks that the miner has an address, fetching an address from
// the wallet if not.
func (m *Miner) checkAddress() error {
	if m.persist.Address != (types.UnlockHash{}) {
		return nil
	}
	uc, err := m.wallet.NextAddress()
	if err != nil {
		return err
	}
	m.persist.Address = uc.UnlockHash()
	return nil
}

// BlocksMined returns the number of good blocks and stale blocks that have
// been mined by the miner.
func (m *Miner) BlocksMined() (goodBlocks, staleBlocks int) {
	m.mu.Lock()
	defer m.mu.Unlock()

	for _, blockID := range m.persist.BlocksFound {
		if m.cs.InCurrentPath(blockID) {
			goodBlocks++
		} else {
			staleBlocks++
		}
	}
	return
}
<|endoftext|>


<commit_before>package daemon

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/containers/image/docker/reference"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/docker/docker/client"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
)

type daemonImageDestination struct {
	ref            daemonReference
	namedTaggedRef reference.NamedTagged // Strictly speaking redundant with ref above; having the field makes it structurally impossible for later users to fail.
	// For talking to imageLoadGoroutine
	goroutineCancel context.CancelFunc
	statusChannel   <-chan error
	writer          *io.PipeWriter
	tar             *tar.Writer
	// Other state
	committed bool                             // writer has been closed
	blobs     map[digest.Digest]types.BlobInfo // list of already-sent blobs
}

// newImageDestination returns a types.ImageDestination for the specified image reference.
func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (types.ImageDestination, error) {
	if ref.ref == nil {
		return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
	}
	namedTaggedRef, ok := ref.ref.(reference.NamedTagged)
	if !ok {
		return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
	}

	c, err := client.NewClient(client.DefaultDockerHost, "1.22", nil, nil) // FIXME: overridable host
	if err != nil {
		return nil, errors.Wrap(err, "Error initializing docker engine client")
	}

	reader, writer := io.Pipe()
	// Commit() may never be called, so we may never read from this channel; so, make this buffered to allow imageLoadGoroutine to write status and terminate even if we never read it.
	statusChannel := make(chan error, 1)

	ctx, goroutineCancel := context.WithCancel(context.Background())
	go imageLoadGoroutine(ctx, c, reader, statusChannel)

	return &daemonImageDestination{
		ref:             ref,
		namedTaggedRef:  namedTaggedRef,
		goroutineCancel: goroutineCancel,
		statusChannel:   statusChannel,
		writer:          writer,
		tar:             tar.NewWriter(writer),
		committed:       false,
		blobs:           make(map[digest.Digest]types.BlobInfo),
	}, nil
}

// imageLoadGoroutine accepts tar stream on reader, sends it to c, and reports error or success by writing to statusChannel
func imageLoadGoroutine(ctx context.Context, c *client.Client, reader *io.PipeReader, statusChannel chan<- error) {
	err := errors.New("Internal error: unexpected panic in imageLoadGoroutine")
	defer func() {
		logrus.Debugf("docker-daemon: sending done, status %v", err)
		statusChannel <- err
	}()
	defer func() {
		if err == nil {
			reader.Close()
		} else {
			reader.CloseWithError(err)
		}
	}()

	resp, err := c.ImageLoad(ctx, reader, true)
	if err != nil {
		err = errors.Wrap(err, "Error saving image to docker engine")
		return
	}
	defer resp.Body.Close()
}

// Close removes resources associated with an initialized ImageDestination, if any.
func (d *daemonImageDestination) Close() {
	if !d.committed {
		logrus.Debugf("docker-daemon: Closing tar stream to abort loading")
		// In principle, goroutineCancel() should abort the HTTP request and stop the process from continuing.
		// In practice, though, various HTTP implementations used by client.Client.ImageLoad() (including
		// https://github.com/golang/net/blob/master/context/ctxhttp/ctxhttp_pre17.go and the
		// net/http version with native Context support in Go 1.7) do not always actually immediately cancel
		// the operation: they may process the HTTP request, or a part of it, to completion in a goroutine, and
		// return early if the context is canceled without terminating the goroutine at all.
		// So we need this CloseWithError to terminate sending the HTTP request Body
		// immediately, and hopefully, through terminating the sending which uses "Transfer-Encoding: chunked"" without sending
		// the terminating zero-length chunk, prevent the docker daemon from processing the tar stream at all.
		// Whether that works or not, closing the PipeWriter seems desirable in any case.
		d.writer.CloseWithError(errors.New("Aborting upload, daemonImageDestination closed without a previous .Commit()"))
	}
	d.goroutineCancel()
}

// Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *daemonImageDestination) Reference() types.ImageReference {
	return d.ref
}

// SupportedManifestMIMETypes tells which manifest mime types the destination supports
// If an empty slice or nil it's returned, then any mime type can be tried to upload
func (d *daemonImageDestination) SupportedManifestMIMETypes() []string {
	return []string{
		manifest.DockerV2Schema2MediaType, // We rely on the types.Image.UpdatedImage schema conversion capabilities.
	}
}

// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
func (d *daemonImageDestination) SupportsSignatures() error {
	return errors.Errorf("Storing signatures for docker-daemon: destinations is not supported")
}

// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination.
func (d *daemonImageDestination) ShouldCompressLayers() bool {
	return false
}

// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
// uploaded to the image destination, true otherwise.
func (d *daemonImageDestination) AcceptsForeignLayerURLs() bool {
	return false
}

// PutBlob writes contents of stream and returns data representing the result (with all data filled in).
// inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it.
// inputInfo.Size is the expected length of stream, if known.
// WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
func (d *daemonImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) {
	if ok, size, err := d.HasBlob(inputInfo); err == nil && ok {
		return types.BlobInfo{Digest: inputInfo.Digest, Size: size}, nil
	}

	if inputInfo.Size == -1 { // Ouch, we need to stream the blob into a temporary file just to determine the size.
		logrus.Debugf("docker-daemon: input with unknown size, streaming to disk first…")
		streamCopy, err := ioutil.TempFile(temporaryDirectoryForBigFiles, "docker-daemon-blob")
		if err != nil {
			return types.BlobInfo{}, err
		}
		defer os.Remove(streamCopy.Name())
		defer streamCopy.Close()

		size, err := io.Copy(streamCopy, stream)
		if err != nil {
			return types.BlobInfo{}, err
		}
		_, err = streamCopy.Seek(0, os.SEEK_SET)
		if err != nil {
			return types.BlobInfo{}, err
		}
		inputInfo.Size = size // inputInfo is a struct, so we are only modifying our copy.
		stream = streamCopy
		logrus.Debugf("… streaming done")
	}

	digester := digest.Canonical.Digester()
	tee := io.TeeReader(stream, digester.Hash())
	if err := d.sendFile(inputInfo.Digest.String(), inputInfo.Size, tee); err != nil {
		return types.BlobInfo{}, err
	}
	d.blobs[inputInfo.Digest] = types.BlobInfo{Digest: digester.Digest(), Size: inputInfo.Size}
	return types.BlobInfo{Digest: digester.Digest(), Size: inputInfo.Size}, nil
}

func (d *daemonImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) {
	if info.Digest == "" {
		return false, -1, errors.Errorf(`"Can not check for a blob with unknown digest`)
	}
	if blob, ok := d.blobs[info.Digest]; ok {
		return true, blob.Size, nil
	}
	return false, -1, types.ErrBlobNotFound
}

func (d *daemonImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) {
	return info, nil
}

func (d *daemonImageDestination) PutManifest(m []byte) error {
	var man schema2Manifest
	if err := json.Unmarshal(m, &man); err != nil {
		return errors.Wrap(err, "Error parsing manifest")
	}
	if man.SchemaVersion != 2 || man.MediaType != manifest.DockerV2Schema2MediaType {
		return errors.Errorf("Unsupported manifest type, need a Docker schema 2 manifest")
	}

	layerPaths := []string{}
	for _, l := range man.Layers {
		layerPaths = append(layerPaths, l.Digest.String())
	}

	// For github.com/docker/docker consumers, this works just as well as
	//   refString := d.namedTaggedRef.String()  [i.e. d.ref.ref.String()]
	// because when reading the RepoTags strings, github.com/docker/docker/reference
	// normalizes both of them to the same value.
	//
	// Doing it this way to include the normalized-out `docker.io[/library]` does make
	// a difference for github.com/projectatomic/docker consumers, with the
	// “Add --add-registry and --block-registry options to docker daemon” patch.
	// These consumers treat reference strings which include a hostname and reference
	// strings without a hostname differently.
	//
	// Using the host name here is more explicit about the intent, and it has the same
	// effect as (docker pull) in projectatomic/docker, which tags the result using
	// a hostname-qualified reference.
	// See https://github.com/containers/image/issues/72 for a more detailed
	// analysis and explanation.
	refString := fmt.Sprintf("%s:%s", d.namedTaggedRef.FullName(), d.namedTaggedRef.Tag())

	items := []manifestItem{{
		Config:       man.Config.Digest.String(),
		RepoTags:     []string{refString},
		Layers:       layerPaths,
		Parent:       "",
		LayerSources: nil,
	}}
	itemsBytes, err := json.Marshal(&items)
	if err != nil {
		return err
	}

	// FIXME? Do we also need to support the legacy format?
	return d.sendFile(manifestFileName, int64(len(itemsBytes)), bytes.NewReader(itemsBytes))
}

type tarFI struct {
	path string
	size int64
}

func (t *tarFI) Name() string {
	return t.path
}
func (t *tarFI) Size() int64 {
	return t.size
}
func (t *tarFI) Mode() os.FileMode {
	return 0444
}
func (t *tarFI) ModTime() time.Time {
	return time.Unix(0, 0)
}
func (t *tarFI) IsDir() bool {
	return false
}
func (t *tarFI) Sys() interface{} {
	return nil
}

// sendFile sends a file into the tar stream.
func (d *daemonImageDestination) sendFile(path string, expectedSize int64, stream io.Reader) error {
	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: expectedSize}, "")
	if err != nil {
		return nil
	}
	logrus.Debugf("Sending as tar file %s", path)
	if err := d.tar.WriteHeader(hdr); err != nil {
		return err
	}
	size, err := io.Copy(d.tar, stream)
	if err != nil {
		return err
	}
	if size != expectedSize {
		return errors.Errorf("Size mismatch when copying %s, expected %d, got %d", path, expectedSize, size)
	}
	return nil
}

func (d *daemonImageDestination) PutSignatures(signatures [][]byte) error {
	if len(signatures) != 0 {
		return errors.Errorf("Storing signatures for docker-daemon: destinations is not supported")
	}
	return nil
}

// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *daemonImageDestination) Commit() error {
	logrus.Debugf("docker-daemon: Closing tar stream")
	if err := d.tar.Close(); err != nil {
		return err
	}
	if err := d.writer.Close(); err != nil {
		return err
	}
	d.committed = true // We may still fail, but we are done sending to imageLoadGoroutine.

	logrus.Debugf("docker-daemon: Waiting for status")
	err := <-d.statusChannel
	return err
}
<commit_msg>Restore a check for unknown digest in daemonImageDestination.PutBlob<commit_after>package daemon

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/containers/image/docker/reference"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/docker/docker/client"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
)

type daemonImageDestination struct {
	ref            daemonReference
	namedTaggedRef reference.NamedTagged // Strictly speaking redundant with ref above; having the field makes it structurally impossible for later users to fail.
	// For talking to imageLoadGoroutine
	goroutineCancel context.CancelFunc
	statusChannel   <-chan error
	writer          *io.PipeWriter
	tar             *tar.Writer
	// Other state
	committed bool                             // writer has been closed
	blobs     map[digest.Digest]types.BlobInfo // list of already-sent blobs
}

// newImageDestination returns a types.ImageDestination for the specified image reference.
func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (types.ImageDestination, error) {
	if ref.ref == nil {
		return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
	}
	namedTaggedRef, ok := ref.ref.(reference.NamedTagged)
	if !ok {
		return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
	}

	c, err := client.NewClient(client.DefaultDockerHost, "1.22", nil, nil) // FIXME: overridable host
	if err != nil {
		return nil, errors.Wrap(err, "Error initializing docker engine client")
	}

	reader, writer := io.Pipe()
	// Commit() may never be called, so we may never read from this channel; so, make this buffered to allow imageLoadGoroutine to write status and terminate even if we never read it.
	statusChannel := make(chan error, 1)

	ctx, goroutineCancel := context.WithCancel(context.Background())
	go imageLoadGoroutine(ctx, c, reader, statusChannel)

	return &daemonImageDestination{
		ref:             ref,
		namedTaggedRef:  namedTaggedRef,
		goroutineCancel: goroutineCancel,
		statusChannel:   statusChannel,
		writer:          writer,
		tar:             tar.NewWriter(writer),
		committed:       false,
		blobs:           make(map[digest.Digest]types.BlobInfo),
	}, nil
}

// imageLoadGoroutine accepts tar stream on reader, sends it to c, and reports error or success by writing to statusChannel
func imageLoadGoroutine(ctx context.Context, c *client.Client, reader *io.PipeReader, statusChannel chan<- error) {
	err := errors.New("Internal error: unexpected panic in imageLoadGoroutine")
	defer func() {
		logrus.Debugf("docker-daemon: sending done, status %v", err)
		statusChannel <- err
	}()
	defer func() {
		if err == nil {
			reader.Close()
		} else {
			reader.CloseWithError(err)
		}
	}()

	resp, err := c.ImageLoad(ctx, reader, true)
	if err != nil {
		err = errors.Wrap(err, "Error saving image to docker engine")
		return
	}
	defer resp.Body.Close()
}

// Close removes resources associated with an initialized ImageDestination, if any.
func (d *daemonImageDestination) Close() {
	if !d.committed {
		logrus.Debugf("docker-daemon: Closing tar stream to abort loading")
		// In principle, goroutineCancel() should abort the HTTP request and stop the process from continuing.
		// In practice, though, various HTTP implementations used by client.Client.ImageLoad() (including
		// https://github.com/golang/net/blob/master/context/ctxhttp/ctxhttp_pre17.go and the
		// net/http version with native Context support in Go 1.7) do not always actually immediately cancel
		// the operation: they may process the HTTP request, or a part of it, to completion in a goroutine, and
		// return early if the context is canceled without terminating the goroutine at all.
		// So we need this CloseWithError to terminate sending the HTTP request Body
		// immediately, and hopefully, through terminating the sending which uses "Transfer-Encoding: chunked"" without sending
		// the terminating zero-length chunk, prevent the docker daemon from processing the tar stream at all.
		// Whether that works or not, closing the PipeWriter seems desirable in any case.
		d.writer.CloseWithError(errors.New("Aborting upload, daemonImageDestination closed without a previous .Commit()"))
	}
	d.goroutineCancel()
}

// Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *daemonImageDestination) Reference() types.ImageReference {
	return d.ref
}

// SupportedManifestMIMETypes tells which manifest mime types the destination supports
// If an empty slice or nil it's returned, then any mime type can be tried to upload
func (d *daemonImageDestination) SupportedManifestMIMETypes() []string {
	return []string{
		manifest.DockerV2Schema2MediaType, // We rely on the types.Image.UpdatedImage schema conversion capabilities.
	}
}

// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
func (d *daemonImageDestination) SupportsSignatures() error {
	return errors.Errorf("Storing signatures for docker-daemon: destinations is not supported")
}

// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination.
func (d *daemonImageDestination) ShouldCompressLayers() bool {
	return false
}

// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
// uploaded to the image destination, true otherwise.
func (d *daemonImageDestination) AcceptsForeignLayerURLs() bool {
	return false
}

// PutBlob writes contents of stream and returns data representing the result (with all data filled in).
// inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it.
// inputInfo.Size is the expected length of stream, if known.
// WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
func (d *daemonImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) {
	if inputInfo.Digest.String() == "" {
		return types.BlobInfo{}, errors.Errorf(`Can not stream a blob with unknown digest to "docker-daemon:"`)
	}

	if ok, size, err := d.HasBlob(inputInfo); err == nil && ok {
		return types.BlobInfo{Digest: inputInfo.Digest, Size: size}, nil
	}

	if inputInfo.Size == -1 { // Ouch, we need to stream the blob into a temporary file just to determine the size.
		logrus.Debugf("docker-daemon: input with unknown size, streaming to disk first…")
		streamCopy, err := ioutil.TempFile(temporaryDirectoryForBigFiles, "docker-daemon-blob")
		if err != nil {
			return types.BlobInfo{}, err
		}
		defer os.Remove(streamCopy.Name())
		defer streamCopy.Close()

		size, err := io.Copy(streamCopy, stream)
		if err != nil {
			return types.BlobInfo{}, err
		}
		_, err = streamCopy.Seek(0, os.SEEK_SET)
		if err != nil {
			return types.BlobInfo{}, err
		}
		inputInfo.Size = size // inputInfo is a struct, so we are only modifying our copy.
		stream = streamCopy
		logrus.Debugf("… streaming done")
	}

	digester := digest.Canonical.Digester()
	tee := io.TeeReader(stream, digester.Hash())
	if err := d.sendFile(inputInfo.Digest.String(), inputInfo.Size, tee); err != nil {
		return types.BlobInfo{}, err
	}
	d.blobs[inputInfo.Digest] = types.BlobInfo{Digest: digester.Digest(), Size: inputInfo.Size}
	return types.BlobInfo{Digest: digester.Digest(), Size: inputInfo.Size}, nil
}

func (d *daemonImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) {
	if info.Digest == "" {
		return false, -1, errors.Errorf(`"Can not check for a blob with unknown digest`)
	}
	if blob, ok := d.blobs[info.Digest]; ok {
		return true, blob.Size, nil
	}
	return false, -1, types.ErrBlobNotFound
}

func (d *daemonImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) {
	return info, nil
}

func (d *daemonImageDestination) PutManifest(m []byte) error {
	var man schema2Manifest
	if err := json.Unmarshal(m, &man); err != nil {
		return errors.Wrap(err, "Error parsing manifest")
	}
	if man.SchemaVersion != 2 || man.MediaType != manifest.DockerV2Schema2MediaType {
		return errors.Errorf("Unsupported manifest type, need a Docker schema 2 manifest")
	}

	layerPaths := []string{}
	for _, l := range man.Layers {
		layerPaths = append(layerPaths, l.Digest.String())
	}

	// For github.com/docker/docker consumers, this works just as well as
	//   refString := d.namedTaggedRef.String()  [i.e. d.ref.ref.String()]
	// because when reading the RepoTags strings, github.com/docker/docker/reference
	// normalizes both of them to the same value.
	//
	// Doing it this way to include the normalized-out `docker.io[/library]` does make
	// a difference for github.com/projectatomic/docker consumers, with the
	// “Add --add-registry and --block-registry options to docker daemon” patch.
	// These consumers treat reference strings which include a hostname and reference
	// strings without a hostname differently.
	//
	// Using the host name here is more explicit about the intent, and it has the same
	// effect as (docker pull) in projectatomic/docker, which tags the result using
	// a hostname-qualified reference.
	// See https://github.com/containers/image/issues/72 for a more detailed
	// analysis and explanation.
	refString := fmt.Sprintf("%s:%s", d.namedTaggedRef.FullName(), d.namedTaggedRef.Tag())

	items := []manifestItem{{
		Config:       man.Config.Digest.String(),
		RepoTags:     []string{refString},
		Layers:       layerPaths,
		Parent:       "",
		LayerSources: nil,
	}}
	itemsBytes, err := json.Marshal(&items)
	if err != nil {
		return err
	}

	// FIXME? Do we also need to support the legacy format?
	return d.sendFile(manifestFileName, int64(len(itemsBytes)), bytes.NewReader(itemsBytes))
}

type tarFI struct {
	path string
	size int64
}

func (t *tarFI) Name() string {
	return t.path
}
func (t *tarFI) Size() int64 {
	return t.size
}
func (t *tarFI) Mode() os.FileMode {
	return 0444
}
func (t *tarFI) ModTime() time.Time {
	return time.Unix(0, 0)
}
func (t *tarFI) IsDir() bool {
	return false
}
func (t *tarFI) Sys() interface{} {
	return nil
}

// sendFile sends a file into the tar stream.
func (d *daemonImageDestination) sendFile(path string, expectedSize int64, stream io.Reader) error {
	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: expectedSize}, "")
	if err != nil {
		return nil
	}
	logrus.Debugf("Sending as tar file %s", path)
	if err := d.tar.WriteHeader(hdr); err != nil {
		return err
	}
	size, err := io.Copy(d.tar, stream)
	if err != nil {
		return err
	}
	if size != expectedSize {
		return errors.Errorf("Size mismatch when copying %s, expected %d, got %d", path, expectedSize, size)
	}
	return nil
}

func (d *daemonImageDestination) PutSignatures(signatures [][]byte) error {
	if len(signatures) != 0 {
		return errors.Errorf("Storing signatures for docker-daemon: destinations is not supported")
	}
	return nil
}

// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *daemonImageDestination) Commit() error {
	logrus.Debugf("docker-daemon: Closing tar stream")
	if err := d.tar.Close(); err != nil {
		return err
	}
	if err := d.writer.Close(); err != nil {
		return err
	}
	d.committed = true // We may still fail, but we are done sending to imageLoadGoroutine.

	logrus.Debugf("docker-daemon: Waiting for status")
	err := <-d.statusChannel
	return err
}
<|endoftext|>