<commit_before>// gogetdoc gets documentation for Go objects given their locations in the source code
package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"
)

var (
	cpuprofile           = flag.String("cpuprofile", "", "write cpu profile to file")
	pos                  = flag.String("pos", "", "Filename and byte offset of item to document, e.g. foo.go:#123")
	modified             = flag.Bool("modified", false, "read an archive of modified files from standard input")
	linelength           = flag.Int("linelength", 80, "maximum length of a line in the output (in Unicode code points)")
	jsonOutput           = flag.Bool("json", false, "enable extended JSON output")
	showUnexportedFields = flag.Bool("u", false, "show unexported fields")
)

const modifiedUsage = `
The archive format for the -modified flag consists of the file name, followed
by a newline, the decimal file size, another newline, and the contents of the file.

This allows editors to supply gogetdoc with the contents of their unsaved buffers.
`

func main() {
	// disable GC as gogetdoc is a short-lived program
	debug.SetGCPercent(-1)

	flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s\n", os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, modifiedUsage)
	}
	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}
	filename, offset, err := parsePos(*pos)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	ctx := &build.Default
	ctx.CgoEnabled = false
	if *modified {
		var overlay map[string][]byte
		overlay, err = buildutil.ParseOverlayArchive(os.Stdin)
		if err != nil {
			log.Fatalln("invalid archive:", err)
		}
		ctx = buildutil.OverlayContext(ctx, overlay)
	}

	d, err := Run(ctx, filename, offset)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	if *jsonOutput {
		json.NewEncoder(os.Stdout).Encode(d)
	} else {
		fmt.Println(d.String())
	}
}

// Run is a wrapper for the gogetdoc command.  It is broken out of main for easier testing.
func Run(ctx *build.Context, filename string, offset int64) (*Doc, error) {
	wd, err := os.Getwd()
	if err != nil {
		return nil, errors.New("gogetdoc: couldn't get working directory")
	}
	bp, err := buildutil.ContainingPackage(ctx, wd, filename)
	if err != nil {
		return nil, fmt.Errorf("gogetdoc: couldn't get package for %s: %s", filename, err.Error())
	}
	var parseError error
	conf := &loader.Config{
		Build:      ctx,
		ParserMode: parser.ParseComments,
		TypeCheckFuncBodies: func(pkg string) bool {
			match := pkg == bp.ImportPath
			if strings.HasSuffix(filename, "_test.go") {
				return pkg == bp.ImportPath+"_test" || match
			}
			return match
		},
		AllowErrors: true,
		TypeChecker: types.Config{
			DisableUnusedImportCheck: true,
			Error: func(err error) {
				if parseError != nil {
					return
				}
				parseError = err
			},
		},
	}

	if isTestFile := strings.HasSuffix(filename, "_test.go"); isTestFile {
		conf.ImportWithTests(bp.ImportPath)
	} else {
		conf.Import(bp.ImportPath)
	}

	lprog, err := conf.Load()
	if err != nil {
		return nil, fmt.Errorf("gogetdoc: error loading program: %s", err.Error())
	}
	doc, err := DocForPos(ctx, lprog, filename, offset)
	if err != nil && parseError != nil {
		fmt.Fprintln(os.Stderr, parseError)
	}
	return doc, err
}

// DocForPos attempts to get the documentation for an item given a filename and byte offset.
func DocForPos(ctxt *build.Context, lprog *loader.Program, filename string, offset int64) (*Doc, error) {
	tokFile := FileFromProgram(lprog, filename)
	if tokFile == nil {
		return nil, fmt.Errorf("gogetdoc: couldn't find %s in program", filename)
	}
	offPos := tokFile.Pos(int(offset))

	pkgInfo, nodes, _ := lprog.PathEnclosingInterval(offPos, offPos)
	for _, node := range nodes {
		switch i := node.(type) {
		case *ast.ImportSpec:
			abs, err := filepath.Abs(filename)
			if err != nil {
				return nil, err
			}
			return PackageDoc(ctxt, lprog.Fset, filepath.Dir(abs), ImportPath(i))
		case *ast.Ident:
			// if we can't find the object denoted by the identifier, keep searching)
			if obj := pkgInfo.ObjectOf(i); obj == nil {
				continue
			}
			return IdentDoc(ctxt, i, pkgInfo, lprog)
		default:
			break
		}
	}
	return nil, errors.New("gogetdoc: no documentation found")
}

// FileFromProgram attempts to locate a token.File from a loaded program.
func FileFromProgram(prog *loader.Program, name string) *token.File {
	for _, info := range prog.AllPackages {
		for _, astFile := range info.Files {
			tokFile := prog.Fset.File(astFile.Pos())
			if tokFile == nil {
				continue
			}
			tokName := tokFile.Name()
			if runtime.GOOS == "windows" {
				tokName = filepath.ToSlash(tokName)
				name = filepath.ToSlash(name)
			}
			if tokName == name {
				return tokFile
			}
			if sameFile(tokName, name) {
				return tokFile
			}
		}
	}
	return nil
}

func parsePos(p string) (filename string, offset int64, err error) {
	// foo.go:#123
	if p == "" {
		err = errors.New("missing required -pos flag")
		return
	}
	sep := strings.LastIndex(p, ":")
	// need at least 2 characters after the ':'
	// (the # sign and the offset)
	if sep == -1 || sep > len(p)-2 || p[sep+1] != '#' {
		err = fmt.Errorf("invalid option: -pos=%s", p)
		return
	}
	filename = p[:sep]
	offset, err = strconv.ParseInt(p[sep+2:], 10, 32)
	return
}

func sameFile(a, b string) bool {
	if filepath.Base(a) != filepath.Base(b) {
		// We only care about symlinks for the GOPATH itself. File
		// names need to match.
		return false
	}
	if ai, err := os.Stat(a); err == nil {
		if bi, err := os.Stat(b); err == nil {
			return os.SameFile(ai, bi)
		}
	}
	return false
}
<commit_msg>Write errors to stderr<commit_after>// gogetdoc gets documentation for Go objects given their locations in the source code
package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"
)

var (
	cpuprofile           = flag.String("cpuprofile", "", "write cpu profile to file")
	pos                  = flag.String("pos", "", "Filename and byte offset of item to document, e.g. foo.go:#123")
	modified             = flag.Bool("modified", false, "read an archive of modified files from standard input")
	linelength           = flag.Int("linelength", 80, "maximum length of a line in the output (in Unicode code points)")
	jsonOutput           = flag.Bool("json", false, "enable extended JSON output")
	showUnexportedFields = flag.Bool("u", false, "show unexported fields")
)

const modifiedUsage = `
The archive format for the -modified flag consists of the file name, followed
by a newline, the decimal file size, another newline, and the contents of the file.

This allows editors to supply gogetdoc with the contents of their unsaved buffers.
`

func main() {
	// disable GC as gogetdoc is a short-lived program
	debug.SetGCPercent(-1)

	flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s\n", os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, modifiedUsage)
	}
	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}
	filename, offset, err := parsePos(*pos)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	ctx := &build.Default
	ctx.CgoEnabled = false
	if *modified {
		var overlay map[string][]byte
		overlay, err = buildutil.ParseOverlayArchive(os.Stdin)
		if err != nil {
			log.Fatalln("invalid archive:", err)
		}
		ctx = buildutil.OverlayContext(ctx, overlay)
	}

	d, err := Run(ctx, filename, offset)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if *jsonOutput {
		json.NewEncoder(os.Stdout).Encode(d)
	} else {
		fmt.Println(d.String())
	}
}

// Run is a wrapper for the gogetdoc command.  It is broken out of main for easier testing.
func Run(ctx *build.Context, filename string, offset int64) (*Doc, error) {
	wd, err := os.Getwd()
	if err != nil {
		return nil, errors.New("gogetdoc: couldn't get working directory")
	}
	bp, err := buildutil.ContainingPackage(ctx, wd, filename)
	if err != nil {
		return nil, fmt.Errorf("gogetdoc: couldn't get package for %s: %s", filename, err.Error())
	}
	var parseError error
	conf := &loader.Config{
		Build:      ctx,
		ParserMode: parser.ParseComments,
		TypeCheckFuncBodies: func(pkg string) bool {
			match := pkg == bp.ImportPath
			if strings.HasSuffix(filename, "_test.go") {
				return pkg == bp.ImportPath+"_test" || match
			}
			return match
		},
		AllowErrors: true,
		TypeChecker: types.Config{
			DisableUnusedImportCheck: true,
			Error: func(err error) {
				if parseError != nil {
					return
				}
				parseError = err
			},
		},
	}

	if isTestFile := strings.HasSuffix(filename, "_test.go"); isTestFile {
		conf.ImportWithTests(bp.ImportPath)
	} else {
		conf.Import(bp.ImportPath)
	}

	lprog, err := conf.Load()
	if err != nil {
		return nil, fmt.Errorf("gogetdoc: error loading program: %s", err.Error())
	}
	doc, err := DocForPos(ctx, lprog, filename, offset)
	if err != nil && parseError != nil {
		fmt.Fprintln(os.Stderr, parseError)
	}
	return doc, err
}

// DocForPos attempts to get the documentation for an item given a filename and byte offset.
func DocForPos(ctxt *build.Context, lprog *loader.Program, filename string, offset int64) (*Doc, error) {
	tokFile := FileFromProgram(lprog, filename)
	if tokFile == nil {
		return nil, fmt.Errorf("gogetdoc: couldn't find %s in program", filename)
	}
	offPos := tokFile.Pos(int(offset))

	pkgInfo, nodes, _ := lprog.PathEnclosingInterval(offPos, offPos)
	for _, node := range nodes {
		switch i := node.(type) {
		case *ast.ImportSpec:
			abs, err := filepath.Abs(filename)
			if err != nil {
				return nil, err
			}
			return PackageDoc(ctxt, lprog.Fset, filepath.Dir(abs), ImportPath(i))
		case *ast.Ident:
			// if we can't find the object denoted by the identifier, keep searching)
			if obj := pkgInfo.ObjectOf(i); obj == nil {
				continue
			}
			return IdentDoc(ctxt, i, pkgInfo, lprog)
		default:
			break
		}
	}
	return nil, errors.New("gogetdoc: no documentation found")
}

// FileFromProgram attempts to locate a token.File from a loaded program.
func FileFromProgram(prog *loader.Program, name string) *token.File {
	for _, info := range prog.AllPackages {
		for _, astFile := range info.Files {
			tokFile := prog.Fset.File(astFile.Pos())
			if tokFile == nil {
				continue
			}
			tokName := tokFile.Name()
			if runtime.GOOS == "windows" {
				tokName = filepath.ToSlash(tokName)
				name = filepath.ToSlash(name)
			}
			if tokName == name {
				return tokFile
			}
			if sameFile(tokName, name) {
				return tokFile
			}
		}
	}
	return nil
}

func parsePos(p string) (filename string, offset int64, err error) {
	// foo.go:#123
	if p == "" {
		err = errors.New("missing required -pos flag")
		return
	}
	sep := strings.LastIndex(p, ":")
	// need at least 2 characters after the ':'
	// (the # sign and the offset)
	if sep == -1 || sep > len(p)-2 || p[sep+1] != '#' {
		err = fmt.Errorf("invalid option: -pos=%s", p)
		return
	}
	filename = p[:sep]
	offset, err = strconv.ParseInt(p[sep+2:], 10, 32)
	return
}

func sameFile(a, b string) bool {
	if filepath.Base(a) != filepath.Base(b) {
		// We only care about symlinks for the GOPATH itself. File
		// names need to match.
		return false
	}
	if ai, err := os.Stat(a); err == nil {
		if bi, err := os.Stat(b); err == nil {
			return os.SameFile(ai, bi)
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
)

var prog = filepath.Base(os.Args[0])

var loaders = map[string]string{}

func main() {
	log.SetFlags(0)
	log.SetPrefix(prog + ": ")

	if err := os.Setenv("GOOS", "nacl"); err != nil {
		log.Fatal(err)
	}

	arch := runtime.GOARCH
	if userArch := os.Getenv("GOARCH"); userArch != "" {
		// explicit in environment
		arch = userArch
	}

	// translate to nacl-specific architecture
	switch arch {
	case "amd64":
		arch = "amd64p32"
	}

	if err := os.Setenv("GOARCH", arch); err != nil {
		log.Fatal(err)
	}

	var bin string
	var err error
	if bin, err = exec.LookPath("go"); err != nil {
		log.Fatal(err)
	}
	args := os.Args
	args[0] = bin

	if len(args) >= 2 {
		switch args[1] {
		case "test", "run":
			var loader string
			switch arch {
			case "amd64p32":
				loader = "sel_ldr_x86_64"
			case "386":
				loader = "sel_ldr_x86_32"
			default:
				log.Fatalf("no NaCl loader for architecture: %v", arch)
			}
			args = append(args, "")
			copy(args[2+1:], args[2:])
			args[2] = "-exec=" + loader
		}
	}

	if err := syscall.Exec(bin, args, os.Environ()); err != nil {
		log.Fatal(err)
	}
	panic("still running after successful exec")
}
<commit_msg>Remove dead code<commit_after>package main

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
)

var prog = filepath.Base(os.Args[0])

func main() {
	log.SetFlags(0)
	log.SetPrefix(prog + ": ")

	if err := os.Setenv("GOOS", "nacl"); err != nil {
		log.Fatal(err)
	}

	arch := runtime.GOARCH
	if userArch := os.Getenv("GOARCH"); userArch != "" {
		// explicit in environment
		arch = userArch
	}

	// translate to nacl-specific architecture
	switch arch {
	case "amd64":
		arch = "amd64p32"
	}

	if err := os.Setenv("GOARCH", arch); err != nil {
		log.Fatal(err)
	}

	var bin string
	var err error
	if bin, err = exec.LookPath("go"); err != nil {
		log.Fatal(err)
	}
	args := os.Args
	args[0] = bin

	if len(args) >= 2 {
		switch args[1] {
		case "test", "run":
			var loader string
			switch arch {
			case "amd64p32":
				loader = "sel_ldr_x86_64"
			case "386":
				loader = "sel_ldr_x86_32"
			default:
				log.Fatalf("no NaCl loader for architecture: %v", arch)
			}
			args = append(args, "")
			copy(args[2+1:], args[2:])
			args[2] = "-exec=" + loader
		}
	}

	if err := syscall.Exec(bin, args, os.Environ()); err != nil {
		log.Fatal(err)
	}
	panic("still running after successful exec")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"errors"
	"github.com/rvillablanca/godiff/diff"
	"gopkg.in/alecthomas/kingpin.v2"
)

type diffconf struct {
	oldDir string
	newDir string
	destDir string
}

var (
	oldDir = kingpin.Arg("old", "Fuentes antiguos").Required().String()
	newDir = kingpin.Arg("new", "Fuentes nuevos").Required().String()
	destDir = kingpin.Arg("dest", "Destino del parche").Required().String()
)

func main() {
	kingpin.Parse()

	conf, err := generateAbsoluteDirectories()
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Versión anterior:", conf.oldDir)
	fmt.Println("Versión nueva:", conf.newDir)

	oldFiles := []string{}
	newFiles := []string{}

	fmt.Println("Buscando archivos en directorios...")
	oldFiles = diff.FindFilesIn(oldFiles, conf.oldDir)
	newFiles = diff.FindFilesIn(newFiles, conf.newDir)
}

func generateAbsoluteDirectories() (conf diffconf, err error) {
	dir1, err := filepath.Abs(*oldDir)
	if err != nil {
		err = errors.New("No fue posible verificar el directorio" + *oldDir)
		return
	}
	dir2, err := filepath.Abs(*newDir)
	if err != nil {
		err = errors.New("No fue posible verificar el directorio" + *newDir)
		return
	}
	valid, err := checkDirectories(dir1, dir2)
	if err != nil {
		err = errors.New("No fue posible verificar los directorios")
		return
	} else if !valid {
		err = errors.New("Todos los argumentos deben ser directorios")
		return
	}
	return diffconf{dir1, dir2, *destDir}, nil
}

func checkDirectories(dir1, dir2 string) (bool, error) {
	isDir1, err := checkDirectory(dir1)
	if err != nil {
		return false, err
	}
	isDir2, err := checkDirectory(dir2)
	if err != nil {
		return false, err
	}
	if !isDir1 || !isDir2 {
		return false, nil
	}
	return true, nil
}

func checkDirectory(dirname string) (result bool, err error) {
	f, err := os.Open(dirname)
	if err != nil {
		return
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return
	}

	result = fi.Mode().IsDir()
	return
}
<commit_msg>Avances<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"errors"
	"github.com/rvillablanca/godiff/diff"
	"gopkg.in/alecthomas/kingpin.v2"
)

type diffconf struct {
	oldDir string
	newDir string
	destDir string
}

var (
	oldDir = kingpin.Arg("old", "Fuentes antiguos").Required().String()
	newDir = kingpin.Arg("new", "Fuentes nuevos").Required().String()
	destDir = kingpin.Arg("dest", "Destino del parche").Required().String()
)

func main() {
	kingpin.Parse()

	conf, err := generateAbsoluteDirectories()
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Versión anterior:", conf.oldDir)
	fmt.Println("Versión nueva:", conf.newDir)

	oldFiles := []string{}
	newFiles := []string{}

	fmt.Println("Buscando archivos en directorios...")
	oldFiles = diff.FindFilesIn(oldFiles, conf.oldDir)
	newFiles = diff.FindFilesIn(newFiles, conf.newDir)

    toAdd := []string{}
    toRemove := []string{}
    toModify := []string{}

    for _, oldFile := range oldFiles {
		for _, newFile := range newFiles {

		}
	}

}

func generateAbsoluteDirectories() (conf diffconf, err error) {
	dir1, err := filepath.Abs(*oldDir)
	if err != nil {
		err = errors.New("No fue posible verificar el directorio" + *oldDir)
		return
	}
	dir2, err := filepath.Abs(*newDir)
	if err != nil {
		err = errors.New("No fue posible verificar el directorio" + *newDir)
		return
	}
	valid, err := checkDirectories(dir1, dir2)
	if err != nil {
		err = errors.New("No fue posible verificar los directorios")
		return
	} else if !valid {
		err = errors.New("Todos los argumentos deben ser directorios")
		return
	}
	return diffconf{dir1, dir2, *destDir}, nil
}

func checkDirectories(dir1, dir2 string) (bool, error) {
	isDir1, err := checkDirectory(dir1)
	if err != nil {
		return false, err
	}
	isDir2, err := checkDirectory(dir2)
	if err != nil {
		return false, err
	}
	if !isDir1 || !isDir2 {
		return false, nil
	}
	return true, nil
}

func checkDirectory(dirname string) (result bool, err error) {
	f, err := os.Open(dirname)
	if err != nil {
		return
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return
	}

	result = fi.Mode().IsDir()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/cloudfoundry-community/go-cfenv"
	"github.com/go-martini/martini"
	"github.com/jasonlvhit/gocron"
	"github.com/martini-contrib/render"
	"github.com/pivotal-pez/admin-portal/applications"
	"github.com/pivotal-pez/admin-portal/events"
	"github.com/pivotal-pez/admin-portal/users"
	cf "github.com/pivotal-pez/pezdispenser/cloudfoundryclient"
	"github.com/xchapter7x/cloudcontroller-client"
)

const (
	//SuccessStatus - success status of a call to the rest API
	SuccessStatus = 200
)

const (
	//AdminServiceName - userprovided service name for admin user of foundation
	AdminServiceName = "admin-user-information"
	//AdminURI - foundatio uri cups fieldname
	AdminURI = "cf-base-uri"
	//AdminUser - admin users cups fieldname
	AdminUser = "cf-user"
	//AdminPass - admin pass cups fieldname
	AdminPass = "cf-pass"
)

var (
	localCache ghettoCache
)

type ghettoCache struct {
	UserBlob   *users.UserAggregate
	AppsBlob   *applications.AppAggregate
	EventsBlob []cf.APIResponse
}

type cfAdminCreds struct {
	AdminURI  string
	AdminUser string
	AdminPass string
	LoginURI  string
	APIURI    string
}

type heritage struct {
	*ccclient.Client
	ccTarget string
}

func (s *heritage) CCTarget() string {
	return s.ccTarget
}

func main() {
	scheduler := gocron.NewScheduler()
	localLogger := log.New(os.Stdout, "[martini] ", 0)
	cfapp, _ := cfenv.Current()
	m := martini.Classic()
	m.Use(render.Renderer())
	m.Use(martini.Static("public"))
	m.Get("/", func(params martini.Params, log *log.Logger, r render.Render) {
		r.HTML(SuccessStatus, "index", nil)
	})

	m.Get("/v1/info/apps", func(log *log.Logger, r render.Render) {
		r.JSON(200, localCache.AppsBlob)
	})

	m.Get("/v1/info/users", func(log *log.Logger, r render.Render) {
		r.JSON(200, localCache.UserBlob)
	})

	m.Get("/v1/info/events", func(log *log.Logger, r render.Render) {
		r.JSON(200, localCache.EventsBlob)
	})

	scheduler.Every(10).Seconds().Do(func() {
		FetchEventsInfo(cfapp, localLogger)
	})
	scheduler.Every(1).Minute().Do(func() {
		FetchUserInfo(cfapp, localLogger)
	})
	scheduler.Every(1).Minute().Do(func() {
		FetchAppsInfo(cfapp, localLogger)
	})
	scheduler.RunAll()
	go func() {
		scheduler.Start()
	}()
	m.Run()
}

//FetchEventsInfo - function to poll for events information in the CCDB via REST API
func FetchEventsInfo(cfapp *cfenv.App, log *log.Logger) {
	log.Println("running FetchEventsInfo cron")
	heritageClient := getHeritageClient(cfapp)
	cfclient := cf.NewCloudFoundryClient(heritageClient, log)
	eventSearch := new(events.EventSearch).Init(cfclient)
	eventSearch.CompileRecentEvents()
	localCache.EventsBlob = eventSearch.EventsBlob
}

//FetchAppsInfo - function to poll for app information in the CCDB via REST API
func FetchAppsInfo(cfapp *cfenv.App, log *log.Logger) {
	log.Println("running FetchAppsInfo cron")
	heritageClient := getHeritageClient(cfapp)
	cfclient := cf.NewCloudFoundryClient(heritageClient, log)
	appSearch := new(applications.AppSearch).Init(cfclient)
	appSearch.CompileAllApps()
	localCache.AppsBlob = appSearch.AppStats
}

//FetchUserInfo - function to poll for user information in the UAA via REST API
func FetchUserInfo(cfapp *cfenv.App, log *log.Logger) {
	log.Printf("running FetchUserInfo cron")
	heritageClient := getHeritageClient(cfapp)
	cfclient := cf.NewCloudFoundryClient(heritageClient, log)
	userSearch := new(users.UserSearch).Init(cfclient)
	userList, _ := userSearch.List("", "")
	localCache.UserBlob = new(users.UserAggregate)
	localCache.UserBlob.Compile(userList)
}

func getHeritageClient(cfapp *cfenv.App) (heritageClient *heritage) {
	creds := getAdminCreds(cfapp)
	heritageClient = &heritage{
		Client:   ccclient.New(creds.LoginURI, creds.AdminUser, creds.AdminPass, new(http.Client)),
		ccTarget: creds.APIURI,
	}
	heritageClient.Login()
	return
}

func getAdminCreds(cfapp *cfenv.App) (adminCreds *cfAdminCreds) {
	if cfAdminService, err := cfapp.Services.WithName(AdminServiceName); err == nil {
		creds := cfAdminService.Credentials
		adminCreds = &cfAdminCreds{
			AdminURI:  creds[AdminURI],
			AdminUser: creds[AdminUser],
			AdminPass: creds[AdminPass],
			LoginURI:  fmt.Sprintf("https://%s.%s", "login", creds[AdminURI]),
			APIURI:    fmt.Sprintf("https://%s.%s", "api", creds[AdminURI]),
		}
	} else {
		panic(fmt.Sprintf("There is a problem with your required service binding %s: %s", AdminServiceName, err.Error()))
	}
	return
}
<commit_msg>Add type assertions to silence 1.6<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/cloudfoundry-community/go-cfenv"
	"github.com/go-martini/martini"
	"github.com/jasonlvhit/gocron"
	"github.com/martini-contrib/render"
	"github.com/pivotal-pez/admin-portal/applications"
	"github.com/pivotal-pez/admin-portal/events"
	"github.com/pivotal-pez/admin-portal/users"
	cf "github.com/pivotal-pez/pezdispenser/cloudfoundryclient"
	"github.com/xchapter7x/cloudcontroller-client"
)

const (
	//SuccessStatus - success status of a call to the rest API
	SuccessStatus = 200
)

const (
	//AdminServiceName - userprovided service name for admin user of foundation
	AdminServiceName = "admin-user-information"
	//AdminURI - foundatio uri cups fieldname
	AdminURI = "cf-base-uri"
	//AdminUser - admin users cups fieldname
	AdminUser = "cf-user"
	//AdminPass - admin pass cups fieldname
	AdminPass = "cf-pass"
)

var (
	localCache ghettoCache
)

type ghettoCache struct {
	UserBlob   *users.UserAggregate
	AppsBlob   *applications.AppAggregate
	EventsBlob []cf.APIResponse
}

type cfAdminCreds struct {
	AdminURI  string
	AdminUser string
	AdminPass string
	LoginURI  string
	APIURI    string
}

type heritage struct {
	*ccclient.Client
	ccTarget string
}

func (s *heritage) CCTarget() string {
	return s.ccTarget
}

func main() {
	scheduler := gocron.NewScheduler()
	localLogger := log.New(os.Stdout, "[martini] ", 0)
	cfapp, _ := cfenv.Current()
	m := martini.Classic()
	m.Use(render.Renderer())
	m.Use(martini.Static("public"))
	m.Get("/", func(params martini.Params, log *log.Logger, r render.Render) {
		r.HTML(SuccessStatus, "index", nil)
	})

	m.Get("/v1/info/apps", func(log *log.Logger, r render.Render) {
		r.JSON(200, localCache.AppsBlob)
	})

	m.Get("/v1/info/users", func(log *log.Logger, r render.Render) {
		r.JSON(200, localCache.UserBlob)
	})

	m.Get("/v1/info/events", func(log *log.Logger, r render.Render) {
		r.JSON(200, localCache.EventsBlob)
	})

	scheduler.Every(10).Seconds().Do(func() {
		FetchEventsInfo(cfapp, localLogger)
	})
	scheduler.Every(1).Minute().Do(func() {
		FetchUserInfo(cfapp, localLogger)
	})
	scheduler.Every(1).Minute().Do(func() {
		FetchAppsInfo(cfapp, localLogger)
	})
	scheduler.RunAll()
	go func() {
		scheduler.Start()
	}()
	m.Run()
}

//FetchEventsInfo - function to poll for events information in the CCDB via REST API
func FetchEventsInfo(cfapp *cfenv.App, log *log.Logger) {
	log.Println("running FetchEventsInfo cron")
	heritageClient := getHeritageClient(cfapp)
	cfclient := cf.NewCloudFoundryClient(heritageClient, log)
	eventSearch := new(events.EventSearch).Init(cfclient)
	eventSearch.CompileRecentEvents()
	localCache.EventsBlob = eventSearch.EventsBlob
}

//FetchAppsInfo - function to poll for app information in the CCDB via REST API
func FetchAppsInfo(cfapp *cfenv.App, log *log.Logger) {
	log.Println("running FetchAppsInfo cron")
	heritageClient := getHeritageClient(cfapp)
	cfclient := cf.NewCloudFoundryClient(heritageClient, log)
	appSearch := new(applications.AppSearch).Init(cfclient)
	appSearch.CompileAllApps()
	localCache.AppsBlob = appSearch.AppStats
}

//FetchUserInfo - function to poll for user information in the UAA via REST API
func FetchUserInfo(cfapp *cfenv.App, log *log.Logger) {
	log.Printf("running FetchUserInfo cron")
	heritageClient := getHeritageClient(cfapp)
	cfclient := cf.NewCloudFoundryClient(heritageClient, log)
	userSearch := new(users.UserSearch).Init(cfclient)
	userList, _ := userSearch.List("", "")
	localCache.UserBlob = new(users.UserAggregate)
	localCache.UserBlob.Compile(userList)
}

func getHeritageClient(cfapp *cfenv.App) (heritageClient *heritage) {
	creds := getAdminCreds(cfapp)
	heritageClient = &heritage{
		Client:   ccclient.New(creds.LoginURI, creds.AdminUser, creds.AdminPass, new(http.Client)),
		ccTarget: creds.APIURI,
	}
	heritageClient.Login()
	return
}

func getAdminCreds(cfapp *cfenv.App) (adminCreds *cfAdminCreds) {
	if cfAdminService, err := cfapp.Services.WithName(AdminServiceName); err == nil {
		creds := cfAdminService.Credentials
		adminCreds = &cfAdminCreds{
			AdminURI:  creds[AdminURI].(string),
			AdminUser: creds[AdminUser].(string),
			AdminPass: creds[AdminPass].(string),
			LoginURI:  fmt.Sprintf("https://%s.%s", "login", creds[AdminURI]),
			APIURI:    fmt.Sprintf("https://%s.%s", "api", creds[AdminURI]),
		}
	} else {
		panic(fmt.Sprintf("There is a problem with your required service binding %s: %s", AdminServiceName, err.Error()))
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/doxxan/appindicator"
	"github.com/doxxan/appindicator/gtk-extensions/gotk3"
	"github.com/doxxan/gotk3/gtk"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type NetState int

const (
	StateDisconnected NetState = iota
	StateConnected
)

const (
	SizeKB = 1024
	SizeMB = SizeKB * 1024
	SizeGB = SizeMB * 1024
	SizeTB = SizeGB * 1024
	SizePB = SizeTB * 1024
)

var StrengthTable = []string{"0", "0", "25", "50", "75", "100"}
var RadioTable = []string{"-", "LTE", "EVD0", "CDMA1x", "WCDMA", "GSM", "HSUPA", "HSPA+", "DC-HSPA+", "EDGE", "GPRS"}

type NetworkStatus struct {
	Strength      int
	State         NetState
	Radio         int
	Network       string
	PINStatus     int
	LastTime      int
	ConnectedTime int
	CurrentUp     int
	CurrentDown   int
	TotalUp       int
	TotalDown     int
	ServiceStatus int
}

var modemOnline = false
var networkStatus = NetworkStatus{}

func bytesToHumanReadable(bytes int) string {
	switch {
	case bytes >= SizePB:
		return fmt.Sprintf("%.2fPB", float32(bytes)/float32(SizePB))
	case bytes >= SizeTB:
		return fmt.Sprintf("%.2fTB", float32(bytes)/float32(SizeTB))
	case bytes >= SizeGB:
		return fmt.Sprintf("%.2fGB", float32(bytes)/float32(SizeGB))
	case bytes >= SizeMB:
		return fmt.Sprintf("%.2fMB", float32(bytes)/float32(SizeMB))
	case bytes >= SizeKB:
		return fmt.Sprintf("%.2fKB", float32(bytes)/float32(SizeKB))
	default:
		return fmt.Sprintf("%dB", bytes)
	}
}

//5;2;0;9;Telenor SE;1;;;;797;4044181;1603028;59801;1274938731;83429417;2;3608;4848;
func parseStatusString(data string) {
	status := strings.Split(data, ";")

	networkStatus.Strength, _ = strconv.Atoi(status[0])
	netState, _ := strconv.Atoi(status[1])
	if netState == 2 {
		networkStatus.State = StateConnected
	} else {
		networkStatus.State = StateDisconnected
	}
	networkStatus.Radio, _ = strconv.Atoi(status[3])
	networkStatus.Network = status[4]
	networkStatus.PINStatus, _ = strconv.Atoi(status[5])
	networkStatus.LastTime, _ = strconv.Atoi(status[7])
	networkStatus.ConnectedTime, _ = strconv.Atoi(status[9])
	networkStatus.CurrentDown, _ = strconv.Atoi(status[10])
	networkStatus.CurrentUp, _ = strconv.Atoi(status[11])
	networkStatus.TotalDown, _ = strconv.Atoi(status[13])
	networkStatus.TotalUp, _ = strconv.Atoi(status[14])
	networkStatus.ServiceStatus, _ = strconv.Atoi(status[15])

	if networkStatus.Radio > -1 {
		fmt.Printf("Radio: %s\n", RadioTable[networkStatus.Radio])
	}
	fmt.Printf("Network: %s\n", networkStatus.Network)
	fmt.Printf("PINStatus: %d\n", networkStatus.PINStatus)
	//fmt.Printf("LastTime: %s\n", time.Unix(int64(networkStatus.LastTime), 0).String())
	hour := networkStatus.ConnectedTime / 3000
	min := networkStatus.ConnectedTime / 60
	sec := networkStatus.ConnectedTime - (hour * 360) - (min * 60)
	fmt.Printf("ConnectedTime: %d:%d:%d\n", hour, min, sec)
	fmt.Printf("CurrentDown: %s\n", bytesToHumanReadable(networkStatus.CurrentDown))
	fmt.Printf("CurrentUp: %s\n", bytesToHumanReadable(networkStatus.CurrentUp))
	fmt.Printf("TotalDown: %s\n", bytesToHumanReadable(networkStatus.TotalDown))
	fmt.Printf("TotalUp: %s\n", bytesToHumanReadable(networkStatus.TotalUp))
	fmt.Printf("ServiceStatus: %d\n", networkStatus.ServiceStatus)
}

func pollStatus(indicator *gotk3.AppIndicatorGotk3, menuCurrent, menuTotal *gtk.MenuItem) {
	ticker := time.Tick(time.Second)

	either := false
	for _ = range ticker {
		req := fmt.Sprintf("http://192.168.0.1/goform/status_update?rd=%f", rand.Float32())
		if response, err := http.Get(req); err != nil {
			if modemOnline {
				modemOnline = false
				indicator.SetIcon("network-error", "Modem offline")
			}
			fmt.Println("Could not connect, modem not available?")
		} else {
			defer response.Body.Close()
			if data, err := ioutil.ReadAll(response.Body); err != nil {
				fmt.Println("Error occurred while reading data.")
			} else {
				if !modemOnline {
					modemOnline = true
				}
				fmt.Println(string(data))
				parseStatusString(string(data))

				strength := StrengthTable[networkStatus.Strength]
				indicator.SetIcon("nm-signal-"+strength, "Modem online")

				if either {
					currentStr := fmt.Sprintf("Current Transfer - U:%s | D:%s", bytesToHumanReadable(networkStatus.CurrentUp), bytesToHumanReadable(networkStatus.CurrentDown))
					menuCurrent.SetLabel(currentStr)
				} else {
					totalStr := fmt.Sprintf("Total Transfer - U:%s | D:%s", bytesToHumanReadable(networkStatus.TotalUp), bytesToHumanReadable(networkStatus.TotalDown))
					menuTotal.SetLabel(totalStr)
				}
				either = !either
			}
		}
	}
}

func main() {
	gtk.Init(nil)

	menu, _ := gtk.MenuNew()

	menuCurrentTransfer, _ := gtk.MenuItemNewWithLabel("Current Transfer - ##")
	menuCurrentTransfer.Show()
	menu.Append(menuCurrentTransfer)

	menuTotalTransfer, _ := gtk.MenuItemNewWithLabel("Total Transfer - ##")
	menuTotalTransfer.Show()
	menu.Append(menuTotalTransfer)

	menuQuit, _ := gtk.MenuItemNewWithLabel("Quit")
	menuQuit.Connect("activate", func() {
		gtk.MainQuit()
	})
	menuQuit.Show()
	menu.Append(menuQuit)

	indicator := gotk3.NewAppIndicator("bb-4g-modem-indicator", "nm-signal-0", appindicator.CategoryCommunications)
	indicator.SetStatus(appindicator.StatusActive)
	indicator.SetMenu(menu)

	go pollStatus(indicator, menuCurrentTransfer, menuTotalTransfer)

	gtk.Main()
}
<commit_msg>Missing fields.<commit_after>package main

import (
	"fmt"
	"github.com/doxxan/appindicator"
	"github.com/doxxan/appindicator/gtk-extensions/gotk3"
	"github.com/doxxan/gotk3/gtk"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type NetState int

const (
	StateDisconnected NetState = iota
	StateConnected
)

const (
	SizeKB = 1024
	SizeMB = SizeKB * 1024
	SizeGB = SizeMB * 1024
	SizeTB = SizeGB * 1024
	SizePB = SizeTB * 1024
)

var StrengthTable = []string{"0", "0", "25", "50", "75", "100"}
var RadioTable = []string{"-", "LTE", "EVD0", "CDMA1x", "WCDMA", "GSM", "HSUPA", "HSPA+", "DC-HSPA+", "EDGE", "GPRS"}

type NetworkStatus struct {
	Strength      int
	State         NetState
	RoamStatus    int
	Radio         int
	Network       string
	PINStatus     int
	ShowUnreadSMS int
	LastTime      int
	GetUnreadSMS  int
	ConnectedTime int
	CurrentUp     int
	CurrentDown   int
	TotalUp       int
	TotalDown     int
	ServiceStatus int
}

var modemOnline = false
var networkStatus = NetworkStatus{}

func bytesToHumanReadable(bytes int) string {
	switch {
	case bytes >= SizePB:
		return fmt.Sprintf("%.2fPB", float32(bytes)/float32(SizePB))
	case bytes >= SizeTB:
		return fmt.Sprintf("%.2fTB", float32(bytes)/float32(SizeTB))
	case bytes >= SizeGB:
		return fmt.Sprintf("%.2fGB", float32(bytes)/float32(SizeGB))
	case bytes >= SizeMB:
		return fmt.Sprintf("%.2fMB", float32(bytes)/float32(SizeMB))
	case bytes >= SizeKB:
		return fmt.Sprintf("%.2fKB", float32(bytes)/float32(SizeKB))
	default:
		return fmt.Sprintf("%dB", bytes)
	}
}

//5;2;0;9;Telenor SE;1;;;;797;4044181;1603028;59801;1274938731;83429417;2;3608;4848;
func parseStatusString(data string) {
	status := strings.Split(data, ";")

	networkStatus.Strength, _ = strconv.Atoi(status[0])
	netState, _ := strconv.Atoi(status[1])
	if netState == 2 {
		networkStatus.State = StateConnected
	} else {
		networkStatus.State = StateDisconnected
	}
	networkStatus.RoamStatus, _ = strconv.Atoi(status[2])
	networkStatus.Radio, _ = strconv.Atoi(status[3])
	networkStatus.Network = status[4]
	networkStatus.PINStatus, _ = strconv.Atoi(status[5])
	networkStatus.ShowUnreadSMS, _ = strconv.Atoi(status[6])
	networkStatus.LastTime, _ = strconv.Atoi(status[7])
	networkStatus.GetUnreadSMS, _ = strconv.Atoi(status[8])
	networkStatus.ConnectedTime, _ = strconv.Atoi(status[9])
	networkStatus.CurrentDown, _ = strconv.Atoi(status[10])
	networkStatus.CurrentUp, _ = strconv.Atoi(status[11])
	networkStatus.TotalDown, _ = strconv.Atoi(status[13])
	networkStatus.TotalUp, _ = strconv.Atoi(status[14])
	networkStatus.ServiceStatus, _ = strconv.Atoi(status[15])

	if networkStatus.Radio > -1 {
		fmt.Printf("Radio: %s\n", RadioTable[networkStatus.Radio])
	}
	fmt.Printf("Network: %s\n", networkStatus.Network)
	fmt.Printf("PINStatus: %d\n", networkStatus.PINStatus)
	//fmt.Printf("LastTime: %s\n", time.Unix(int64(networkStatus.LastTime), 0).String())
	hour := networkStatus.ConnectedTime / 3000
	min := networkStatus.ConnectedTime / 60
	sec := networkStatus.ConnectedTime - (hour * 360) - (min * 60)
	fmt.Printf("ConnectedTime: %d:%d:%d\n", hour, min, sec)
	fmt.Printf("CurrentDown: %s\n", bytesToHumanReadable(networkStatus.CurrentDown))
	fmt.Printf("CurrentUp: %s\n", bytesToHumanReadable(networkStatus.CurrentUp))
	fmt.Printf("TotalDown: %s\n", bytesToHumanReadable(networkStatus.TotalDown))
	fmt.Printf("TotalUp: %s\n", bytesToHumanReadable(networkStatus.TotalUp))
	fmt.Printf("ServiceStatus: %d\n", networkStatus.ServiceStatus)
}

func pollStatus(indicator *gotk3.AppIndicatorGotk3, menuCurrent, menuTotal *gtk.MenuItem) {
	ticker := time.Tick(time.Second)

	either := false
	for _ = range ticker {
		req := fmt.Sprintf("http://192.168.0.1/goform/status_update?rd=%f", rand.Float32())
		if response, err := http.Get(req); err != nil {
			if modemOnline {
				modemOnline = false
				indicator.SetIcon("network-error", "Modem offline")
			}
			fmt.Println("Could not connect, modem not available?")
		} else {
			defer response.Body.Close()
			if data, err := ioutil.ReadAll(response.Body); err != nil {
				fmt.Println("Error occurred while reading data.")
			} else {
				if !modemOnline {
					modemOnline = true
				}
				fmt.Println(string(data))
				parseStatusString(string(data))

				strength := StrengthTable[networkStatus.Strength]
				indicator.SetIcon("nm-signal-"+strength, "Modem online")

				if either {
					currentStr := fmt.Sprintf("Current Transfer - U:%s | D:%s", bytesToHumanReadable(networkStatus.CurrentUp), bytesToHumanReadable(networkStatus.CurrentDown))
					menuCurrent.SetLabel(currentStr)
				} else {
					totalStr := fmt.Sprintf("Total Transfer - U:%s | D:%s", bytesToHumanReadable(networkStatus.TotalUp), bytesToHumanReadable(networkStatus.TotalDown))
					menuTotal.SetLabel(totalStr)
				}
				either = !either
			}
		}
	}
}

func main() {
	gtk.Init(nil)

	menu, _ := gtk.MenuNew()

	menuCurrentTransfer, _ := gtk.MenuItemNewWithLabel("Current Transfer - ##")
	menuCurrentTransfer.Show()
	menu.Append(menuCurrentTransfer)

	menuTotalTransfer, _ := gtk.MenuItemNewWithLabel("Total Transfer - ##")
	menuTotalTransfer.Show()
	menu.Append(menuTotalTransfer)

	menuQuit, _ := gtk.MenuItemNewWithLabel("Quit")
	menuQuit.Connect("activate", func() {
		gtk.MainQuit()
	})
	menuQuit.Show()
	menu.Append(menuQuit)

	indicator := gotk3.NewAppIndicator("bb-4g-modem-indicator", "nm-signal-0", appindicator.CategoryCommunications)
	indicator.SetStatus(appindicator.StatusActive)
	indicator.SetMenu(menu)

	go pollStatus(indicator, menuCurrentTransfer, menuTotalTransfer)

	gtk.Main()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
)

var (
	envPath  = os.Getenv("PATH")
	envLevel = os.Getenv("LF_LEVEL")
)

var (
	gClientID      int
	gHostname      string
	gLastDirPath   string
	gSelectionPath string
	gSocketProt    string
	gSocketPath    string
	gLogPath       string
	gServerLogPath string
	gSelect        string
	gCommand       string
	gVersion       string
)

func init() {
	h, err := os.Hostname()
	if err != nil {
		log.Printf("hostname: %s", err)
	}
	gHostname = h

	if envLevel == "" {
		envLevel = "0"
	}
}

func exportEnvVars() {
	os.Setenv("id", strconv.Itoa(gClientID))

	os.Setenv("OPENER", envOpener)
	os.Setenv("EDITOR", envEditor)
	os.Setenv("PAGER", envPager)
	os.Setenv("SHELL", envShell)
	os.Setenv("TCELL_TRUECOLOR", envTcellTruecolor)

	level, err := strconv.Atoi(envLevel)
	if err != nil {
		log.Printf("reading lf level: %s", err)
	}

	level++

	os.Setenv("LF_LEVEL", strconv.Itoa(level))
}

func startServer() {
	cmd := detachedCommand(os.Args[0], "-server")
	cmd.Dir = gUser.HomeDir
	if err := cmd.Start(); err != nil {
		log.Printf("starting server: %s", err)
	}
}

func checkServer() {
	if gSocketProt == "unix" {
		if _, err := os.Stat(gSocketPath); os.IsNotExist(err) {
			startServer()
		} else if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			os.Remove(gSocketPath)
			startServer()
		}
	} else {
		if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			startServer()
		}
	}
}

func main() {
	showDoc := flag.Bool(
		"doc",
		false,
		"show documentation")

	showVersion := flag.Bool(
		"version",
		false,
		"show version")

	remoteCmd := flag.String(
		"remote",
		"",
		"send remote command to server")

	serverMode := flag.Bool(
		"server",
		false,
		"start server (automatic)")

	cpuprofile := flag.String(
		"cpuprofile",
		"",
		"path to the file to write the CPU profile")

	memprofile := flag.String(
		"memprofile",
		"",
		"path to the file to write the memory profile")

	flag.StringVar(&gLastDirPath,
		"last-dir-path",
		"",
		"path to the file to write the last dir on exit (to use for cd)")

	flag.StringVar(&gSelectionPath,
		"selection-path",
		"",
		"path to the file to write selected files on open (to use as open file dialog)")

	flag.StringVar(&gCommand,
		"command",
		"",
		"command to execute on client initialization")

	flag.Parse()

	gSocketProt = gDefaultSocketProt
	gSocketPath = gDefaultSocketPath

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalf("could not create CPU profile: %s", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatalf("could not start CPU profile: %s", err)
		}
		defer pprof.StopCPUProfile()
	}

	switch {
	case *showDoc:
		fmt.Print(genDocString)
	case *showVersion:
		fmt.Println(gVersion)
	case *remoteCmd != "":
		if err := remote(*remoteCmd); err != nil {
			log.Fatalf("remote command: %s", err)
		}
	case *serverMode:
		gServerLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.server.log", gUser.Username))
		serve()
	default:
		checkServer()

		gClientID = 1000
		gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		for _, err := os.Stat(gLogPath); !os.IsNotExist(err); _, err = os.Stat(gLogPath) {
			gClientID++
			gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		}

		switch flag.NArg() {
		case 0:
			_, err := os.Getwd()
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err)
				os.Exit(2)
			}
		case 1:
			gSelect = flag.Arg(0)
		default:
			fmt.Fprintf(os.Stderr, "only single file or directory is allowed\n")
			os.Exit(2)
		}

		exportEnvVars()

		run()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal("could not create memory profile: ", err)
		}
		runtime.GC()
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("could not write memory profile: ", err)
		}
		f.Close()
	}
}
<commit_msg>A more robust server starting mechanism (#463)<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
)

var (
	envPath  = os.Getenv("PATH")
	envLevel = os.Getenv("LF_LEVEL")
)

var (
	gClientID      int
	gHostname      string
	gLastDirPath   string
	gSelectionPath string
	gSocketProt    string
	gSocketPath    string
	gLogPath       string
	gServerLogPath string
	gSelect        string
	gCommand       string
	gVersion       string
)

func init() {
	h, err := os.Hostname()
	if err != nil {
		log.Printf("hostname: %s", err)
	}
	gHostname = h

	if envLevel == "" {
		envLevel = "0"
	}
}

func exportEnvVars() {
	os.Setenv("id", strconv.Itoa(gClientID))

	os.Setenv("OPENER", envOpener)
	os.Setenv("EDITOR", envEditor)
	os.Setenv("PAGER", envPager)
	os.Setenv("SHELL", envShell)
	os.Setenv("TCELL_TRUECOLOR", envTcellTruecolor)

	level, err := strconv.Atoi(envLevel)
	if err != nil {
		log.Printf("reading lf level: %s", err)
	}

	level++

	os.Setenv("LF_LEVEL", strconv.Itoa(level))
}

func startServer() {
	cmd := detachedCommand(os.Args[0], "-server")
	if err := cmd.Start(); err != nil {
		log.Printf("starting server: %s", err)
	}
}

func checkServer() {
	if gSocketProt == "unix" {
		if _, err := os.Stat(gSocketPath); os.IsNotExist(err) {
			startServer()
		} else if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			os.Remove(gSocketPath)
			startServer()
		}
	} else {
		if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			startServer()
		}
	}
}

func main() {
	showDoc := flag.Bool(
		"doc",
		false,
		"show documentation")

	showVersion := flag.Bool(
		"version",
		false,
		"show version")

	remoteCmd := flag.String(
		"remote",
		"",
		"send remote command to server")

	serverMode := flag.Bool(
		"server",
		false,
		"start server (automatic)")

	cpuprofile := flag.String(
		"cpuprofile",
		"",
		"path to the file to write the CPU profile")

	memprofile := flag.String(
		"memprofile",
		"",
		"path to the file to write the memory profile")

	flag.StringVar(&gLastDirPath,
		"last-dir-path",
		"",
		"path to the file to write the last dir on exit (to use for cd)")

	flag.StringVar(&gSelectionPath,
		"selection-path",
		"",
		"path to the file to write selected files on open (to use as open file dialog)")

	flag.StringVar(&gCommand,
		"command",
		"",
		"command to execute on client initialization")

	flag.Parse()

	gSocketProt = gDefaultSocketProt
	gSocketPath = gDefaultSocketPath

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalf("could not create CPU profile: %s", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatalf("could not start CPU profile: %s", err)
		}
		defer pprof.StopCPUProfile()
	}

	switch {
	case *showDoc:
		fmt.Print(genDocString)
	case *showVersion:
		fmt.Println(gVersion)
	case *remoteCmd != "":
		if err := remote(*remoteCmd); err != nil {
			log.Fatalf("remote command: %s", err)
		}
	case *serverMode:
		os.Chdir(gUser.HomeDir)
		gServerLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.server.log", gUser.Username))
		serve()
	default:
		checkServer()

		gClientID = 1000
		gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		for _, err := os.Stat(gLogPath); !os.IsNotExist(err); _, err = os.Stat(gLogPath) {
			gClientID++
			gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		}

		switch flag.NArg() {
		case 0:
			_, err := os.Getwd()
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err)
				os.Exit(2)
			}
		case 1:
			gSelect = flag.Arg(0)
		default:
			fmt.Fprintf(os.Stderr, "only single file or directory is allowed\n")
			os.Exit(2)
		}

		exportEnvVars()

		run()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal("could not create memory profile: ", err)
		}
		runtime.GC()
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("could not write memory profile: ", err)
		}
		f.Close()
	}
}
<|endoftext|>


<commit_before>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"github.com/crowdmob/goamz/s3"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type Storer struct {
	StorageChannel      chan *DPNResult
	CleanupChannel      chan *DPNResult
	BagCreateChannel    chan *DPNResult
	PostProcessChannel  chan *DPNResult
	ProcUtil            *bagman.ProcessUtil
	DPNConfig           *DPNConfig
	LocalRESTClient     *DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

func NewStorer(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Storer, error) {
	// Set up a DPN REST client that talks to our local DPN REST service.
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}

	storer := &Storer{
		LocalRESTClient: localClient,
		ProcUtil: procUtil,
		DPNConfig: dpnConfig,
	}
	workerBufferSize := procUtil.Config.DPNStoreWorker.Workers * 10
	storer.StorageChannel = make(chan *DPNResult, workerBufferSize)
	storer.BagCreateChannel = make(chan *DPNResult, workerBufferSize)
	storer.CleanupChannel = make(chan *DPNResult, workerBufferSize)
	storer.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNStoreWorker.Workers; i++ {
		go storer.createBagRecord()
		go storer.cleanup()
		go storer.postProcess()
	}
	for i := 0; i < procUtil.Config.DPNStoreWorker.NetworkConnections; i++ {
		go storer.store()
	}
	return storer, nil
}

func (storer *Storer) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	result := &DPNResult{}
	err := json.Unmarshal(message.Body, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	if result.BagMd5Digest == "" || result.BagSha256Digest == "" {
		errMsg := "Bag cannot be stored because DPNResult is missing either md5 or sha256 checksum."
		storer.ProcUtil.MessageLog.Error(errMsg)
		result.ErrorMessage = errMsg
		SendToTroubleQueue(result, storer.ProcUtil)
		message.Finish()
		return fmt.Errorf(errMsg)
	}
	result.NsqMessage = message
	result.Stage = STAGE_STORE
	bagIdentifier := result.BagIdentifier
	if bagIdentifier == "" {
		bagIdentifier = "DPN Replication Bag"
	}
	storer.ProcUtil.MessageLog.Info("Putting %s into the storage queue (%s)",
		result.TarFilePath(), bagIdentifier)
	storer.StorageChannel <- result
	return nil
}

func (storer *Storer) store() {
	for result := range storer.StorageChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// By the time we get our hands on this replication request,
		// it may be many hours old. Fetch the request again from the
		// originating node and make sure it hasn't been cancelled.
		if result.TransferRequest != nil {
			storer.refetchReplicationRequest(result)
		}
		if result.ErrorMessage != "" {
			storer.PostProcessChannel <- result
			continue
		}
		// Remember that TransferRequest will be nil
		// for bags we built ourselves.
		if result.TransferRequest != nil && result.TransferRequest.Status == "Cancelled" {
			result.Stage = STAGE_CANCELLED
			result.Retry = false
			storer.PostProcessChannel <- result
			continue
		}

		err := os.MkdirAll(filepath.Dir(result.TarFilePath()), 0755)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error creating directory '%s': %v",
				filepath.Dir(result.TarFilePath()), err)
			storer.PostProcessChannel <- result
			continue
		}

		// Now we'll open a file reader and stream the tar file
		// up to S3.
		reader, err := os.Open(result.TarFilePath())
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		options, err := storer.GetS3Options(result)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error generating S3 options: %v", err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}

		bagUUID := ""
		if result.DPNBag != nil {
			bagUUID = result.DPNBag.UUID
		} else {
			bagUUID = result.PackageResult.BagBuilder.UUID
		}
		fileName := fmt.Sprintf("%s.tar", bagUUID)
		url := ""
		if fileInfo.Size() > bagman.S3_LARGE_FILE {
			url, err = storer.ProcUtil.S3Client.SaveLargeFileToS3(
				storer.ProcUtil.Config.DPNPreservationBucket,
				fileName,
				"application/x-tar",
				reader,
				fileInfo.Size(),
				options,
				bagman.S3_CHUNK_SIZE)
		} else {
			url, err = storer.ProcUtil.S3Client.SaveToS3(
				storer.ProcUtil.Config.DPNPreservationBucket,
				fileName,
				"application/x-tar",
				reader,
				fileInfo.Size(),
				options)
		}

		// Close the reader
		reader.Close()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error saving file to S3/Glacier: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		// Record where we stored the file
		result.StorageURL = url

		// Update the transfer request, if there is one.
		if result.TransferRequest != nil {
			result.TransferRequest.Status = "Stored"
		}
		if result.ErrorMessage != "" {
			result.ErrorMessage = fmt.Sprintf("Bag was stored, but we couldn't " +
				"update the replication request on the originating node: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// This channel really only applies to bags we created
		// at our own node. (Not replication requests.)
		storer.BagCreateChannel <- result
	}
}

// Get the replication request from the originating node and
// make sure it's still valid.
func (storer *Storer) refetchReplicationRequest(result *DPNResult) {
	storer.makeReplicationRequest(result, "GET")
}

func (storer *Storer) makeReplicationRequest(result *DPNResult, whatKind string) {

	if result.TransferRequest == nil {
		// This is a local bag and there is no replication request.
		storer.ProcUtil.MessageLog.Error("makeReplicationRequest was called for bag %s (%s), " +
			"which has no associated replication request. Is this a local bag?",
			result.BagIdentifier, result.PackageResult.BagBuilder.UUID)
		return
	}

	// Get a REST client that can talk the remote node.
	// This client will be configured with the correct URL
	// and API token to talk to the node that issues the
	// replication request.
	remoteClient, err := storer.LocalRESTClient.GetRemoteClient(
		result.TransferRequest.FromNode,  // the node we want to talk to
		storer.DPNConfig,
		storer.ProcUtil.MessageLog)
	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not create a REST client to connect to node %s: %v",
			result.TransferRequest.FromNode, err)
		return
	}

	var xferRequest *DPNReplicationTransfer
	if whatKind == "GET" {
		storer.ProcUtil.MessageLog.Debug("Replication request id: %s, " +
			"FromNode: %s, ToNode: %s", result.TransferRequest.ReplicationId,
			result.TransferRequest.FromNode, result.TransferRequest.ToNode)
		xferRequest, err = remoteClient.ReplicationTransferGet(result.TransferRequest.ReplicationId)
	} else if whatKind == "POST" {
		xferRequest, err = remoteClient.ReplicationTransferUpdate(result.TransferRequest)
	} else {
		panic("makeReplicationRequest doesn't understand that kind of request")
	}

	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not get replication record %s from node %s: %v",
			result.TransferRequest.ReplicationId, result.TransferRequest.FromNode, err)
		return
	}
	result.TransferRequest = xferRequest

}

func (storer *Storer) createBagRecord() {
	for result := range storer.BagCreateChannel {
		// If result has a local identifier, it's a bag we created
		// here at the local node (as opposed to a bag we're just
		// replicating to fulfill a transfer request). Since we
		// created this bag here, we need to create a Bag record
		// for it in DPN. We could also check result.PackageResult
		// to determine if bag was made here. That will be non-nil
		// for bags we built, nil for bags built elsewhere.
		bagWasCreatedHere := result.BagIdentifier != ""
		bagStoredSuccessfully := (result.ErrorMessage == "" && result.StorageURL != "")
		if bagWasCreatedHere && bagStoredSuccessfully {
			storer.ProcUtil.MessageLog.Debug("Creating bag record for %s with md5 %s and sha256 %s",
				result.BagIdentifier, result.BagMd5Digest, result.BagSha256Digest)
			fixity := &DPNFixity{
				Sha256: result.BagSha256Digest,
			}
			fileInfo, err := os.Stat(result.TarFilePath())
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Cannot stat %s to get file size: %v",
					result.TarFilePath(), err)
				storer.CleanupChannel <- result
				continue
			}
			newBag := &DPNBag{
				UUID: result.PackageResult.BagBuilder.UUID,
				LocalId: result.BagIdentifier,
				Size: uint64(fileInfo.Size()),
				FirstVersionUUID: result.PackageResult.BagBuilder.UUID,
				Version: 1,
				IngestNode: storer.DPNConfig.LocalNode,
				AdminNode: storer.DPNConfig.LocalNode,
				BagType: "D",
				// No spec yet on how to specify rights; cannot be nil
				Rights: make([]string, 0),
				// No spec yet on how to specify interpretive; cannot be nil
				Interpretive: make([]string, 0),
				ReplicatingNodes: []string{"aptrust"},
				Fixities: fixity,
			}
			result.DPNBag = newBag
		}
		storer.CleanupChannel <- result
	}
}

func (storer *Storer) cleanup() {
	for result := range storer.CleanupChannel {
		thisIsNotATest := (result.NsqMessage != nil)
		storageSucceeded := (result.ErrorMessage == "" && result.StorageURL != "")
		if storageSucceeded && thisIsNotATest {
			err := os.Remove(result.TarFilePath())
			if err != nil {
				storer.ProcUtil.MessageLog.Warning("Error cleaning up %s: %v",
					result.TarFilePath(), err)
			} else {
				storer.ProcUtil.MessageLog.Info(
					"After successful upload, deleted local DPN bag at %s",
					result.TarFilePath())
			}
		}
		storer.PostProcessChannel <- result
	}
}

func (storer *Storer) postProcess() {
	for result := range storer.PostProcessChannel {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.DPNBag.UUID
		}
		if result.ErrorMessage == "" && result.StorageURL != "" {
			// SUCCESS :)
			storer.ProcUtil.MessageLog.Info("Bag %s successfully stored at %s",
				bagIdentifier, result.StorageURL)
			storer.ProcUtil.IncrementSucceeded()
			// Send to queue for recording in Fluctus and/or DPN REST
			if result.NsqMessage != nil {
				result.NsqMessage.Finish()
				SendToRecordQueue(result, storer.ProcUtil)
			}
		} else {
			// FAILURE :(
			storer.ProcUtil.MessageLog.Error(result.ErrorMessage)
			storer.ProcUtil.IncrementFailed()
			// Item failed after max attempts. Put in trouble queue
			// for admin review.
			if result.NsqMessage != nil {
				if result.NsqMessage.Attempts >= uint16(storer.ProcUtil.Config.DPNStoreWorker.MaxAttempts) {
					// No more retries
					result.NsqMessage.Finish()
					SendToTroubleQueue(result, storer.ProcUtil)
				} else {
					storer.ProcUtil.MessageLog.Info("Requeuing %s (%s)",
						bagIdentifier, result.TarFilePath())
					result.NsqMessage.Requeue(1 * time.Minute)
				}
			}
		}
		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			storer.WaitGroup.Done()
		}
	}
}

func (storer *Storer) GetS3Options(result *DPNResult) (s3.Options, error) {
	// Prepare metadata for save to S3
	s3Metadata := make(map[string][]string)
	if result.BagIdentifier != "" {
		s3Metadata["aptrust-bag"] = []string{result.BagIdentifier}
	}
	// Save to S3 with the base64-encoded md5 sum
	base64md5, err := bagman.Base64EncodeMd5(result.BagMd5Digest)
	if err != nil {
		return s3.Options{}, err
	}
	options := storer.ProcUtil.S3Client.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

func (storer *Storer) RunTest(result *DPNResult) {
	storer.WaitGroup.Add(1)
	storer.ProcUtil.MessageLog.Info("Putting %s into digest channel",
		result.BagIdentifier)
	storer.StorageChannel <- result
	storer.WaitGroup.Wait()
	fmt.Println("Storer is done")
}
<commit_msg>Don't delete tar file if we created the DPN bag<commit_after>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"github.com/crowdmob/goamz/s3"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type Storer struct {
	StorageChannel      chan *DPNResult
	CleanupChannel      chan *DPNResult
	BagCreateChannel    chan *DPNResult
	PostProcessChannel  chan *DPNResult
	ProcUtil            *bagman.ProcessUtil
	DPNConfig           *DPNConfig
	LocalRESTClient     *DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

func NewStorer(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Storer, error) {
	// Set up a DPN REST client that talks to our local DPN REST service.
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}

	storer := &Storer{
		LocalRESTClient: localClient,
		ProcUtil: procUtil,
		DPNConfig: dpnConfig,
	}
	workerBufferSize := procUtil.Config.DPNStoreWorker.Workers * 10
	storer.StorageChannel = make(chan *DPNResult, workerBufferSize)
	storer.BagCreateChannel = make(chan *DPNResult, workerBufferSize)
	storer.CleanupChannel = make(chan *DPNResult, workerBufferSize)
	storer.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNStoreWorker.Workers; i++ {
		go storer.createBagRecord()
		go storer.cleanup()
		go storer.postProcess()
	}
	for i := 0; i < procUtil.Config.DPNStoreWorker.NetworkConnections; i++ {
		go storer.store()
	}
	return storer, nil
}

func (storer *Storer) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	result := &DPNResult{}
	err := json.Unmarshal(message.Body, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	if result.BagMd5Digest == "" || result.BagSha256Digest == "" {
		errMsg := "Bag cannot be stored because DPNResult is missing either md5 or sha256 checksum."
		storer.ProcUtil.MessageLog.Error(errMsg)
		result.ErrorMessage = errMsg
		SendToTroubleQueue(result, storer.ProcUtil)
		message.Finish()
		return fmt.Errorf(errMsg)
	}
	result.NsqMessage = message
	result.Stage = STAGE_STORE
	bagIdentifier := result.BagIdentifier
	if bagIdentifier == "" {
		bagIdentifier = "DPN Replication Bag"
	}
	storer.ProcUtil.MessageLog.Info("Putting %s into the storage queue (%s)",
		result.TarFilePath(), bagIdentifier)
	storer.StorageChannel <- result
	return nil
}

func (storer *Storer) store() {
	for result := range storer.StorageChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// By the time we get our hands on this replication request,
		// it may be many hours old. Fetch the request again from the
		// originating node and make sure it hasn't been cancelled.
		if result.TransferRequest != nil {
			storer.refetchReplicationRequest(result)
		}
		if result.ErrorMessage != "" {
			storer.PostProcessChannel <- result
			continue
		}
		// Remember that TransferRequest will be nil
		// for bags we built ourselves.
		if result.TransferRequest != nil && result.TransferRequest.Status == "Cancelled" {
			result.Stage = STAGE_CANCELLED
			result.Retry = false
			storer.PostProcessChannel <- result
			continue
		}

		err := os.MkdirAll(filepath.Dir(result.TarFilePath()), 0755)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error creating directory '%s': %v",
				filepath.Dir(result.TarFilePath()), err)
			storer.PostProcessChannel <- result
			continue
		}

		// Now we'll open a file reader and stream the tar file
		// up to S3.
		reader, err := os.Open(result.TarFilePath())
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		options, err := storer.GetS3Options(result)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error generating S3 options: %v", err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}

		bagUUID := ""
		if result.DPNBag != nil {
			bagUUID = result.DPNBag.UUID
		} else {
			bagUUID = result.PackageResult.BagBuilder.UUID
		}
		fileName := fmt.Sprintf("%s.tar", bagUUID)
		url := ""
		if fileInfo.Size() > bagman.S3_LARGE_FILE {
			url, err = storer.ProcUtil.S3Client.SaveLargeFileToS3(
				storer.ProcUtil.Config.DPNPreservationBucket,
				fileName,
				"application/x-tar",
				reader,
				fileInfo.Size(),
				options,
				bagman.S3_CHUNK_SIZE)
		} else {
			url, err = storer.ProcUtil.S3Client.SaveToS3(
				storer.ProcUtil.Config.DPNPreservationBucket,
				fileName,
				"application/x-tar",
				reader,
				fileInfo.Size(),
				options)
		}

		// Close the reader
		reader.Close()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error saving file to S3/Glacier: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		// Record where we stored the file
		result.StorageURL = url

		// Update the transfer request, if there is one.
		if result.TransferRequest != nil {
			result.TransferRequest.Status = "Stored"
		}
		if result.ErrorMessage != "" {
			result.ErrorMessage = fmt.Sprintf("Bag was stored, but we couldn't " +
				"update the replication request on the originating node: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// This channel really only applies to bags we created
		// at our own node. (Not replication requests.)
		storer.BagCreateChannel <- result
	}
}

// Get the replication request from the originating node and
// make sure it's still valid.
func (storer *Storer) refetchReplicationRequest(result *DPNResult) {
	storer.makeReplicationRequest(result, "GET")
}

func (storer *Storer) makeReplicationRequest(result *DPNResult, whatKind string) {

	if result.TransferRequest == nil {
		// This is a local bag and there is no replication request.
		storer.ProcUtil.MessageLog.Error("makeReplicationRequest was called for bag %s (%s), " +
			"which has no associated replication request. Is this a local bag?",
			result.BagIdentifier, result.PackageResult.BagBuilder.UUID)
		return
	}

	// Get a REST client that can talk the remote node.
	// This client will be configured with the correct URL
	// and API token to talk to the node that issues the
	// replication request.
	remoteClient, err := storer.LocalRESTClient.GetRemoteClient(
		result.TransferRequest.FromNode,  // the node we want to talk to
		storer.DPNConfig,
		storer.ProcUtil.MessageLog)
	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not create a REST client to connect to node %s: %v",
			result.TransferRequest.FromNode, err)
		return
	}

	var xferRequest *DPNReplicationTransfer
	if whatKind == "GET" {
		storer.ProcUtil.MessageLog.Debug("Replication request id: %s, " +
			"FromNode: %s, ToNode: %s", result.TransferRequest.ReplicationId,
			result.TransferRequest.FromNode, result.TransferRequest.ToNode)
		xferRequest, err = remoteClient.ReplicationTransferGet(result.TransferRequest.ReplicationId)
	} else if whatKind == "POST" {
		xferRequest, err = remoteClient.ReplicationTransferUpdate(result.TransferRequest)
	} else {
		panic("makeReplicationRequest doesn't understand that kind of request")
	}

	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not get replication record %s from node %s: %v",
			result.TransferRequest.ReplicationId, result.TransferRequest.FromNode, err)
		return
	}
	result.TransferRequest = xferRequest

}

func (storer *Storer) createBagRecord() {
	for result := range storer.BagCreateChannel {
		// If result has a local identifier, it's a bag we created
		// here at the local node (as opposed to a bag we're just
		// replicating to fulfill a transfer request). Since we
		// created this bag here, we need to create a Bag record
		// for it in DPN. We could also check result.PackageResult
		// to determine if bag was made here. That will be non-nil
		// for bags we built, nil for bags built elsewhere.
		bagWasCreatedHere := result.BagIdentifier != ""
		bagStoredSuccessfully := (result.ErrorMessage == "" && result.StorageURL != "")
		if bagWasCreatedHere && bagStoredSuccessfully {
			storer.ProcUtil.MessageLog.Debug("Creating bag record for %s with md5 %s and sha256 %s",
				result.BagIdentifier, result.BagMd5Digest, result.BagSha256Digest)
			fixity := &DPNFixity{
				Sha256: result.BagSha256Digest,
			}
			fileInfo, err := os.Stat(result.TarFilePath())
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Cannot stat %s to get file size: %v",
					result.TarFilePath(), err)
				storer.CleanupChannel <- result
				continue
			}
			newBag := &DPNBag{
				UUID: result.PackageResult.BagBuilder.UUID,
				LocalId: result.BagIdentifier,
				Size: uint64(fileInfo.Size()),
				FirstVersionUUID: result.PackageResult.BagBuilder.UUID,
				Version: 1,
				IngestNode: storer.DPNConfig.LocalNode,
				AdminNode: storer.DPNConfig.LocalNode,
				BagType: "D",
				// No spec yet on how to specify rights; cannot be nil
				Rights: make([]string, 0),
				// No spec yet on how to specify interpretive; cannot be nil
				Interpretive: make([]string, 0),
				ReplicatingNodes: []string{"aptrust"},
				Fixities: fixity,
			}
			result.DPNBag = newBag
		}
		storer.CleanupChannel <- result
	}
}

func (storer *Storer) cleanup() {
	for result := range storer.CleanupChannel {
		thisIsNotATest := (result.NsqMessage != nil)
		storageSucceeded := (result.ErrorMessage == "" && result.StorageURL != "")
		// If this bag came from another node, we can delete it after storing it.
		// If it came from our node, we need to keep it around until it's been
		// replicated.
		thisBagCameFromAnotherNode := (result.ProcessedItemId == 0)
		if storageSucceeded && thisIsNotATest && thisBagCameFromAnotherNode {
			err := os.Remove(result.TarFilePath())
			if err != nil {
				storer.ProcUtil.MessageLog.Warning("Error cleaning up %s: %v",
					result.TarFilePath(), err)
			} else {
				storer.ProcUtil.MessageLog.Info(
					"After successful upload, deleted local DPN bag at %s",
					result.TarFilePath())
			}
		}
		storer.PostProcessChannel <- result
	}
}

func (storer *Storer) postProcess() {
	for result := range storer.PostProcessChannel {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.DPNBag.UUID
		}
		if result.ErrorMessage == "" && result.StorageURL != "" {
			// SUCCESS :)
			storer.ProcUtil.MessageLog.Info("Bag %s successfully stored at %s",
				bagIdentifier, result.StorageURL)
			storer.ProcUtil.IncrementSucceeded()
			// Send to queue for recording in Fluctus and/or DPN REST
			if result.NsqMessage != nil {
				result.NsqMessage.Finish()
				SendToRecordQueue(result, storer.ProcUtil)
			}
		} else {
			// FAILURE :(
			storer.ProcUtil.MessageLog.Error(result.ErrorMessage)
			storer.ProcUtil.IncrementFailed()
			// Item failed after max attempts. Put in trouble queue
			// for admin review.
			if result.NsqMessage != nil {
				if result.NsqMessage.Attempts >= uint16(storer.ProcUtil.Config.DPNStoreWorker.MaxAttempts) {
					// No more retries
					result.NsqMessage.Finish()
					SendToTroubleQueue(result, storer.ProcUtil)
				} else {
					storer.ProcUtil.MessageLog.Info("Requeuing %s (%s)",
						bagIdentifier, result.TarFilePath())
					result.NsqMessage.Requeue(1 * time.Minute)
				}
			}
		}
		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			storer.WaitGroup.Done()
		}
	}
}

func (storer *Storer) GetS3Options(result *DPNResult) (s3.Options, error) {
	// Prepare metadata for save to S3
	s3Metadata := make(map[string][]string)
	if result.BagIdentifier != "" {
		s3Metadata["aptrust-bag"] = []string{result.BagIdentifier}
	}
	// Save to S3 with the base64-encoded md5 sum
	base64md5, err := bagman.Base64EncodeMd5(result.BagMd5Digest)
	if err != nil {
		return s3.Options{}, err
	}
	options := storer.ProcUtil.S3Client.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

func (storer *Storer) RunTest(result *DPNResult) {
	storer.WaitGroup.Add(1)
	storer.ProcUtil.MessageLog.Info("Putting %s into digest channel",
		result.BagIdentifier)
	storer.StorageChannel <- result
	storer.WaitGroup.Wait()
	fmt.Println("Storer is done")
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strings"
)

type gitHandler struct {
	method      string
	regexp      *regexp.Regexp
	handle_func func(string, string, string, http.ResponseWriter, *http.Request)
	rpc         string
}

var http_client = &http.Client{}

// Command-line options
var repo_root string
var listen_addr = flag.String("listen_addr", "localhost:8181", "Listen address for HTTP server")
var auth_backend = flag.String("auth_backend", "http://localhost:8080", "Authentication/authorization backend")

var git_handlers = [...]gitHandler{
	gitHandler{"GET", regexp.MustCompile(`\A(/..*)/info/refs\z`), handle_get_info_refs, ""},
	gitHandler{"POST", regexp.MustCompile(`\A(/..*)/git-upload-pack\z`), handle_post_rpc, "git-upload-pack"},
	gitHandler{"POST", regexp.MustCompile(`\A(/..*)/git-receive-pack\z`), handle_post_rpc, "git-receive-pack"},
}

func main() {
	// Parse the command-line
	flag.Parse()
	repo_root = flag.Arg(0)
	log.Printf("repo_root: %s", repo_root)

	http.HandleFunc("/", git_handler)
	log.Fatal(http.ListenAndServe(*listen_addr, nil))
}

func git_handler(w http.ResponseWriter, r *http.Request) {
	var user string

	log.Print(r.Method, " ", r.URL)

	for _, g := range git_handlers {
		path_match := g.regexp.FindStringSubmatch(r.URL.Path)
		if r.Method == g.method && path_match != nil {
			auth_response, err := do_auth_request(r)
			if err != nil {
				fail_500(w, err)
				return
			}
			if auth_response.StatusCode != 200 {
				// The Git request is not allowed by the backend. Maybe the client
				// needs to send HTTP Basic credentials.
				// Forward the response from the auth backend to our client
				for k, v := range auth_response.Header {
					w.Header()[k] = v
				}
				w.WriteHeader(auth_response.StatusCode)
				io.Copy(w, auth_response.Body)
				return
			}
			// The auth backend told us who the user is according to them
			if _, err := fmt.Fscan(auth_response.Body, &user); err != nil {
				fail_500(w, err)
				return
			}
			g.handle_func(user, g.rpc, path.Join(repo_root, path_match[1]), w, r)
			return
		}
	}
	http.Error(w, "Not found", 404)
}

func do_auth_request(r *http.Request) (result *http.Response, err error) {
	url := fmt.Sprintf("%s%s", *auth_backend, r.URL.RequestURI())
	auth_req, err := http.NewRequest(r.Method, url, nil)
	if err != nil {
		return nil, err
	}
	for k, v := range r.Header {
		auth_req.Header[k] = v
	}
	result, err = http_client.Do(auth_req)
	if err != nil {
		return nil, err
	}
	return result, nil
}

func handle_get_info_refs(user string, _ string, path string, w http.ResponseWriter, r *http.Request) {
	rpc := r.URL.Query().Get("service")
	switch rpc {
	case "git-upload-pack", "git-receive-pack":
		cmd := exec.Command("git", strings.TrimPrefix(rpc, "git-"), "--stateless-rpc", "--advertise-refs", path)
		set_cmd_env(cmd, user)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			fail_500(w, err)
			return
		}
		if err := cmd.Start(); err != nil {
			fail_500(w, err)
			return
		}
		w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
		no_cache(w)
		w.WriteHeader(200)
		fmt.Fprintf(w, "%s0000", pkt_line(fmt.Sprintf("# service=%s\n", rpc)))
		if _, err := io.Copy(w, stdout); err != nil {
			panic(err) // No point in sending 500 to the client
		}
		if err := cmd.Wait(); err != nil {
			panic(err) // No point in sending 500 to the client
		}
	case "":
		// The 'dumb' Git HTTP protocol is not supported
		http.Error(w, "Not found", 404)
	}
}

func set_cmd_env(cmd *exec.Cmd, user string) {
	cmd.Env = []string{fmt.Sprintf("PATH=%s", os.Getenv("PATH")), fmt.Sprintf("GL_ID=%s", user)}
}

func handle_post_rpc(user string, rpc string, path string, w http.ResponseWriter, r *http.Request) {
	var body io.Reader
	var err error

	if r.Header.Get("Content-Encoding") == "gzip" {
		body, err = gzip.NewReader(r.Body)
		if err != nil {
			fail_500(w, err)
			return
		}
	} else {
		body = r.Body
	}
	cmd := exec.Command("git", strings.TrimPrefix(rpc, "git-"), "--stateless-rpc", path)
	set_cmd_env(cmd, user)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fail_500(w, err)
		return
	}
	stdin, err := cmd.StdinPipe()
	if err != nil {
		fail_500(w, err)
		return
	}
	if err := cmd.Start(); err != nil {
		fail_500(w, err)
		return
	}
	if _, err := io.Copy(stdin, body); err != nil {
		fail_500(w, err)
		return
	}
	stdin.Close()

	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", rpc))
	no_cache(w)
	w.WriteHeader(200)
	if _, err := io.Copy(w, stdout); err != nil {
		panic(err) // No point in sending 500 to the client
	}
	if err := cmd.Wait(); err != nil {
		panic(err) // No point in sending 500 to the client
	}
}

func pkt_line(s string) string {
	return fmt.Sprintf("%04x%s", len(s)+4, s)
}

func fail_500(w http.ResponseWriter, err error) {
	http.Error(w, "Internal server error", 500)
	log.Print(err)
}

func no_cache(w http.ResponseWriter) {
	w.Header().Add("Cache-Control", "no-cache")
}
<commit_msg>Move Git line protocol into separate methods<commit_after>package main

import (
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strings"
)

type gitHandler struct {
	method      string
	regexp      *regexp.Regexp
	handle_func func(string, string, string, http.ResponseWriter, *http.Request)
	rpc         string
}

var http_client = &http.Client{}

// Command-line options
var repo_root string
var listen_addr = flag.String("listen_addr", "localhost:8181", "Listen address for HTTP server")
var auth_backend = flag.String("auth_backend", "http://localhost:8080", "Authentication/authorization backend")

var git_handlers = [...]gitHandler{
	gitHandler{"GET", regexp.MustCompile(`\A(/..*)/info/refs\z`), handle_get_info_refs, ""},
	gitHandler{"POST", regexp.MustCompile(`\A(/..*)/git-upload-pack\z`), handle_post_rpc, "git-upload-pack"},
	gitHandler{"POST", regexp.MustCompile(`\A(/..*)/git-receive-pack\z`), handle_post_rpc, "git-receive-pack"},
}

func main() {
	// Parse the command-line
	flag.Parse()
	repo_root = flag.Arg(0)
	log.Printf("repo_root: %s", repo_root)

	http.HandleFunc("/", git_handler)
	log.Fatal(http.ListenAndServe(*listen_addr, nil))
}

func git_handler(w http.ResponseWriter, r *http.Request) {
	var user string

	log.Print(r.Method, " ", r.URL)

	for _, g := range git_handlers {
		path_match := g.regexp.FindStringSubmatch(r.URL.Path)
		if r.Method == g.method && path_match != nil {
			auth_response, err := do_auth_request(r)
			if err != nil {
				fail_500(w, err)
				return
			}
			if auth_response.StatusCode != 200 {
				// The Git request is not allowed by the backend. Maybe the client
				// needs to send HTTP Basic credentials.
				// Forward the response from the auth backend to our client
				for k, v := range auth_response.Header {
					w.Header()[k] = v
				}
				w.WriteHeader(auth_response.StatusCode)
				io.Copy(w, auth_response.Body)
				return
			}
			// The auth backend told us who the user is according to them
			if _, err := fmt.Fscan(auth_response.Body, &user); err != nil {
				fail_500(w, err)
				return
			}
			g.handle_func(user, g.rpc, path.Join(repo_root, path_match[1]), w, r)
			return
		}
	}
	http.Error(w, "Not found", 404)
}

func do_auth_request(r *http.Request) (result *http.Response, err error) {
	url := fmt.Sprintf("%s%s", *auth_backend, r.URL.RequestURI())
	auth_req, err := http.NewRequest(r.Method, url, nil)
	if err != nil {
		return nil, err
	}
	for k, v := range r.Header {
		auth_req.Header[k] = v
	}
	result, err = http_client.Do(auth_req)
	if err != nil {
		return nil, err
	}
	return result, nil
}

func handle_get_info_refs(user string, _ string, path string, w http.ResponseWriter, r *http.Request) {
	rpc := r.URL.Query().Get("service")
	switch rpc {
	case "git-upload-pack", "git-receive-pack":
		cmd := exec.Command("git", strings.TrimPrefix(rpc, "git-"), "--stateless-rpc", "--advertise-refs", path)
		set_cmd_env(cmd, user)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			fail_500(w, err)
			return
		}
		if err := cmd.Start(); err != nil {
			fail_500(w, err)
			return
		}
		w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
		no_cache(w)
		w.WriteHeader(200)
		if err := pkt_line(w, fmt.Sprintf("# service=%s\n", rpc)); err != nil {
			panic(err)
		}
		if err := pkt_flush(w); err != nil {
			panic(err)
		}
		if _, err := io.Copy(w, stdout); err != nil {
			panic(err) // No point in sending 500 to the client
		}
		if err := cmd.Wait(); err != nil {
			panic(err) // No point in sending 500 to the client
		}
	case "":
		// The 'dumb' Git HTTP protocol is not supported
		http.Error(w, "Not found", 404)
	}
}

func set_cmd_env(cmd *exec.Cmd, user string) {
	cmd.Env = []string{fmt.Sprintf("PATH=%s", os.Getenv("PATH")), fmt.Sprintf("GL_ID=%s", user)}
}

func handle_post_rpc(user string, rpc string, path string, w http.ResponseWriter, r *http.Request) {
	var body io.Reader
	var err error

	if r.Header.Get("Content-Encoding") == "gzip" {
		body, err = gzip.NewReader(r.Body)
		if err != nil {
			fail_500(w, err)
			return
		}
	} else {
		body = r.Body
	}
	cmd := exec.Command("git", strings.TrimPrefix(rpc, "git-"), "--stateless-rpc", path)
	set_cmd_env(cmd, user)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fail_500(w, err)
		return
	}
	stdin, err := cmd.StdinPipe()
	if err != nil {
		fail_500(w, err)
		return
	}
	if err := cmd.Start(); err != nil {
		fail_500(w, err)
		return
	}
	if _, err := io.Copy(stdin, body); err != nil {
		fail_500(w, err)
		return
	}
	stdin.Close()

	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", rpc))
	no_cache(w)
	w.WriteHeader(200)
	if _, err := io.Copy(w, stdout); err != nil {
		panic(err) // No point in sending 500 to the client
	}
	if err := cmd.Wait(); err != nil {
		panic(err) // No point in sending 500 to the client
	}
}

func pkt_line(w io.Writer, s string) error {
	_, err := fmt.Fprintf(w, "%04x%s", len(s)+4, s)
	return err
}

func pkt_flush(w io.Writer) error {
	_, err := fmt.Fprint(w, "0000")
	return err
}

func fail_500(w http.ResponseWriter, err error) {
	http.Error(w, "Internal server error", 500)
	log.Print(err)
}

func no_cache(w http.ResponseWriter) {
	w.Header().Add("Cache-Control", "no-cache")
}
<|endoftext|>