<commit_before>package monico

type Moniter struct {
	path string
}

func NewMoniter(path string) (*Moniter, error) {
	return &Moniter{
		path: path,
	}, nil
}

func (m *Moniter) Path() string {
	return m.path
}
<commit_msg>Finish obvious implementation<commit_after>package monico

import (
	"os"
)

type Moniter struct {
	path string
}

func NewMoniter(path string) (*Moniter, error) {
	_, err := os.Stat(path)
	if err != nil {
		return nil, err
	}
	return &Moniter{
		path: path,
	}, nil
}

func (m *Moniter) Path() string {
	return m.path
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

var cmdGenerate = &Command{
	Run:       runGenerate,
	UsageLine: "generate [-run regexp] [file.go... | packages]",
	Short:     "generate Go files by processing source",
	Long: `
Generate runs commands described by directives within existing
files. Those commands can run any process but the intent is to
create or update Go source files, for instance by running yacc.

Go generate is never run automatically by go build, go get, go test,
and so on. It must be run explicitly.

Directives are written as a whole-line comment of the form

	//go:generate command argument...

(note: no space in "//go") where command is the generator to be
run, corresponding to an executable file that can be run locally.
It must either be in the shell path (gofmt), a fully qualified path
(/usr/you/bin/mytool), or a command alias, described below.

The arguments are space-separated tokens or double-quoted strings
passed to the generator as individual arguments when it is run.

Quoted strings use Go syntax and are evaluated before execution; a
quoted string appears a single argument to the generator.

Go generate sets several variables when it runs the generator:

	$GOARCH
		The execution architecture (arm, amd64, etc.)
	$GOOS
		The execution operating system (linux, windows, etc.)
	$GOFILE
		The base name of the file.
	$GOPACKAGE
		The name of the package of the file containing the directive.

Other than variable substition and quoted-string evaluation, no
special processing such as "globbing" is performed on the command
line.

As a last step before running the command, any invocations of any
environment variables with alphanumeric names, such as $GOFILE or
$HOME, are expanded throughout the command line. The syntax for
variable expansion is $NAME on all operating systems.  Due to the
order of evaluation, variables are expanded even inside quoted
strings. If the variable NAME is not set, $NAME expands to the
empty string.

A directive of the form,

	//go:generate -command xxx args...

specifies, for the remainder of this source file only, that the
string xxx represents the command identified by the arguments. This
can be used to create aliases or to handle multiword generators.
For example,

	//go:generate -command yacc go tool yacc

specifies that the command "yacc" represents the generator
"go tool yacc".

Generate processes packages in the order given on the command line,
one at a time. If the command line lists .go files, they are treated
as a single package. Within a package, generate processes the
source files in a package in file name order, one at a time. Within
a source file, generate runs generators in the order they appear
in the file, one at a time.

If any generator returns an error exit status, "go generate" skips
all further processing for that package.

The generator is run in the package's source directory.

Go generate accepts one specific flag:

	-run=""
		if non-empty, specifies a regular expression to
		select directives whose command matches the expression.

It also accepts the standard build flags -v, -n, and -x.
The -v flag prints the names of packages and files as they are
processed.
The -n flag prints commands that would be executed.
The -x flag prints commands as they are executed.

For more about specifying packages, see 'go help packages'.
	`,
}

var generateRunFlag string // generate -run flag

func init() {
	addBuildFlags(cmdGenerate)
	cmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
}

func runGenerate(cmd *Command, args []string) {
	// Even if the arguments are .go files, this loop suffices.
	for _, pkg := range packages(args) {
		for _, file := range pkg.gofiles {
			if !generate(pkg.Name, file) {
				break
			}
		}
	}
}

// generate runs the generation directives for a single file.
func generate(pkg, absFile string) bool {
	fd, err := os.Open(absFile)
	if err != nil {
		log.Fatalf("generate: %s", err)
	}
	defer fd.Close()
	g := &Generator{
		r:        fd,
		path:     absFile,
		pkg:      pkg,
		commands: make(map[string][]string),
	}
	return g.run()
}

// A Generator represents the state of a single Go source file
// being scanned for generator commands.
type Generator struct {
	r        io.Reader
	path     string // full rooted path name.
	dir      string // full rooted directory of file.
	file     string // base name of file.
	pkg      string
	commands map[string][]string
	lineNum  int
}

// run runs the generators in the current file.
func (g *Generator) run() (ok bool) {
	// Processing below here calls g.errorf on failure, which does panic(stop).
	// If we encounter an error, we abort the package.
	defer func() {
		e := recover()
		if e != nil {
			ok = false
			if e != stop {
				panic(e)
			}
		}
	}()
	g.dir, g.file = filepath.Split(g.path)
	g.dir = filepath.Clean(g.dir) // No final separator please.
	if buildV {
		fmt.Fprintf(os.Stderr, "%s\n", shortPath(g.path))
	}

	s := bufio.NewScanner(g.r)
	for s.Scan() {
		g.lineNum++
		if !bytes.HasPrefix(s.Bytes(), []byte("//go:generate ")) && !bytes.HasPrefix(s.Bytes(), []byte("//go:generate\t")) {
			continue
		}
		words := g.split(s.Text())
		if len(words) == 0 {
			g.errorf("no arguments to directive")
		}
		if words[0] == "-command" {
			g.setShorthand(words)
			continue
		}
		// Run the command line.
		if buildN || buildX {
			fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " "))
		}
		if buildN {
			continue
		}
		g.exec(words)
	}
	if s.Err() != nil {
		g.errorf("error reading %s: %s", shortPath(g.path), s.Err())
	}
	return true
}

// split breaks the line into words, evaluating quoted
// strings and evaluating environment variables.
// The initial //go:generate element is dropped.
func (g *Generator) split(line string) []string {
	// Parse line, obeying quoted strings.
	var words []string
	line = line[len("//go:generate "):]
	// One (possibly quoted) word per iteration.
Words:
	for {
		line = strings.TrimLeft(line, " \t")
		if len(line) == 0 {
			break
		}
		if line[0] == '"' {
			for i := 1; i < len(line); i++ {
				c := line[i] // Only looking for ASCII so this is OK.
				switch c {
				case '\\':
					if i+1 == len(line) {
						g.errorf("bad backslash")
					}
					i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
				case '"':
					word, err := strconv.Unquote(line[0 : i+1])
					if err != nil {
						g.errorf("bad quoted string")
					}
					words = append(words, word)
					line = line[i+1:]
					// Check the next character is space or end of line.
					if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
						g.errorf("expect space after quoted argument")
					}
					continue Words
				}
			}
			g.errorf("mismatched quoted string")
		}
		i := strings.IndexAny(line, " \t")
		if i < 0 {
			i = len(line)
		}
		words = append(words, line[0:i])
		line = line[i:]
	}
	// Substitute command if required.
	if len(words) > 0 && g.commands[words[0]] != nil {
		// Replace 0th word by command substitution.
		words = append(g.commands[words[0]], words[1:]...)
	}
	// Substitute environment variables.
	for i, word := range words {
		words[i] = g.expandEnv(word)
	}
	return words
}

var stop = fmt.Errorf("error in generation")

// errorf logs an error message prefixed with the file and line number.
// It then exits the program because generation stops at the first error.
func (g *Generator) errorf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s:%d: %s\n", shortPath(g.path), g.lineNum,
		fmt.Sprintf(format, args...))
	panic(stop)
}

// expandEnv expands any $XXX invocations in word.
func (g *Generator) expandEnv(word string) string {
	if !strings.ContainsRune(word, '$') {
		return word
	}
	var buf bytes.Buffer
	var w int
	var r rune
	for i := 0; i < len(word); i += w {
		r, w = utf8.DecodeRuneInString(word[i:])
		if r != '$' {
			buf.WriteRune(r)
			continue
		}
		w += g.identLength(word[i+w:])
		envVar := word[i+1 : i+w]
		var sub string
		switch envVar {
		case "GOARCH":
			sub = runtime.GOARCH
		case "GOOS":
			sub = runtime.GOOS
		case "GOFILE":
			sub = g.file
		case "GOPACKAGE":
			sub = g.pkg
		default:
			sub = os.Getenv(envVar)
		}
		buf.WriteString(sub)
	}
	return buf.String()
}

// identLength returns the length of the identifier beginning the string.
func (g *Generator) identLength(word string) int {
	for i, r := range word {
		if r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) {
			continue
		}
		return i
	}
	return len(word)
}

// setShorthand installs a new shorthand as defined by a -command directive.
func (g *Generator) setShorthand(words []string) {
	// Create command shorthand.
	if len(words) == 1 {
		g.errorf("no command specified for -command")
	}
	command := words[1]
	if g.commands[command] != nil {
		g.errorf("command %q defined multiply defined", command)
	}
	g.commands[command] = words[2:len(words):len(words)] // force later append to make copy
}

// exec runs the command specified by the argument. The first word is
// the command name itself.
func (g *Generator) exec(words []string) {
	cmd := exec.Command(words[0], words[1:]...)
	// Standard in and out of generator should be the usual.
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	// Run the command in the package directory.
	cmd.Dir = g.dir
	env := []string{
		"GOARCH=" + runtime.GOARCH,
		"GOOS=" + runtime.GOOS,
		"GOFILE=" + g.file,
		"GOPACKAGE=" + g.pkg,
	}
	cmd.Env = mergeEnvLists(env, os.Environ())
	err := cmd.Run()
	if err != nil {
		g.errorf("running %q: %s", words[0], err)
	}
}
<commit_msg>cmd/go: set exit status for failing "go generate" run.<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

var cmdGenerate = &Command{
	Run:       runGenerate,
	UsageLine: "generate [-run regexp] [file.go... | packages]",
	Short:     "generate Go files by processing source",
	Long: `
Generate runs commands described by directives within existing
files. Those commands can run any process but the intent is to
create or update Go source files, for instance by running yacc.

Go generate is never run automatically by go build, go get, go test,
and so on. It must be run explicitly.

Directives are written as a whole-line comment of the form

	//go:generate command argument...

(note: no space in "//go") where command is the generator to be
run, corresponding to an executable file that can be run locally.
It must either be in the shell path (gofmt), a fully qualified path
(/usr/you/bin/mytool), or a command alias, described below.

The arguments are space-separated tokens or double-quoted strings
passed to the generator as individual arguments when it is run.

Quoted strings use Go syntax and are evaluated before execution; a
quoted string appears a single argument to the generator.

Go generate sets several variables when it runs the generator:

	$GOARCH
		The execution architecture (arm, amd64, etc.)
	$GOOS
		The execution operating system (linux, windows, etc.)
	$GOFILE
		The base name of the file.
	$GOPACKAGE
		The name of the package of the file containing the directive.

Other than variable substition and quoted-string evaluation, no
special processing such as "globbing" is performed on the command
line.

As a last step before running the command, any invocations of any
environment variables with alphanumeric names, such as $GOFILE or
$HOME, are expanded throughout the command line. The syntax for
variable expansion is $NAME on all operating systems.  Due to the
order of evaluation, variables are expanded even inside quoted
strings. If the variable NAME is not set, $NAME expands to the
empty string.

A directive of the form,

	//go:generate -command xxx args...

specifies, for the remainder of this source file only, that the
string xxx represents the command identified by the arguments. This
can be used to create aliases or to handle multiword generators.
For example,

	//go:generate -command yacc go tool yacc

specifies that the command "yacc" represents the generator
"go tool yacc".

Generate processes packages in the order given on the command line,
one at a time. If the command line lists .go files, they are treated
as a single package. Within a package, generate processes the
source files in a package in file name order, one at a time. Within
a source file, generate runs generators in the order they appear
in the file, one at a time.

If any generator returns an error exit status, "go generate" skips
all further processing for that package.

The generator is run in the package's source directory.

Go generate accepts one specific flag:

	-run=""
		if non-empty, specifies a regular expression to
		select directives whose command matches the expression.

It also accepts the standard build flags -v, -n, and -x.
The -v flag prints the names of packages and files as they are
processed.
The -n flag prints commands that would be executed.
The -x flag prints commands as they are executed.

For more about specifying packages, see 'go help packages'.
	`,
}

var generateRunFlag string // generate -run flag

func init() {
	addBuildFlags(cmdGenerate)
	cmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
}

func runGenerate(cmd *Command, args []string) {
	// Even if the arguments are .go files, this loop suffices.
	for _, pkg := range packages(args) {
		for _, file := range pkg.gofiles {
			if !generate(pkg.Name, file) {
				break
			}
		}
	}
}

// generate runs the generation directives for a single file.
func generate(pkg, absFile string) bool {
	fd, err := os.Open(absFile)
	if err != nil {
		log.Fatalf("generate: %s", err)
	}
	defer fd.Close()
	g := &Generator{
		r:        fd,
		path:     absFile,
		pkg:      pkg,
		commands: make(map[string][]string),
	}
	return g.run()
}

// A Generator represents the state of a single Go source file
// being scanned for generator commands.
type Generator struct {
	r        io.Reader
	path     string // full rooted path name.
	dir      string // full rooted directory of file.
	file     string // base name of file.
	pkg      string
	commands map[string][]string
	lineNum  int
}

// run runs the generators in the current file.
func (g *Generator) run() (ok bool) {
	// Processing below here calls g.errorf on failure, which does panic(stop).
	// If we encounter an error, we abort the package.
	defer func() {
		e := recover()
		if e != nil {
			ok = false
			if e != stop {
				panic(e)
			}
			setExitStatus(1)
		}
	}()
	g.dir, g.file = filepath.Split(g.path)
	g.dir = filepath.Clean(g.dir) // No final separator please.
	if buildV {
		fmt.Fprintf(os.Stderr, "%s\n", shortPath(g.path))
	}

	s := bufio.NewScanner(g.r)
	for s.Scan() {
		g.lineNum++
		if !bytes.HasPrefix(s.Bytes(), []byte("//go:generate ")) && !bytes.HasPrefix(s.Bytes(), []byte("//go:generate\t")) {
			continue
		}
		words := g.split(s.Text())
		if len(words) == 0 {
			g.errorf("no arguments to directive")
		}
		if words[0] == "-command" {
			g.setShorthand(words)
			continue
		}
		// Run the command line.
		if buildN || buildX {
			fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " "))
		}
		if buildN {
			continue
		}
		g.exec(words)
	}
	if s.Err() != nil {
		g.errorf("error reading %s: %s", shortPath(g.path), s.Err())
	}
	return true
}

// split breaks the line into words, evaluating quoted
// strings and evaluating environment variables.
// The initial //go:generate element is dropped.
func (g *Generator) split(line string) []string {
	// Parse line, obeying quoted strings.
	var words []string
	line = line[len("//go:generate "):]
	// One (possibly quoted) word per iteration.
Words:
	for {
		line = strings.TrimLeft(line, " \t")
		if len(line) == 0 {
			break
		}
		if line[0] == '"' {
			for i := 1; i < len(line); i++ {
				c := line[i] // Only looking for ASCII so this is OK.
				switch c {
				case '\\':
					if i+1 == len(line) {
						g.errorf("bad backslash")
					}
					i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
				case '"':
					word, err := strconv.Unquote(line[0 : i+1])
					if err != nil {
						g.errorf("bad quoted string")
					}
					words = append(words, word)
					line = line[i+1:]
					// Check the next character is space or end of line.
					if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
						g.errorf("expect space after quoted argument")
					}
					continue Words
				}
			}
			g.errorf("mismatched quoted string")
		}
		i := strings.IndexAny(line, " \t")
		if i < 0 {
			i = len(line)
		}
		words = append(words, line[0:i])
		line = line[i:]
	}
	// Substitute command if required.
	if len(words) > 0 && g.commands[words[0]] != nil {
		// Replace 0th word by command substitution.
		words = append(g.commands[words[0]], words[1:]...)
	}
	// Substitute environment variables.
	for i, word := range words {
		words[i] = g.expandEnv(word)
	}
	return words
}

var stop = fmt.Errorf("error in generation")

// errorf logs an error message prefixed with the file and line number.
// It then exits the program (with exit status 1) because generation stops
// at the first error.
func (g *Generator) errorf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s:%d: %s\n", shortPath(g.path), g.lineNum,
		fmt.Sprintf(format, args...))
	panic(stop)
}

// expandEnv expands any $XXX invocations in word.
func (g *Generator) expandEnv(word string) string {
	if !strings.ContainsRune(word, '$') {
		return word
	}
	var buf bytes.Buffer
	var w int
	var r rune
	for i := 0; i < len(word); i += w {
		r, w = utf8.DecodeRuneInString(word[i:])
		if r != '$' {
			buf.WriteRune(r)
			continue
		}
		w += g.identLength(word[i+w:])
		envVar := word[i+1 : i+w]
		var sub string
		switch envVar {
		case "GOARCH":
			sub = runtime.GOARCH
		case "GOOS":
			sub = runtime.GOOS
		case "GOFILE":
			sub = g.file
		case "GOPACKAGE":
			sub = g.pkg
		default:
			sub = os.Getenv(envVar)
		}
		buf.WriteString(sub)
	}
	return buf.String()
}

// identLength returns the length of the identifier beginning the string.
func (g *Generator) identLength(word string) int {
	for i, r := range word {
		if r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) {
			continue
		}
		return i
	}
	return len(word)
}

// setShorthand installs a new shorthand as defined by a -command directive.
func (g *Generator) setShorthand(words []string) {
	// Create command shorthand.
	if len(words) == 1 {
		g.errorf("no command specified for -command")
	}
	command := words[1]
	if g.commands[command] != nil {
		g.errorf("command %q defined multiply defined", command)
	}
	g.commands[command] = words[2:len(words):len(words)] // force later append to make copy
}

// exec runs the command specified by the argument. The first word is
// the command name itself.
func (g *Generator) exec(words []string) {
	cmd := exec.Command(words[0], words[1:]...)
	// Standard in and out of generator should be the usual.
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	// Run the command in the package directory.
	cmd.Dir = g.dir
	env := []string{
		"GOARCH=" + runtime.GOARCH,
		"GOOS=" + runtime.GOOS,
		"GOFILE=" + g.file,
		"GOPACKAGE=" + g.pkg,
	}
	cmd.Env = mergeEnvLists(env, os.Environ())
	err := cmd.Run()
	if err != nil {
		g.errorf("running %q: %s", words[0], err)
	}
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"github.com/shurcooL/httpfs/union"
	"github.com/shurcooL/vfsgen"
	"log"
	"net/http"
)

// zeroModTimeFileSystem is an http.FileSystem wrapper.
// It exposes a filesystem exactly like Source, except
// all file modification times are changed to zero.
// See https://github.com/shurcooL/vfsgen/pull/40#issuecomment-355416103
type zeroModTimeFileSystem struct {
	Source http.FileSystem
}

func (fs zeroModTimeFileSystem) Open(name string) (http.File, error) {
	f, err := fs.Source.Open(name)
	return file{f}, err
}

type file struct {
	http.File
}

func (f file) Stat() (os.FileInfo, error) {
	fi, err := f.File.Stat()
	return fileInfo{fi}, err
}

type fileInfo struct {
	os.FileInfo
}

func (fi fileInfo) ModTime() time.Time { return time.Time{} }

func main() {
	fs := zeroModTimeFileSystem{
		Source: union.New(map[string]http.FileSystem{
			"/templates": http.Dir("templates"),
			"/source":    http.Dir("source"),
		}),
	}

	err := vfsgen.Generate(fs, vfsgen.Options{})

	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>add missing os package<commit_after>// +build ignore

package main

import (
	"github.com/shurcooL/httpfs/union"
	"github.com/shurcooL/vfsgen"
	"log"
	"net/http"
	"os"
	"time"
)

// zeroModTimeFileSystem is an http.FileSystem wrapper.
// It exposes a filesystem exactly like Source, except
// all file modification times are changed to zero.
// See https://github.com/shurcooL/vfsgen/pull/40#issuecomment-355416103
type zeroModTimeFileSystem struct {
	Source http.FileSystem
}

func (fs zeroModTimeFileSystem) Open(name string) (http.File, error) {
	f, err := fs.Source.Open(name)
	return file{f}, err
}

type file struct {
	http.File
}

func (f file) Stat() (os.FileInfo, error) {
	fi, err := f.File.Stat()
	return fileInfo{fi}, err
}

type fileInfo struct {
	os.FileInfo
}

func (fi fileInfo) ModTime() time.Time { return time.Time{} }

func main() {
	fs := zeroModTimeFileSystem{
		Source: union.New(map[string]http.FileSystem{
			"/templates": http.Dir("templates"),
			"/source":    http.Dir("source"),
		}),
	}

	err := vfsgen.Generate(fs, vfsgen.Options{})

	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"strconv"

	"github.com/kajic/paginate"
)

type Comment struct {
	text       string
	created_at int
	updated_at int
}

func (c *Comment) OrderValue(order string) string {
	switch {
	case order == "created_at":
		return strconv.Itoa(c.created_at)
	case order == "updated_at":
		return strconv.Itoa(c.updated_at)
	default:
		return ""
	}
}

func OpenDatabase(addr string) (*sql.DB, error) {
	db, err := sql.Open("mysql", addr)
	if err != nil {
		return nil, err
	}
	return db, nil
}

func GetComments(p *paginate.Pagination) []paginate.Item {
	var where string
	if p.Direction == paginate.ASC {
		where = fmt.Sprintf("%s >= %s", p.Order, p.Value)
	} else {
		where = fmt.Sprintf("%s <= %s", p.Order, p.Value)
	}
	order := fmt.Sprintf("%s %s", p.Order, p.DirectionString())

	q := `
	SELECT text, created_at, updated_at
	FROM   comments
	WHERE  ` + where + `
	ORDER BY ` + order + `
	LIMIT ?, ?
	`
	db, _ := OpenDatabase("database url")
	rows, _ := db.Query(q, p.Offset, p.Count+1)

	var items []paginate.Item
	for rows.Next() {
		var c *Comment
		if err := rows.Scan(&c.text, &c.created_at, &c.updated_at); err != nil {
			panic(err)
		}
		items = append(items, c)
	}
	return items
}

func commentsHandler(w http.ResponseWriter, r *http.Request) {
	// 1. Create pagination object based on request url.
	c := paginate.Config{Count: 5, Order: "updated_at", Direction: paginate.DESC}
	p, _ := paginate.FromUrl(r.URL, c)

	// 2. Query data source based on request parameters.
	comments := &paginate.Page{GetComments(p)}

	// 3. Create pagination urls based on data.
	next := p.Next(comments, true)
	prev := p.Prev(comments)

	// 4. Respond with data and pagination urls.
	fmt.Fprint(w, comments)
	if next != nil {
		nexturl, _ := next.ToUrl(r.URL)
		fmt.Fprint(w, nexturl)
	}
	if prev != nil {
		prevurl, _ := prev.ToUrl(r.URL)
		fmt.Fprint(w, prevurl)
	}
}

func main() {
	http.HandleFunc("/comments", commentsHandler)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Update comments example<commit_after>package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"strconv"

	"github.com/kajic/paginate"
)

type Comment struct {
	text       string
	created_at int
	updated_at int
}

func (c *Comment) PaginationValue(p *paginate.Pagination) string {
	switch {
	case p.Order == "created_at":
		return strconv.Itoa(c.created_at)
	case p.Order == "updated_at":
		return strconv.Itoa(c.updated_at)
	default:
		return ""
	}
}

func OpenDatabase(addr string) (*sql.DB, error) {
	db, err := sql.Open("mysql", addr)
	if err != nil {
		return nil, err
	}
	return db, nil
}

func GetComments(p *paginate.Pagination) ([]paginate.Pager, error) {
	var where string
	if p.Direction == paginate.ASC {
		where = fmt.Sprintf("%s >= %s", p.Order, p.Value)
	} else {
		where = fmt.Sprintf("%s <= %s", p.Order, p.Value)
	}
	var direction string
	if p.Direction == paginate.ASC {
		direction = "ASC"
	} else {
		direction = "DESC"
	}
	order := fmt.Sprintf("%s %s", p.Order, direction)

	q := `
	SELECT text, created_at, updated_at
	FROM   comments
	WHERE  ` + where + `
	ORDER BY ` + order + `
	LIMIT ?, ?
	`
	db, err := OpenDatabase("database url")
	if err != nil {
		return nil, err
	}
	// Note we fech an additional item to allow the pagination library to immedialely determine
	// if there is a next page after the current one.
	rows, err := db.Query(q, p.Offset, p.Count+1)
	if err != nil {
		return nil, err
	}

	var items []paginate.Pager
	for rows.Next() {
		var c *Comment
		if err := rows.Scan(&c.text, &c.created_at, &c.updated_at); err != nil {
			panic(err)
		}
		items = append(items, c)
	}
	return items, nil
}

func commentsHandler(w http.ResponseWriter, r *http.Request) {
	// 1. Create pagination object based on request url.
	c := paginate.Cursor{Count: 5, Order: "updated_at", Direction: paginate.DESC}
	p, _ := paginate.FromUrl(r.URL, c)

	// 2. Query data source based on request parameters.
	items, _ := GetComments(p)

	// 3. Create pagination urls based on data.
	next := p.Next(items, true)
	prev := p.Prev(items)

	// 4. Respond with data and pagination urls.
	comments := make([]*Comment, len(items))
	for i, item := range items {
		comments[i] = item.(*Comment)
	}

	fmt.Fprint(w, comments)
	if next != nil {
		nexturl, _ := next.ToUrl(r.URL)
		fmt.Fprint(w, nexturl)
	}
	if prev != nil {
		prevurl, _ := prev.ToUrl(r.URL)
		fmt.Fprint(w, prevurl)
	}
}

func main() {
	http.HandleFunc("/comments", commentsHandler)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"fmt"
	"math"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/karlseguin/ccache"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"
)

type Option struct {
	FilerGrpcAddress   string
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	ChunkSizeLimit     int64
	DataCenter         string
	DirListCacheLimit  int64
	EntryCacheTtl      time.Duration
	Umask              os.FileMode

	MountUid   uint32
	MountGid   uint32
	MountMode  os.FileMode
	MountCtime time.Time
	MountMtime time.Time
}

var _ = fs.FS(&WFS{})
var _ = fs.FSStatfser(&WFS{})

type WFS struct {
	option                    *Option
	listDirectoryEntriesCache *ccache.Cache

	// contains all open handles, protected by handlesLock
	handlesLock       sync.Mutex
	handles           []*FileHandle
	pathToHandleIndex map[filer2.FullPath]int

	bufPool sync.Pool

	stats statsCache

	// nodes, protected by nodesLock
	nodesLock sync.Mutex
	nodes     map[uint64]fs.Node
	root      fs.Node
}
type statsCache struct {
	filer_pb.StatisticsResponse
	lastChecked int64 // unix time in seconds
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		option:                    option,
		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(option.DirListCacheLimit * 3).ItemsToPrune(100)),
		pathToHandleIndex:         make(map[filer2.FullPath]int),
		bufPool: sync.Pool{
			New: func() interface{} {
				return make([]byte, option.ChunkSizeLimit)
			},
		},
		nodes: make(map[uint64]fs.Node),
	}

	wfs.root = &Dir{Path: wfs.option.FilerMountRootPath, wfs: wfs}

	return wfs
}

func (wfs *WFS) Root() (fs.Node, error) {
	return wfs.root, nil
}

func (wfs *WFS) WithFilerClient(ctx context.Context, fn func(context.Context, filer_pb.SeaweedFilerClient) error) error {

	err := util.WithCachedGrpcClient(ctx, func(ctx2 context.Context, grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(ctx2, client)
	}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)

	if err == nil {
		return nil
	}
	if strings.Contains(err.Error(), "context canceled") {
		time.Sleep(3337 * time.Millisecond)
		glog.V(2).Infoln("retry context canceled request...")
		return util.WithCachedGrpcClient(context.Background(), func(ctx2 context.Context, grpcConnection *grpc.ClientConn) error {
			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
			return fn(ctx2, client)
		}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
	}
	return err

}

func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {

	fullpath := file.fullpath()
	glog.V(4).Infof("%s AcquireHandle uid=%d gid=%d", fullpath, uid, gid)

	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	index, found := wfs.pathToHandleIndex[fullpath]
	if found && wfs.handles[index] != nil {
		glog.V(2).Infoln(fullpath, "found fileHandle id", index)
		return wfs.handles[index]
	}

	fileHandle = newFileHandle(file, uid, gid)
	for i, h := range wfs.handles {
		if h == nil {
			wfs.handles[i] = fileHandle
			fileHandle.handle = uint64(i)
			wfs.pathToHandleIndex[fullpath] = i
			glog.V(4).Infof("%s reuse fh %d", fullpath, fileHandle.handle)
			return
		}
	}

	wfs.handles = append(wfs.handles, fileHandle)
	fileHandle.handle = uint64(len(wfs.handles) - 1)
	wfs.pathToHandleIndex[fullpath] = int(fileHandle.handle)
	glog.V(4).Infof("%s new fh %d", fullpath, fileHandle.handle)

	return
}

func (wfs *WFS) ReleaseHandle(fullpath filer2.FullPath, handleId fuse.HandleID) {
	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	glog.V(4).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles))
	delete(wfs.pathToHandleIndex, fullpath)
	if int(handleId) < len(wfs.handles) {
		wfs.handles[int(handleId)] = nil
	}

	return
}

// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {

	glog.V(4).Infof("reading fs stats: %+v", req)

	if wfs.stats.lastChecked < time.Now().Unix()-20 {

		err := wfs.WithFilerClient(ctx, func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {

			request := &filer_pb.StatisticsRequest{
				Collection:  wfs.option.Collection,
				Replication: wfs.option.Replication,
				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
			}

			glog.V(4).Infof("reading filer stats: %+v", request)
			resp, err := client.Statistics(ctx, request)
			if err != nil {
				glog.V(0).Infof("reading filer stats %v: %v", request, err)
				return err
			}
			glog.V(4).Infof("read filer stats: %+v", resp)

			wfs.stats.TotalSize = resp.TotalSize
			wfs.stats.UsedSize = resp.UsedSize
			wfs.stats.FileCount = resp.FileCount
			wfs.stats.lastChecked = time.Now().Unix()

			return nil
		})
		if err != nil {
			glog.V(0).Infof("filer Statistics: %v", err)
			return err
		}
	}

	totalDiskSize := wfs.stats.TotalSize
	usedDiskSize := wfs.stats.UsedSize
	actualFileCount := wfs.stats.FileCount

	// Compute the total number of available blocks
	resp.Blocks = totalDiskSize / blockSize

	// Compute the number of used blocks
	numBlocks := uint64(usedDiskSize / blockSize)

	// Report the number of free and available blocks for the block size
	resp.Bfree = resp.Blocks - numBlocks
	resp.Bavail = resp.Blocks - numBlocks
	resp.Bsize = uint32(blockSize)

	// Report the total number of possible files in the file system (and those free)
	resp.Files = math.MaxInt64
	resp.Ffree = math.MaxInt64 - actualFileCount

	// Report the maximum length of a name and the minimum fragment size
	resp.Namelen = 1024
	resp.Frsize = uint32(blockSize)

	return nil
}

func (wfs *WFS) cacheGet(path filer2.FullPath) *filer_pb.Entry {
	item := wfs.listDirectoryEntriesCache.Get(string(path))
	if item != nil && !item.Expired() {
		return item.Value().(*filer_pb.Entry)
	}
	return nil
}
func (wfs *WFS) cacheSet(path filer2.FullPath, entry *filer_pb.Entry, ttl time.Duration) {
	if entry == nil {
		wfs.listDirectoryEntriesCache.Delete(string(path))
	} else {
		wfs.listDirectoryEntriesCache.Set(string(path), entry, ttl)
	}
}
func (wfs *WFS) cacheDelete(path filer2.FullPath) {
	wfs.listDirectoryEntriesCache.Delete(string(path))
}

func (wfs *WFS) getNode(fullpath filer2.FullPath, fn func() fs.Node) fs.Node {
	wfs.nodesLock.Lock()
	defer wfs.nodesLock.Unlock()

	node, found := wfs.nodes[fullpath.AsInode()]
	if found {
		return node
	}
	node = fn()
	if node != nil {
		wfs.nodes[fullpath.AsInode()] = node
	}
	return node
}

func (wfs *WFS) forgetNode(fullpath filer2.FullPath) {
	wfs.nodesLock.Lock()
	defer wfs.nodesLock.Unlock()

	delete(wfs.nodes, fullpath.AsInode())

}
<commit_msg>avoid retry delay<commit_after>package filesys

import (
	"context"
	"fmt"
	"math"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/karlseguin/ccache"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"
)

type Option struct {
	FilerGrpcAddress   string
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	ChunkSizeLimit     int64
	DataCenter         string
	DirListCacheLimit  int64
	EntryCacheTtl      time.Duration
	Umask              os.FileMode

	MountUid   uint32
	MountGid   uint32
	MountMode  os.FileMode
	MountCtime time.Time
	MountMtime time.Time
}

var _ = fs.FS(&WFS{})
var _ = fs.FSStatfser(&WFS{})

type WFS struct {
	option                    *Option
	listDirectoryEntriesCache *ccache.Cache

	// contains all open handles, protected by handlesLock
	handlesLock       sync.Mutex
	handles           []*FileHandle
	pathToHandleIndex map[filer2.FullPath]int

	bufPool sync.Pool

	stats statsCache

	// nodes, protected by nodesLock
	nodesLock sync.Mutex
	nodes     map[uint64]fs.Node
	root      fs.Node
}
type statsCache struct {
	filer_pb.StatisticsResponse
	lastChecked int64 // unix time in seconds
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		option:                    option,
		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(option.DirListCacheLimit * 3).ItemsToPrune(100)),
		pathToHandleIndex:         make(map[filer2.FullPath]int),
		bufPool: sync.Pool{
			New: func() interface{} {
				return make([]byte, option.ChunkSizeLimit)
			},
		},
		nodes: make(map[uint64]fs.Node),
	}

	wfs.root = &Dir{Path: wfs.option.FilerMountRootPath, wfs: wfs}

	return wfs
}

func (wfs *WFS) Root() (fs.Node, error) {
	return wfs.root, nil
}

func (wfs *WFS) WithFilerClient(ctx context.Context, fn func(context.Context, filer_pb.SeaweedFilerClient) error) error {

	err := util.WithCachedGrpcClient(ctx, func(ctx2 context.Context, grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(ctx2, client)
	}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)

	if err == nil {
		return nil
	}
	if strings.Contains(err.Error(), "context canceled") {
		glog.V(2).Infoln("retry context canceled request...")
		return util.WithCachedGrpcClient(context.Background(), func(ctx2 context.Context, grpcConnection *grpc.ClientConn) error {
			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
			return fn(ctx2, client)
		}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
	}
	return err

}

func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {

	fullpath := file.fullpath()
	glog.V(4).Infof("%s AcquireHandle uid=%d gid=%d", fullpath, uid, gid)

	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	index, found := wfs.pathToHandleIndex[fullpath]
	if found && wfs.handles[index] != nil {
		glog.V(2).Infoln(fullpath, "found fileHandle id", index)
		return wfs.handles[index]
	}

	fileHandle = newFileHandle(file, uid, gid)
	for i, h := range wfs.handles {
		if h == nil {
			wfs.handles[i] = fileHandle
			fileHandle.handle = uint64(i)
			wfs.pathToHandleIndex[fullpath] = i
			glog.V(4).Infof("%s reuse fh %d", fullpath, fileHandle.handle)
			return
		}
	}

	wfs.handles = append(wfs.handles, fileHandle)
	fileHandle.handle = uint64(len(wfs.handles) - 1)
	wfs.pathToHandleIndex[fullpath] = int(fileHandle.handle)
	glog.V(4).Infof("%s new fh %d", fullpath, fileHandle.handle)

	return
}

func (wfs *WFS) ReleaseHandle(fullpath filer2.FullPath, handleId fuse.HandleID) {
	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	glog.V(4).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles))
	delete(wfs.pathToHandleIndex, fullpath)
	if int(handleId) < len(wfs.handles) {
		wfs.handles[int(handleId)] = nil
	}

	return
}

// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {

	glog.V(4).Infof("reading fs stats: %+v", req)

	if wfs.stats.lastChecked < time.Now().Unix()-20 {

		err := wfs.WithFilerClient(ctx, func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {

			request := &filer_pb.StatisticsRequest{
				Collection:  wfs.option.Collection,
				Replication: wfs.option.Replication,
				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
			}

			glog.V(4).Infof("reading filer stats: %+v", request)
			resp, err := client.Statistics(ctx, request)
			if err != nil {
				glog.V(0).Infof("reading filer stats %v: %v", request, err)
				return err
			}
			glog.V(4).Infof("read filer stats: %+v", resp)

			wfs.stats.TotalSize = resp.TotalSize
			wfs.stats.UsedSize = resp.UsedSize
			wfs.stats.FileCount = resp.FileCount
			wfs.stats.lastChecked = time.Now().Unix()

			return nil
		})
		if err != nil {
			glog.V(0).Infof("filer Statistics: %v", err)
			return err
		}
	}

	totalDiskSize := wfs.stats.TotalSize
	usedDiskSize := wfs.stats.UsedSize
	actualFileCount := wfs.stats.FileCount

	// Compute the total number of available blocks
	resp.Blocks = totalDiskSize / blockSize

	// Compute the number of used blocks
	numBlocks := uint64(usedDiskSize / blockSize)

	// Report the number of free and available blocks for the block size
	resp.Bfree = resp.Blocks - numBlocks
	resp.Bavail = resp.Blocks - numBlocks
	resp.Bsize = uint32(blockSize)

	// Report the total number of possible files in the file system (and those free)
	resp.Files = math.MaxInt64
	resp.Ffree = math.MaxInt64 - actualFileCount

	// Report the maximum length of a name and the minimum fragment size
	resp.Namelen = 1024
	resp.Frsize = uint32(blockSize)

	return nil
}

func (wfs *WFS) cacheGet(path filer2.FullPath) *filer_pb.Entry {
	item := wfs.listDirectoryEntriesCache.Get(string(path))
	if item != nil && !item.Expired() {
		return item.Value().(*filer_pb.Entry)
	}
	return nil
}
func (wfs *WFS) cacheSet(path filer2.FullPath, entry *filer_pb.Entry, ttl time.Duration) {
	if entry == nil {
		wfs.listDirectoryEntriesCache.Delete(string(path))
	} else {
		wfs.listDirectoryEntriesCache.Set(string(path), entry, ttl)
	}
}
func (wfs *WFS) cacheDelete(path filer2.FullPath) {
	wfs.listDirectoryEntriesCache.Delete(string(path))
}

func (wfs *WFS) getNode(fullpath filer2.FullPath, fn func() fs.Node) fs.Node {
	wfs.nodesLock.Lock()
	defer wfs.nodesLock.Unlock()

	node, found := wfs.nodes[fullpath.AsInode()]
	if found {
		return node
	}
	node = fn()
	if node != nil {
		wfs.nodes[fullpath.AsInode()] = node
	}
	return node
}

func (wfs *WFS) forgetNode(fullpath filer2.FullPath) {
	wfs.nodesLock.Lock()
	defer wfs.nodesLock.Unlock()

	delete(wfs.nodes, fullpath.AsInode())

}
<|endoftext|>


<commit_before>package repo_manager

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/go/autoroll"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/util"
)

const (
	DEPS_ROLL_BRANCH = "roll_branch"

	GCLIENT  = "gclient"
	ROLL_DEP = "roll-dep"

	REPO_CHROMIUM = "https://chromium.googlesource.com/chromium/src.git"

	TMPL_CQ_INCLUDE_TRYBOTS = "CQ_INCLUDE_TRYBOTS=%s"
)

var (
	// Use this function to instantiate a RepoManager. This is able to be
	// overridden for testing.
	NewRepoManager func(string, string, time.Duration, string) (RepoManager, error) = NewDefaultRepoManager

	DEPOT_TOOLS_AUTH_USER_REGEX = regexp.MustCompile(fmt.Sprintf("Logged in to %s as ([\\w-]+).", autoroll.RIETVELD_URL))
)

// issueJson is the structure of "git cl issue --json"
type issueJson struct {
	Issue    int64  `json:"issue"`
	IssueUrl string `json:"issue_url"`
}

// RepoManager is used by AutoRoller for managing checkouts.
type RepoManager interface {
	ForceUpdate() error
	FullChildHash(string) (string, error)
	LastRollRev() string
	RolledPast(string) bool
	ChildHead() string
	CreateNewRoll([]string, string, bool, bool) (int64, error)
	User() string
}

// repoManager is a struct used by AutoRoller for managing checkouts.
type repoManager struct {
	chromiumDir       string
	chromiumParentDir string
	depot_tools       string
	gclient           string
	infoMtx           sync.RWMutex
	lastRollRev       string
	repoMtx           sync.RWMutex
	rollDep           string
	childDir          string
	childHead         string
	childPath         string
	childRepo         *gitinfo.GitInfo
	user              string
}

// getEnv returns the environment used for most commands.
func getEnv(depotTools string) []string {
	return []string{
		fmt.Sprintf("PATH=%s:%s", depotTools, os.Getenv("PATH")),
		fmt.Sprintf("HOME=%s", os.Getenv("HOME")),
	}
}

// getDepotToolsUser returns the authorized depot tools user.
func getDepotToolsUser(depotTools string) (string, error) {
	output, err := exec.RunCommand(&exec.Command{
		Env:  getEnv(depotTools),
		Name: path.Join(depotTools, "depot-tools-auth"),
		Args: []string{"info", autoroll.RIETVELD_URL},
	})
	if err != nil {
		return "", err
	}
	m := DEPOT_TOOLS_AUTH_USER_REGEX.FindStringSubmatch(output)
	if len(m) != 2 {
		return "", fmt.Errorf("Unable to parse the output of depot-tools-auth.")
	}
	return m[1], nil
}

// NewDefaultRepoManager returns a RepoManager instance which operates in the given
// working directory and updates at the given frequency.
func NewDefaultRepoManager(workdir, childPath string, frequency time.Duration, depot_tools string) (RepoManager, error) {
	gclient := GCLIENT
	rollDep := ROLL_DEP
	if depot_tools != "" {
		gclient = path.Join(depot_tools, gclient)
		rollDep = path.Join(depot_tools, rollDep)
	}

	chromiumParentDir := path.Join(workdir, "chromium")
	chromiumDir := path.Join(chromiumParentDir, "src")

	user, err := getDepotToolsUser(depot_tools)
	if err != nil {
		return nil, fmt.Errorf("Failed to determine depot tools user: %s", err)
	}

	r := &repoManager{
		chromiumDir:       chromiumDir,
		chromiumParentDir: chromiumParentDir,
		depot_tools:       depot_tools,
		gclient:           gclient,
		rollDep:           rollDep,
		childDir:          path.Join(chromiumParentDir, childPath),
		childPath:         childPath,
		childRepo:         nil, // This will be filled in on the first update.
		user:              user,
	}
	if err := r.update(); err != nil {
		return nil, err
	}
	go func() {
		for _ = range time.Tick(frequency) {
			util.LogErr(r.update())
		}
	}()
	return r, nil
}

// update syncs code in the relevant repositories.
func (r *repoManager) update() error {
	// Sync the projects.
	r.repoMtx.Lock()
	defer r.repoMtx.Unlock()

	// Create the chromium parent directory if needed.
	if _, err := os.Stat(r.chromiumParentDir); err != nil {
		if err := os.MkdirAll(r.chromiumParentDir, 0755); err != nil {
			return err
		}
	}

	if _, err := os.Stat(path.Join(r.chromiumDir, ".git")); err == nil {
		if err := r.cleanChromium(); err != nil {
			return err
		}
		// Update the repo.
		if _, err := exec.RunCwd(r.chromiumDir, "git", "fetch"); err != nil {
			return err
		}
		if _, err := exec.RunCwd(r.chromiumDir, "git", "reset", "--hard", "origin/master"); err != nil {
			return err
		}
	}

	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumParentDir,
		Env:  getEnv(r.depot_tools),
		Name: r.gclient,
		Args: []string{"config", REPO_CHROMIUM},
	}); err != nil {
		return err
	}
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumParentDir,
		Env:  getEnv(r.depot_tools),
		Name: r.gclient,
		Args: []string{"sync", "--nohooks"},
	}); err != nil {
		return err
	}

	// Create the child GitInfo if needed.
	if r.childRepo == nil {
		childRepo, err := gitinfo.NewGitInfo(r.childDir, false, true)
		if err != nil {
			return err
		}
		r.childRepo = childRepo
	}

	// Get the last roll revision.
	lastRollRev, err := r.getLastRollRev()
	if err != nil {
		return err
	}

	// Record child HEAD
	childHead, err := r.childRepo.FullHash("origin/master")
	if err != nil {
		return err
	}
	r.infoMtx.Lock()
	defer r.infoMtx.Unlock()
	r.lastRollRev = lastRollRev
	r.childHead = childHead
	return nil
}

// ForceUpdate forces the repoManager to update.
func (r *repoManager) ForceUpdate() error {
	return r.update()
}

// getLastRollRev returns the commit hash of the last-completed DEPS roll.
func (r *repoManager) getLastRollRev() (string, error) {
	output, err := exec.RunCwd(r.chromiumDir, r.gclient, "revinfo")
	if err != nil {
		return "", err
	}
	split := strings.Split(output, "\n")
	for _, s := range split {
		if strings.HasPrefix(s, r.childPath) {
			subs := strings.Split(s, "@")
			if len(subs) != 2 {
				return "", fmt.Errorf("Failed to parse output of `gclient revinfo`")
			}
			return subs[1], nil
		}
	}
	return "", fmt.Errorf("Failed to parse output of `gclient revinfo`")
}

// FullChildHash returns the full hash of the given short hash or ref in the
// child repo.
func (r *repoManager) FullChildHash(shortHash string) (string, error) {
	r.repoMtx.RLock()
	defer r.repoMtx.RUnlock()
	return r.childRepo.FullHash(shortHash)
}

// LastRollRev returns the last-rolled child commit.
func (r *repoManager) LastRollRev() string {
	r.infoMtx.RLock()
	defer r.infoMtx.RUnlock()
	return r.lastRollRev
}

// RolledPast determines whether DEPS has rolled past the given commit.
func (r *repoManager) RolledPast(hash string) bool {
	r.repoMtx.RLock()
	defer r.repoMtx.RUnlock()
	if _, err := exec.RunCwd(r.childDir, "git", "merge-base", "--is-ancestor", hash, r.lastRollRev); err != nil {
		return false
	}
	return true
}

// ChildHead returns the current child origin/master branch head.
func (r *repoManager) ChildHead() string {
	r.infoMtx.RLock()
	defer r.infoMtx.RUnlock()
	return r.childHead
}

// cleanChromium forces the Chromium checkout into a clean state.
func (r *repoManager) cleanChromium() error {
	if _, err := exec.RunCwd(r.chromiumDir, "git", "clean", "-d", "-f", "-f"); err != nil {
		return err
	}
	_, _ = exec.RunCwd(r.chromiumDir, "git", "rebase", "--abort")
	if _, err := exec.RunCwd(r.chromiumDir, "git", "checkout", "origin/master", "-f"); err != nil {
		return err
	}
	_, _ = exec.RunCwd(r.chromiumDir, "git", "branch", "-D", DEPS_ROLL_BRANCH)
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: r.gclient,
		Args: []string{"revert", "--nohooks"},
	}); err != nil {
		return err
	}
	return nil
}

// CreateNewRoll creates and uploads a new DEPS roll to the given commit.
// Returns the issue number of the uploaded roll.
func (r *repoManager) CreateNewRoll(emails []string, cqExtraTrybots string, dryRun, gerrit bool) (int64, error) {
	r.repoMtx.Lock()
	defer r.repoMtx.Unlock()

	// Clean the checkout, get onto a fresh branch.
	if err := r.cleanChromium(); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(r.chromiumDir, "git", "checkout", "-b", DEPS_ROLL_BRANCH, "-t", "origin/master", "-f"); err != nil {
		return 0, err
	}

	// Defer some more cleanup.
	defer func() {
		util.LogErr(r.cleanChromium())
	}()

	// Create the roll CL.
	if _, err := exec.RunCwd(r.chromiumDir, "git", "config", "user.name", autoroll.ROLL_AUTHOR); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(r.chromiumDir, "git", "config", "user.email", autoroll.ROLL_AUTHOR); err != nil {
		return 0, err
	}

	// Find Chromium bugs.
	bugs := []string{}
	cr := r.childRepo
	commits, err := cr.RevList(fmt.Sprintf("%s..%s", r.lastRollRev, r.childHead))
	if err != nil {
		return 0, fmt.Errorf("Failed to list revisions: %s", err)
	}
	for _, c := range commits {
		d, err := cr.Details(c, false)
		if err != nil {
			return 0, fmt.Errorf("Failed to obtain commit details: %s", err)
		}
		b := util.BugsFromCommitMsg(d.Body)
		for _, bug := range b[util.PROJECT_CHROMIUM] {
			bugs = append(bugs, bug)
		}
	}

	args := []string{r.childPath, r.childHead}
	if len(bugs) > 0 {
		args = append(args, "--bug", strings.Join(bugs, ","))
	}
	sklog.Infof("Running command: roll-dep %s", strings.Join(args, " "))
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: r.rollDep,
		Args: args,
	}); err != nil {
		return 0, err
	}
	// Build the commit message, starting with the message provided by roll-dep.
	commitMsg, err := exec.RunCwd(r.chromiumDir, "git", "log", "-n1", "--format=%B", "HEAD")
	if err != nil {
		return 0, err
	}
	commitMsg += `
Documentation for the AutoRoller is here:
https://skia.googlesource.com/buildbot/+/master/autoroll/README.md

If the roll is causing failures, see:
http://www.chromium.org/developers/tree-sheriffs/sheriff-details-chromium#TOC-Failures-due-to-DEPS-rolls

`
	if cqExtraTrybots != "" {
		commitMsg += "\n" + fmt.Sprintf(TMPL_CQ_INCLUDE_TRYBOTS, cqExtraTrybots)
	}
	uploadCmd := &exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: "git",
		Args: []string{"cl", "upload", "--bypass-hooks", "-f"},
	}
	if dryRun {
		uploadCmd.Args = append(uploadCmd.Args, "--cq-dry-run")
	} else {
		uploadCmd.Args = append(uploadCmd.Args, "--use-commit-queue")
	}
	if gerrit {
		uploadCmd.Args = append(uploadCmd.Args, "--gerrit")
	}
	tbr := "\nTBR="
	if emails != nil && len(emails) > 0 {
		emailStr := strings.Join(emails, ",")
		tbr += emailStr
		uploadCmd.Args = append(uploadCmd.Args, "--send-mail", "--cc", emailStr)
	}
	commitMsg += tbr
	uploadCmd.Args = append(uploadCmd.Args, "-m", commitMsg)

	// Upload the CL.
	if _, err := exec.RunCommand(uploadCmd); err != nil {
		return 0, err
	}

	// Obtain the issue number.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return 0, err
	}
	defer util.RemoveAll(tmp)
	jsonFile := path.Join(tmp, "issue.json")
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: "git",
		Args: []string{"cl", "issue", fmt.Sprintf("--json=%s", jsonFile)},
	}); err != nil {
		return 0, err
	}
	f, err := os.Open(jsonFile)
	if err != nil {
		return 0, err
	}
	var issue issueJson
	if err := json.NewDecoder(f).Decode(&issue); err != nil {
		return 0, err
	}
	return issue.Issue, nil
}

func (r *repoManager) User() string {
	return r.user
}
<commit_msg>[autoroll] Disable GCE auth for CL uploads<commit_after>package repo_manager

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/go/autoroll"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/util"
)

const (
	DEPS_ROLL_BRANCH = "roll_branch"

	GCLIENT  = "gclient"
	ROLL_DEP = "roll-dep"

	REPO_CHROMIUM = "https://chromium.googlesource.com/chromium/src.git"

	TMPL_CQ_INCLUDE_TRYBOTS = "CQ_INCLUDE_TRYBOTS=%s"
)

var (
	// Use this function to instantiate a RepoManager. This is able to be
	// overridden for testing.
	NewRepoManager func(string, string, time.Duration, string) (RepoManager, error) = NewDefaultRepoManager

	DEPOT_TOOLS_AUTH_USER_REGEX = regexp.MustCompile(fmt.Sprintf("Logged in to %s as ([\\w-]+).", autoroll.RIETVELD_URL))
)

// issueJson is the structure of "git cl issue --json"
type issueJson struct {
	Issue    int64  `json:"issue"`
	IssueUrl string `json:"issue_url"`
}

// RepoManager is used by AutoRoller for managing checkouts.
type RepoManager interface {
	ForceUpdate() error
	FullChildHash(string) (string, error)
	LastRollRev() string
	RolledPast(string) bool
	ChildHead() string
	CreateNewRoll([]string, string, bool, bool) (int64, error)
	User() string
}

// repoManager is a struct used by AutoRoller for managing checkouts.
type repoManager struct {
	chromiumDir       string
	chromiumParentDir string
	depot_tools       string
	gclient           string
	infoMtx           sync.RWMutex
	lastRollRev       string
	repoMtx           sync.RWMutex
	rollDep           string
	childDir          string
	childHead         string
	childPath         string
	childRepo         *gitinfo.GitInfo
	user              string
}

// getEnv returns the environment used for most commands.
func getEnv(depotTools string) []string {
	return []string{
		fmt.Sprintf("PATH=%s:%s", depotTools, os.Getenv("PATH")),
		fmt.Sprintf("HOME=%s", os.Getenv("HOME")),
		fmt.Sprintf("SKIP_GCE_AUTH_FOR_GIT=1"),
	}
}

// getDepotToolsUser returns the authorized depot tools user.
func getDepotToolsUser(depotTools string) (string, error) {
	output, err := exec.RunCommand(&exec.Command{
		Env:  getEnv(depotTools),
		Name: path.Join(depotTools, "depot-tools-auth"),
		Args: []string{"info", autoroll.RIETVELD_URL},
	})
	if err != nil {
		return "", err
	}
	m := DEPOT_TOOLS_AUTH_USER_REGEX.FindStringSubmatch(output)
	if len(m) != 2 {
		return "", fmt.Errorf("Unable to parse the output of depot-tools-auth.")
	}
	return m[1], nil
}

// NewDefaultRepoManager returns a RepoManager instance which operates in the given
// working directory and updates at the given frequency.
func NewDefaultRepoManager(workdir, childPath string, frequency time.Duration, depot_tools string) (RepoManager, error) {
	gclient := GCLIENT
	rollDep := ROLL_DEP
	if depot_tools != "" {
		gclient = path.Join(depot_tools, gclient)
		rollDep = path.Join(depot_tools, rollDep)
	}

	chromiumParentDir := path.Join(workdir, "chromium")
	chromiumDir := path.Join(chromiumParentDir, "src")

	user, err := getDepotToolsUser(depot_tools)
	if err != nil {
		return nil, fmt.Errorf("Failed to determine depot tools user: %s", err)
	}

	r := &repoManager{
		chromiumDir:       chromiumDir,
		chromiumParentDir: chromiumParentDir,
		depot_tools:       depot_tools,
		gclient:           gclient,
		rollDep:           rollDep,
		childDir:          path.Join(chromiumParentDir, childPath),
		childPath:         childPath,
		childRepo:         nil, // This will be filled in on the first update.
		user:              user,
	}
	if err := r.update(); err != nil {
		return nil, err
	}
	go func() {
		for _ = range time.Tick(frequency) {
			util.LogErr(r.update())
		}
	}()
	return r, nil
}

// update syncs code in the relevant repositories.
func (r *repoManager) update() error {
	// Sync the projects.
	r.repoMtx.Lock()
	defer r.repoMtx.Unlock()

	// Create the chromium parent directory if needed.
	if _, err := os.Stat(r.chromiumParentDir); err != nil {
		if err := os.MkdirAll(r.chromiumParentDir, 0755); err != nil {
			return err
		}
	}

	if _, err := os.Stat(path.Join(r.chromiumDir, ".git")); err == nil {
		if err := r.cleanChromium(); err != nil {
			return err
		}
		// Update the repo.
		if _, err := exec.RunCwd(r.chromiumDir, "git", "fetch"); err != nil {
			return err
		}
		if _, err := exec.RunCwd(r.chromiumDir, "git", "reset", "--hard", "origin/master"); err != nil {
			return err
		}
	}

	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumParentDir,
		Env:  getEnv(r.depot_tools),
		Name: r.gclient,
		Args: []string{"config", REPO_CHROMIUM},
	}); err != nil {
		return err
	}
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumParentDir,
		Env:  getEnv(r.depot_tools),
		Name: r.gclient,
		Args: []string{"sync", "--nohooks"},
	}); err != nil {
		return err
	}

	// Create the child GitInfo if needed.
	if r.childRepo == nil {
		childRepo, err := gitinfo.NewGitInfo(r.childDir, false, true)
		if err != nil {
			return err
		}
		r.childRepo = childRepo
	}

	// Get the last roll revision.
	lastRollRev, err := r.getLastRollRev()
	if err != nil {
		return err
	}

	// Record child HEAD
	childHead, err := r.childRepo.FullHash("origin/master")
	if err != nil {
		return err
	}
	r.infoMtx.Lock()
	defer r.infoMtx.Unlock()
	r.lastRollRev = lastRollRev
	r.childHead = childHead
	return nil
}

// ForceUpdate forces the repoManager to update.
func (r *repoManager) ForceUpdate() error {
	return r.update()
}

// getLastRollRev returns the commit hash of the last-completed DEPS roll.
func (r *repoManager) getLastRollRev() (string, error) {
	output, err := exec.RunCwd(r.chromiumDir, r.gclient, "revinfo")
	if err != nil {
		return "", err
	}
	split := strings.Split(output, "\n")
	for _, s := range split {
		if strings.HasPrefix(s, r.childPath) {
			subs := strings.Split(s, "@")
			if len(subs) != 2 {
				return "", fmt.Errorf("Failed to parse output of `gclient revinfo`")
			}
			return subs[1], nil
		}
	}
	return "", fmt.Errorf("Failed to parse output of `gclient revinfo`")
}

// FullChildHash returns the full hash of the given short hash or ref in the
// child repo.
func (r *repoManager) FullChildHash(shortHash string) (string, error) {
	r.repoMtx.RLock()
	defer r.repoMtx.RUnlock()
	return r.childRepo.FullHash(shortHash)
}

// LastRollRev returns the last-rolled child commit.
func (r *repoManager) LastRollRev() string {
	r.infoMtx.RLock()
	defer r.infoMtx.RUnlock()
	return r.lastRollRev
}

// RolledPast determines whether DEPS has rolled past the given commit.
func (r *repoManager) RolledPast(hash string) bool {
	r.repoMtx.RLock()
	defer r.repoMtx.RUnlock()
	if _, err := exec.RunCwd(r.childDir, "git", "merge-base", "--is-ancestor", hash, r.lastRollRev); err != nil {
		return false
	}
	return true
}

// ChildHead returns the current child origin/master branch head.
func (r *repoManager) ChildHead() string {
	r.infoMtx.RLock()
	defer r.infoMtx.RUnlock()
	return r.childHead
}

// cleanChromium forces the Chromium checkout into a clean state.
func (r *repoManager) cleanChromium() error {
	if _, err := exec.RunCwd(r.chromiumDir, "git", "clean", "-d", "-f", "-f"); err != nil {
		return err
	}
	_, _ = exec.RunCwd(r.chromiumDir, "git", "rebase", "--abort")
	if _, err := exec.RunCwd(r.chromiumDir, "git", "checkout", "origin/master", "-f"); err != nil {
		return err
	}
	_, _ = exec.RunCwd(r.chromiumDir, "git", "branch", "-D", DEPS_ROLL_BRANCH)
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: r.gclient,
		Args: []string{"revert", "--nohooks"},
	}); err != nil {
		return err
	}
	return nil
}

// CreateNewRoll creates and uploads a new DEPS roll to the given commit.
// Returns the issue number of the uploaded roll.
func (r *repoManager) CreateNewRoll(emails []string, cqExtraTrybots string, dryRun, gerrit bool) (int64, error) {
	r.repoMtx.Lock()
	defer r.repoMtx.Unlock()

	// Clean the checkout, get onto a fresh branch.
	if err := r.cleanChromium(); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(r.chromiumDir, "git", "checkout", "-b", DEPS_ROLL_BRANCH, "-t", "origin/master", "-f"); err != nil {
		return 0, err
	}

	// Defer some more cleanup.
	defer func() {
		util.LogErr(r.cleanChromium())
	}()

	// Create the roll CL.
	if _, err := exec.RunCwd(r.chromiumDir, "git", "config", "user.name", autoroll.ROLL_AUTHOR); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(r.chromiumDir, "git", "config", "user.email", autoroll.ROLL_AUTHOR); err != nil {
		return 0, err
	}

	// Find Chromium bugs.
	bugs := []string{}
	cr := r.childRepo
	commits, err := cr.RevList(fmt.Sprintf("%s..%s", r.lastRollRev, r.childHead))
	if err != nil {
		return 0, fmt.Errorf("Failed to list revisions: %s", err)
	}
	for _, c := range commits {
		d, err := cr.Details(c, false)
		if err != nil {
			return 0, fmt.Errorf("Failed to obtain commit details: %s", err)
		}
		b := util.BugsFromCommitMsg(d.Body)
		for _, bug := range b[util.PROJECT_CHROMIUM] {
			bugs = append(bugs, bug)
		}
	}

	args := []string{r.childPath, r.childHead}
	if len(bugs) > 0 {
		args = append(args, "--bug", strings.Join(bugs, ","))
	}
	sklog.Infof("Running command: roll-dep %s", strings.Join(args, " "))
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: r.rollDep,
		Args: args,
	}); err != nil {
		return 0, err
	}
	// Build the commit message, starting with the message provided by roll-dep.
	commitMsg, err := exec.RunCwd(r.chromiumDir, "git", "log", "-n1", "--format=%B", "HEAD")
	if err != nil {
		return 0, err
	}
	commitMsg += `
Documentation for the AutoRoller is here:
https://skia.googlesource.com/buildbot/+/master/autoroll/README.md

If the roll is causing failures, see:
http://www.chromium.org/developers/tree-sheriffs/sheriff-details-chromium#TOC-Failures-due-to-DEPS-rolls

`
	if cqExtraTrybots != "" {
		commitMsg += "\n" + fmt.Sprintf(TMPL_CQ_INCLUDE_TRYBOTS, cqExtraTrybots)
	}
	uploadCmd := &exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: "git",
		Args: []string{"cl", "upload", "--bypass-hooks", "-f"},
	}
	if dryRun {
		uploadCmd.Args = append(uploadCmd.Args, "--cq-dry-run")
	} else {
		uploadCmd.Args = append(uploadCmd.Args, "--use-commit-queue")
	}
	if gerrit {
		uploadCmd.Args = append(uploadCmd.Args, "--gerrit")
	}
	tbr := "\nTBR="
	if emails != nil && len(emails) > 0 {
		emailStr := strings.Join(emails, ",")
		tbr += emailStr
		uploadCmd.Args = append(uploadCmd.Args, "--send-mail", "--cc", emailStr)
	}
	commitMsg += tbr
	uploadCmd.Args = append(uploadCmd.Args, "-m", commitMsg)

	// Upload the CL.
	if _, err := exec.RunCommand(uploadCmd); err != nil {
		return 0, err
	}

	// Obtain the issue number.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return 0, err
	}
	defer util.RemoveAll(tmp)
	jsonFile := path.Join(tmp, "issue.json")
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumDir,
		Env:  getEnv(r.depot_tools),
		Name: "git",
		Args: []string{"cl", "issue", fmt.Sprintf("--json=%s", jsonFile)},
	}); err != nil {
		return 0, err
	}
	f, err := os.Open(jsonFile)
	if err != nil {
		return 0, err
	}
	var issue issueJson
	if err := json.NewDecoder(f).Decode(&issue); err != nil {
		return 0, err
	}
	return issue.Issue, nil
}

func (r *repoManager) User() string {
	return r.user
}
<|endoftext|>


<commit_before>package database

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	"cloud.google.com/go/spanner"
	database "cloud.google.com/go/spanner/admin/database/apiv1"
	"github.com/Cidan/sheep/stats"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
	adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
	"google.golang.org/grpc/codes"
)

type Spanner struct {
	Database
	client *spanner.Client
	admin  *database.DatabaseAdminClient
}

// SetupSpanner initializes the spanner clients.
func NewSpanner(project, instance, db string) (*Spanner, error) {
	ctx := context.Background()
	sp := &Spanner{}

	adminClient, err := database.NewDatabaseAdminClient(ctx)
	if err != nil {
		return nil, err
	}

	sp.admin = adminClient

	// Create the databases if they don't exist.
	err = sp.createSpannerDatabase(ctx, project, instance, db)

	if err != nil {
		return nil, err
	}

	dbstr := fmt.Sprintf("projects/%s/instances/%s/databases/%s",
		project,
		instance,
		db)

	client, err := spanner.NewClient(context.Background(), dbstr)
	if err != nil {
		return nil, err
	}

	sp.client = client
	return sp, err
}

func (s *Spanner) Read(msg *Message) error {

	stmt := spanner.NewStatement(`
			SELECT SUM(a.Count) as Count
			FROM sheep as a
			WHERE a.Keyspace=@Keyspace
			AND a.Key=@Key
			AND a.Name=@Name
		`)
	stmt.Params["Keyspace"] = msg.Keyspace
	stmt.Params["Key"] = msg.Key
	stmt.Params["Name"] = msg.Name

	// TODO: Expose this stale time.
	// TODO: Stale time breaks tests.
	iter := s.client.
		Single().
		//WithTimestampBound(spanner.MaxStaleness(5*time.Second)).
		Query(context.Background(), stmt)
	defer iter.Stop()

	row, err := iter.Next()

	if err != nil {
		return err
	}

	var value spanner.NullInt64
	err = row.ColumnByName("Count", &value)

	if err != nil {
		return err
	}

	if value.Valid {
		msg.Value = value.Int64
		return nil
	}

	return &spanner.Error{
		Code: codes.NotFound,
		Desc: "counter not found",
	}
}

func (s *Spanner) Save(message *Message) error {
	ctx := context.WithValue(context.Background(), contextKey("message"), message)
	if _, err := s.client.ReadWriteTransaction(ctx, s.doSave); err != nil {
		stats.Incr("spanner.save.error", 1)
		return err
	}
	stats.Incr("spanner.save.success", 1)
	return nil
}

// Here's where the magic happens. Save out message!
func (s *Spanner) doSave(ctx context.Context, rw *spanner.ReadWriteTransaction) error {
	msg := ctx.Value(contextKey("message")).(*Message)
	shards := viper.GetInt("spanner.shards")
	shard := rand.Intn(shards)

	// First, let's check and see if our message has been written.
	row, err := rw.ReadRow(context.Background(), "sheep_transaction", spanner.Key{msg.Keyspace, msg.Key, msg.Name, msg.UUID}, []string{"UUID"})
	if err != nil {
		if spanner.ErrCode(err) != codes.NotFound {
			return err
		}
	} else if err == nil {
		// We need to return if err is nil, this means
		// the UUID was found.
		return nil
	}

	// Let's get our current count
	var move int64
	row, err = rw.ReadRow(context.Background(), "sheep", spanner.Key{msg.Keyspace, msg.Key, msg.Name, shard}, []string{"Count"})
	if err != nil {
		if spanner.ErrCode(err) != codes.NotFound {
			return err
		}
	} else {
		row.ColumnByName("Count", &move)
	}

	// Now we'll do our operation.
	switch msg.Operation {
	case "incr":
		move++
	case "decr":
		move--
	case "set":
		move = msg.Value
	default:
		return &spanner.Error{
			Code: codes.InvalidArgument,
			Desc: "Invalid operation sent from message '" + msg.Operation + "', aborting transaction!",
		}
	}

	m := []*spanner.Mutation{}

	log.Debug().Int("shard", shard).Msg("shard selected for op")
	if msg.Operation == "set" {
		for i := 0; i < shards; i++ {
			m = append(m, spanner.InsertOrUpdate(
				"sheep",
				[]string{"Keyspace", "Key", "Name", "Shard", "Count"},
				[]interface{}{msg.Keyspace, msg.Key, msg.Name, i, move},
			))
		}
	} else {
		m = append(m, spanner.InsertOrUpdate(
			"sheep",
			[]string{"Keyspace", "Key", "Name", "Shard", "Count"},
			[]interface{}{msg.Keyspace, msg.Key, msg.Name, shard, move},
		))
	}

	m = append(m, spanner.InsertOrUpdate(
		"sheep_transaction",
		[]string{"Keyspace", "Key", "Name", "UUID", "Time"},
		[]interface{}{msg.Keyspace, msg.Key, msg.Name, msg.UUID, time.Now()}))

	// ...and write!
	return rw.BufferWrite(m)

}

func (s *Spanner) createSpannerDatabase(ctx context.Context, project, instance, db string) error {
	// Create our database if it doesn't exist.
	_, err := s.admin.GetDatabase(ctx, &adminpb.GetDatabaseRequest{
		Name: "projects/" + project + "/instances/" + instance + "/databases/" + db})
	if err != nil {
		// Database doesn't exist, or error.
		op, err := s.admin.CreateDatabase(ctx, &adminpb.CreateDatabaseRequest{
			Parent:          "projects/" + project + "/instances/" + instance,
			CreateStatement: "CREATE DATABASE `" + db + "`",
			ExtraStatements: []string{
				`CREATE TABLE sheep (
							Keyspace 	STRING(MAX) NOT NULL,
							Key 			STRING(MAX) NOT NULL,
							Name			STRING(MAX) NOT NULL,
							Shard     INT64       NOT NULL,
							Count 		INT64       NOT NULL
					) PRIMARY KEY (Keyspace, Key, Name, Shard)`,
				`CREATE TABLE sheep_transaction (
							Keyspace 	STRING(MAX) NOT NULL,
							Key 			STRING(MAX) NOT NULL,
							Name			STRING(MAX) NOT NULL,
							UUID 			STRING(128) NOT NULL,
							Time      TIMESTAMP   NOT NULL
					) PRIMARY KEY (Keyspace, Key, Name, UUID)`,
				`CREATE TABLE sheep_stats (
					   UUID      STRING(MAX) NOT NULL,
						 Key       STRING(MAX) NOT NULL,
						 Value     FLOAT64     NOT NULL,
						 Hostname  STRING(MAX) NOT NULL,
						 Last      TIMESTAMP   NOT NULL
				 ) PRIMARY KEY (UUID, Key)`,
			},
		})

		if err != nil {
			return err
		}

		_, err = op.Wait(ctx)

		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>typo fix<commit_after>package database

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	"cloud.google.com/go/spanner"
	database "cloud.google.com/go/spanner/admin/database/apiv1"
	"github.com/Cidan/sheep/stats"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
	adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
	"google.golang.org/grpc/codes"
)

type Spanner struct {
	Database
	client *spanner.Client
	admin  *database.DatabaseAdminClient
}

// SetupSpanner initializes the spanner clients.
func NewSpanner(project, instance, db string) (*Spanner, error) {
	ctx := context.Background()
	sp := &Spanner{}

	adminClient, err := database.NewDatabaseAdminClient(ctx)
	if err != nil {
		return nil, err
	}

	sp.admin = adminClient

	// Create the databases if they don't exist.
	err = sp.createSpannerDatabase(ctx, project, instance, db)

	if err != nil {
		return nil, err
	}

	dbstr := fmt.Sprintf("projects/%s/instances/%s/databases/%s",
		project,
		instance,
		db)

	client, err := spanner.NewClient(context.Background(), dbstr)
	if err != nil {
		return nil, err
	}

	sp.client = client
	return sp, err
}

func (s *Spanner) Read(msg *Message) error {

	stmt := spanner.NewStatement(`
			SELECT SUM(a.Count) as Count
			FROM sheep as a
			WHERE a.Keyspace=@Keyspace
			AND a.Key=@Key
			AND a.Name=@Name
		`)
	stmt.Params["Keyspace"] = msg.Keyspace
	stmt.Params["Key"] = msg.Key
	stmt.Params["Name"] = msg.Name

	// TODO: Expose this stale time.
	// TODO: Stale time breaks tests.
	iter := s.client.
		Single().
		//WithTimestampBound(spanner.MaxStaleness(5*time.Second)).
		Query(context.Background(), stmt)
	defer iter.Stop()

	row, err := iter.Next()

	if err != nil {
		return err
	}

	var value spanner.NullInt64
	err = row.ColumnByName("Count", &value)

	if err != nil {
		return err
	}

	if value.Valid {
		msg.Value = value.Int64
		return nil
	}

	return &spanner.Error{
		Code: codes.NotFound,
		Desc: "counter not found",
	}
}

func (s *Spanner) Save(message *Message) error {
	ctx := context.WithValue(context.Background(), contextKey("message"), message)
	if _, err := s.client.ReadWriteTransaction(ctx, s.doSave); err != nil {
		stats.Incr("spanner.save.error", 1)
		return err
	}
	stats.Incr("spanner.save.success", 1)
	return nil
}

// Here's where the magic happens. Save our message!
func (s *Spanner) doSave(ctx context.Context, rw *spanner.ReadWriteTransaction) error {
	msg := ctx.Value(contextKey("message")).(*Message)
	shards := viper.GetInt("spanner.shards")
	shard := rand.Intn(shards)

	// First, let's check and see if our message has been written.
	row, err := rw.ReadRow(context.Background(), "sheep_transaction", spanner.Key{msg.Keyspace, msg.Key, msg.Name, msg.UUID}, []string{"UUID"})
	if err != nil {
		if spanner.ErrCode(err) != codes.NotFound {
			return err
		}
	} else if err == nil {
		// We need to return if err is nil, this means
		// the UUID was found.
		return nil
	}

	// Let's get our current count
	var move int64
	row, err = rw.ReadRow(context.Background(), "sheep", spanner.Key{msg.Keyspace, msg.Key, msg.Name, shard}, []string{"Count"})
	if err != nil {
		if spanner.ErrCode(err) != codes.NotFound {
			return err
		}
	} else {
		row.ColumnByName("Count", &move)
	}

	// Now we'll do our operation.
	switch msg.Operation {
	case "incr":
		move++
	case "decr":
		move--
	case "set":
		move = msg.Value
	default:
		return &spanner.Error{
			Code: codes.InvalidArgument,
			Desc: "Invalid operation sent from message '" + msg.Operation + "', aborting transaction!",
		}
	}

	m := []*spanner.Mutation{}

	log.Debug().Int("shard", shard).Msg("shard selected for op")
	if msg.Operation == "set" {
		for i := 0; i < shards; i++ {
			m = append(m, spanner.InsertOrUpdate(
				"sheep",
				[]string{"Keyspace", "Key", "Name", "Shard", "Count"},
				[]interface{}{msg.Keyspace, msg.Key, msg.Name, i, move},
			))
		}
	} else {
		m = append(m, spanner.InsertOrUpdate(
			"sheep",
			[]string{"Keyspace", "Key", "Name", "Shard", "Count"},
			[]interface{}{msg.Keyspace, msg.Key, msg.Name, shard, move},
		))
	}

	m = append(m, spanner.InsertOrUpdate(
		"sheep_transaction",
		[]string{"Keyspace", "Key", "Name", "UUID", "Time"},
		[]interface{}{msg.Keyspace, msg.Key, msg.Name, msg.UUID, time.Now()}))

	// ...and write!
	return rw.BufferWrite(m)

}

func (s *Spanner) createSpannerDatabase(ctx context.Context, project, instance, db string) error {
	// Create our database if it doesn't exist.
	_, err := s.admin.GetDatabase(ctx, &adminpb.GetDatabaseRequest{
		Name: "projects/" + project + "/instances/" + instance + "/databases/" + db})
	if err != nil {
		// Database doesn't exist, or error.
		op, err := s.admin.CreateDatabase(ctx, &adminpb.CreateDatabaseRequest{
			Parent:          "projects/" + project + "/instances/" + instance,
			CreateStatement: "CREATE DATABASE `" + db + "`",
			ExtraStatements: []string{
				`CREATE TABLE sheep (
							Keyspace 	STRING(MAX) NOT NULL,
							Key 			STRING(MAX) NOT NULL,
							Name			STRING(MAX) NOT NULL,
							Shard     INT64       NOT NULL,
							Count 		INT64       NOT NULL
					) PRIMARY KEY (Keyspace, Key, Name, Shard)`,
				`CREATE TABLE sheep_transaction (
							Keyspace 	STRING(MAX) NOT NULL,
							Key 			STRING(MAX) NOT NULL,
							Name			STRING(MAX) NOT NULL,
							UUID 			STRING(128) NOT NULL,
							Time      TIMESTAMP   NOT NULL
					) PRIMARY KEY (Keyspace, Key, Name, UUID)`,
				`CREATE TABLE sheep_stats (
					   UUID      STRING(MAX) NOT NULL,
						 Key       STRING(MAX) NOT NULL,
						 Value     FLOAT64     NOT NULL,
						 Hostname  STRING(MAX) NOT NULL,
						 Last      TIMESTAMP   NOT NULL
				 ) PRIMARY KEY (UUID, Key)`,
			},
		})

		if err != nil {
			return err
		}

		_, err = op.Wait(ctx)

		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package assets

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/go-playground/bundler"
)

const (
	oldNewSeparator = " --> "
	manifestFile    = "/manifest.txt"
	cssHTMLTag      = "css_tag"
	jsHTMLTag       = "js_tag"
)

// RunMode is the type that determines which mode the template.FuncMap functions whould run in.
type RunMode int

// RunMode's
const (
	Development RunMode = iota
	Production
)

const (
	jsTag  = `<script type="text/javascript" src="%s"></script>`
	cssTag = `<link type="text/css" rel="stylesheet" href="%s">`
)

// Generate processes (bundles, compresses...) the assets for use and creates the Manifest file
// NOTE: no compression yet until there is a native and establishes compressor written in Go
func Generate(dirname string, outputDir string, relativeToDir bool, leftDelim string, rightDelim string, ignoreRegexp *regexp.Regexp) ([]*bundler.ProcessedFile, string, error) {

	dirname = filepath.Clean(dirname)
	outputDir = filepath.Clean(outputDir) + string(filepath.Separator)

	abs, err := filepath.Abs(outputDir)
	if err != nil {
		return nil, "", err
	}

	if err = os.MkdirAll(abs, os.FileMode(0777)); err != nil {
		return nil, "", err
	}

	// verify dirname is actually a DIR + do symlink check
	fi, err := os.Lstat(dirname)
	if err != nil {
		return nil, "", err
	}

	if !fi.IsDir() {

		// check if symlink
		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {

			link, err := filepath.EvalSymlinks(dirname)
			if err != nil {
				return nil, "", errors.New("Error Resolving Symlink:" + err.Error())
			}

			fi, err = os.Stat(link)
			if err != nil {
				return nil, "", err
			}

			if !fi.IsDir() {
				return nil, "", errors.New("dirname passed in is not a directory")
			}

			dirname = link

		} else {
			return nil, "", errors.New("dirname passed in is not a directory")
		}
	}

	var manifest string

	if outputDir == "" {
		manifest = dirname + manifestFile
	} else {
		manifest = outputDir + dirname + manifestFile
	}

	f, err := os.Open(manifest)
	if err == nil {
		defer f.Close()
		scanner := bufio.NewScanner(f)

		for scanner.Scan() {
			files := strings.SplitN(scanner.Text(), oldNewSeparator, 2)

			fmt.Println("Removing Existing File:", files[1])
			os.Remove(files[1])
		}

		os.Remove(manifest)
	}

	processed, err := bundleDir(dirname, "", false, "", ignoreRegexp, outputDir, relativeToDir, dirname, leftDelim, rightDelim)
	if err != nil {
		return nil, "", err
	}

	var buff bytes.Buffer

	for _, file := range processed {
		buff.WriteString(filepath.FromSlash(file.OriginalFilename))
		buff.WriteString(oldNewSeparator)
		buff.WriteString(filepath.FromSlash(file.NewFilename))
		buff.WriteString("\n")
	}

	if err = ioutil.WriteFile(manifest, buff.Bytes(), 0644); err != nil {
		return nil, "", err
	}

	return processed, manifest, nil
}

func bundleDir(path string, dir string, isSymlinkDir bool, symlinkDir string, ignoreRegexp *regexp.Regexp, output string, relativeToDir bool, relativeDir string, leftDelim string, rightDelim string) ([]*bundler.ProcessedFile, error) {

	var p string
	var processed []*bundler.ProcessedFile
	var cp bool

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	files, err := f.Readdir(0)
	if err != nil {
		return nil, err
	}

	for _, file := range files {

		info := file
		p = path + string(os.PathSeparator) + file.Name()
		fPath := p

		if isSymlinkDir {
			fPath = strings.Replace(p, dir, symlinkDir, 1)
		}

		if ignoreRegexp != nil && ignoreRegexp.MatchString(fPath) {
			cp = true
		}

		if file.IsDir() {

			processedFiles, err := bundleDir(p, p, isSymlinkDir, symlinkDir+string(os.PathSeparator)+info.Name(), ignoreRegexp, output, relativeToDir, relativeDir, leftDelim, rightDelim)
			if err != nil {
				return nil, err
			}

			processed = append(processed, processedFiles...)

			continue
		}

		if file.Mode()&os.ModeSymlink == os.ModeSymlink {

			link, err := filepath.EvalSymlinks(p)
			if err != nil {
				log.Panic("Error Resolving Symlink", err)
			}

			fi, err := os.Stat(link)
			if err != nil {
				log.Panic(err)
			}

			info = fi

			if fi.IsDir() {

				processedFiles, err := bundleDir(link, link, true, fPath, ignoreRegexp, output, relativeToDir, relativeDir, leftDelim, rightDelim)
				if err != nil {
					return nil, err
				}

				processed = append(processed, processedFiles...)

				continue
			}
		}

		if cp {
			// just copy file to final location
			if err := copyFile(p, output); err != nil {
				return nil, err
			}
			continue
		}

		// process file
		file, err := bundleFile(p, output, relativeToDir, relativeDir, leftDelim, rightDelim)
		if err != nil {
			return nil, err
		}

		processed = append(processed, file)
	}

	return processed, nil
}

func copyFile(path string, output string) error {

	dirname := output + string(filepath.Separator) + path

	if err := os.MkdirAll(filepath.Dir(dirname), os.FileMode(0777)); err != nil {
		return err
	}

	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	out, err := os.Create(dirname)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = io.Copy(out, f)
	if err != nil {
		return err
	}

	return nil
}

func bundleFile(path string, output string, relativeToDir bool, relativeDir string, leftDelim string, rightDelim string) (*bundler.ProcessedFile, error) {

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	newFile, err := ioutil.TempFile("", filepath.Base(f.Name()))
	if err != nil {
		return nil, err
	}

	if err = bundler.Bundle(f, newFile, filepath.Dir(path), relativeToDir, relativeDir, leftDelim, rightDelim); err != nil {
		return nil, err
	}

	var newName string

	dirname, filename := filepath.Split(path)
	origDir := dirname
	ext := filepath.Ext(filename)
	filename = filepath.Base(filename)

	if output != "" {
		dirname = output + string(filepath.Separator) + dirname + string(filepath.Separator)
	}

	newName = dirname + filename[0:strings.LastIndex(filename, ext)]
	origDir += filename[0:strings.LastIndex(filename, ext)]

	b, err := ioutil.ReadFile(newFile.Name())
	if err != nil {
		return nil, err
	}

	h := md5.New()
	h.Write(b)
	hash := string(h.Sum(nil))

	hashName := "-" + fmt.Sprintf("%x", hash) + ext
	newName += hashName
	origDir += hashName

	if err = os.MkdirAll(filepath.Dir(newName), os.FileMode(0777)); err != nil {
		return nil, err
	}

	if err = os.Rename(newFile.Name(), newName); err != nil {
		return nil, err
	}

	return &bundler.ProcessedFile{OriginalFilename: path, NewFilename: origDir}, nil
}

// LoadManifestFiles reads the manifest file generated by the Generate() command
// in Production mode and returns template.FuncMap for the provided RunMode
func LoadManifestFiles(dirname string, mode RunMode, relativeToDir bool, leftDelim string, rightDelim string) (template.FuncMap, error) {

	var f *os.File
	var err error

	if mode == Production {
		f, err = os.Open(dirname + manifestFile)
		if err != nil {
			return nil, err
		}
		defer f.Close()
	}

	return ProcessManifestFiles(f, dirname, mode, relativeToDir, leftDelim, rightDelim)
}

// ProcessManifestFiles reads an existing manifest file generated by the Generate() command
// in Production mode and returns template.FuncMap for the provided RunMode
func ProcessManifestFiles(manifest io.Reader, dirname string, mode RunMode, relativeToDir bool, leftDelim string, rightDelim string) (template.FuncMap, error) {

	mapped := map[string]string{}
	dirname = filepath.Clean(dirname) + string(os.PathSeparator)

	if mode == Production {

		mapped := map[string]string{}
		var files []string

		scanner := bufio.NewScanner(manifest)

		for scanner.Scan() {
			files = strings.SplitN(scanner.Text(), oldNewSeparator, 2)
			mapped[files[0]] = files[1]
		}
	}

	return loadMapFuncs(dirname, mode, relativeToDir, leftDelim, rightDelim, mapped), nil
}

func loadMapFuncs(dirname string, mode RunMode, relativeToDir bool, leftDelim string, rightDelim string, mapped map[string]string) template.FuncMap {

	funcs := template.FuncMap{}

	if mode == Production {
		funcs[cssHTMLTag] = createProdCSSTemplateFunc(mapped)
		funcs[jsHTMLTag] = createProdJSTemplateFunc(mapped)

		return funcs
	}

	funcs[cssHTMLTag] = createDevCSSTemplateFunc(dirname, relativeToDir, leftDelim, rightDelim)
	funcs[jsHTMLTag] = createDevJSTemplateFunc(dirname, relativeToDir, leftDelim, rightDelim)

	return funcs
}

func createProdCSSTemplateFunc(mapped map[string]string) interface{} {
	return func(name string) template.HTML {
		return template.HTML(fmt.Sprintf(cssTag, mapped[name]))
	}
}

func createProdJSTemplateFunc(mapped map[string]string) interface{} {
	return func(name string) template.HTML {
		return template.HTML(fmt.Sprintf(jsTag, mapped[name]))
	}
}

func createDevCSSTemplateFunc(dirname string, relativeToDir bool, leftDelim string, rightDelim string) interface{} {
	// custom lexer, bytesBuffer

	return func(name string) template.HTML {
		buff := new(bytes.Buffer)

		files, err := loadFromDelims(dirname, name, relativeToDir, dirname, leftDelim, rightDelim)
		if err != nil {
			panic(err)
		}

		for _, file := range files {
			buff.WriteString(fmt.Sprintf(cssTag, file))
		}

		buff.WriteString(fmt.Sprintf(cssTag, "/"+dirname+name))

		return template.HTML(buff.String())
	}
}

func createDevJSTemplateFunc(dirname string, relativeToDir bool, leftDelim string, rightDelim string) interface{} {
	// custom lexer, bytesBuffer

	return func(name string) template.HTML {
		buff := new(bytes.Buffer)

		files, err := loadFromDelims(dirname, name, relativeToDir, dirname, leftDelim, rightDelim)
		if err != nil {
			panic(err)
		}

		for _, file := range files {
			buff.WriteString(fmt.Sprintf(jsTag, file))
		}

		buff.WriteString(fmt.Sprintf(jsTag, "/"+dirname+name))

		return template.HTML(buff.String())
	}
}

func loadFromDelims(dirname string, name string, relativeToDir bool, relativeDir string, leftDelim string, rightDelim string) ([]string, error) {
	var err error
	var files []string
	var ok bool
	var path string

	existing := map[string]struct{}{}

	f, err := os.Open(dirname + name)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	l, err := bundler.NewLexer("assets-bundle", f, leftDelim, rightDelim)
	if err != nil {
		return nil, err
	}

LOOP:
	for {
		itm := l.NextItem()

		switch itm.Type {

		case bundler.ItemFile:

			if relativeToDir {
				path = filepath.FromSlash("/" + itm.Val)
			} else {
				path = filepath.FromSlash("/" + dirname + itm.Val)
			}

			if _, ok = existing[path]; !ok {
				files = append(files, path)
				existing[path] = struct{}{}
			}

			fls, err := loadFromDelims(dirname, itm.Val, relativeToDir, relativeDir, leftDelim, rightDelim)
			if err != nil {
				return nil, err
			}

			files = append(files, fls...)

		case bundler.ItemEOF:
			break LOOP
		case bundler.ItemError:
			return nil, errors.New(itm.Val)
		}
	}

	return files, nil
}
<commit_msg>Correct Embedded asset bug not adding dirname<commit_after>package assets

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/go-playground/bundler"
)

const (
	oldNewSeparator = " --> "
	manifestFile    = "/manifest.txt"
	cssHTMLTag      = "css_tag"
	jsHTMLTag       = "js_tag"
)

// RunMode is the type that determines which mode the template.FuncMap functions whould run in.
type RunMode int

// RunMode's
const (
	Development RunMode = iota
	Production
)

const (
	jsTag  = `<script type="text/javascript" src="%s"></script>`
	cssTag = `<link type="text/css" rel="stylesheet" href="%s">`
)

// Generate processes (bundles, compresses...) the assets for use and creates the Manifest file
// NOTE: no compression yet until there is a native and establishes compressor written in Go
func Generate(dirname string, outputDir string, relativeToDir bool, leftDelim string, rightDelim string, ignoreRegexp *regexp.Regexp) ([]*bundler.ProcessedFile, string, error) {

	dirname = filepath.Clean(dirname)
	outputDir = filepath.Clean(outputDir) + string(filepath.Separator)

	abs, err := filepath.Abs(outputDir)
	if err != nil {
		return nil, "", err
	}

	if err = os.MkdirAll(abs, os.FileMode(0777)); err != nil {
		return nil, "", err
	}

	// verify dirname is actually a DIR + do symlink check
	fi, err := os.Lstat(dirname)
	if err != nil {
		return nil, "", err
	}

	if !fi.IsDir() {

		// check if symlink
		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {

			link, err := filepath.EvalSymlinks(dirname)
			if err != nil {
				return nil, "", errors.New("Error Resolving Symlink:" + err.Error())
			}

			fi, err = os.Stat(link)
			if err != nil {
				return nil, "", err
			}

			if !fi.IsDir() {
				return nil, "", errors.New("dirname passed in is not a directory")
			}

			dirname = link

		} else {
			return nil, "", errors.New("dirname passed in is not a directory")
		}
	}

	var manifest string

	if outputDir == "" {
		manifest = dirname + manifestFile
	} else {
		manifest = outputDir + dirname + manifestFile
	}

	f, err := os.Open(manifest)
	if err == nil {
		defer f.Close()
		scanner := bufio.NewScanner(f)

		for scanner.Scan() {
			files := strings.SplitN(scanner.Text(), oldNewSeparator, 2)

			fmt.Println("Removing Existing File:", files[1])
			os.Remove(files[1])
		}

		os.Remove(manifest)
	}

	processed, err := bundleDir(dirname, "", false, "", ignoreRegexp, outputDir, relativeToDir, dirname, leftDelim, rightDelim)
	if err != nil {
		return nil, "", err
	}

	var buff bytes.Buffer

	for _, file := range processed {
		buff.WriteString(filepath.FromSlash(file.OriginalFilename))
		buff.WriteString(oldNewSeparator)
		buff.WriteString(filepath.FromSlash(file.NewFilename))
		buff.WriteString("\n")
	}

	if err = ioutil.WriteFile(manifest, buff.Bytes(), 0644); err != nil {
		return nil, "", err
	}

	return processed, manifest, nil
}

func bundleDir(path string, dir string, isSymlinkDir bool, symlinkDir string, ignoreRegexp *regexp.Regexp, output string, relativeToDir bool, relativeDir string, leftDelim string, rightDelim string) ([]*bundler.ProcessedFile, error) {

	var p string
	var processed []*bundler.ProcessedFile
	var cp bool

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	files, err := f.Readdir(0)
	if err != nil {
		return nil, err
	}

	for _, file := range files {

		info := file
		p = path + string(os.PathSeparator) + file.Name()
		fPath := p

		if isSymlinkDir {
			fPath = strings.Replace(p, dir, symlinkDir, 1)
		}

		if ignoreRegexp != nil && ignoreRegexp.MatchString(fPath) {
			cp = true
		}

		if file.IsDir() {

			processedFiles, err := bundleDir(p, p, isSymlinkDir, symlinkDir+string(os.PathSeparator)+info.Name(), ignoreRegexp, output, relativeToDir, relativeDir, leftDelim, rightDelim)
			if err != nil {
				return nil, err
			}

			processed = append(processed, processedFiles...)

			continue
		}

		if file.Mode()&os.ModeSymlink == os.ModeSymlink {

			link, err := filepath.EvalSymlinks(p)
			if err != nil {
				log.Panic("Error Resolving Symlink", err)
			}

			fi, err := os.Stat(link)
			if err != nil {
				log.Panic(err)
			}

			info = fi

			if fi.IsDir() {

				processedFiles, err := bundleDir(link, link, true, fPath, ignoreRegexp, output, relativeToDir, relativeDir, leftDelim, rightDelim)
				if err != nil {
					return nil, err
				}

				processed = append(processed, processedFiles...)

				continue
			}
		}

		if cp {
			// just copy file to final location
			if err := copyFile(p, output); err != nil {
				return nil, err
			}
			continue
		}

		// process file
		file, err := bundleFile(p, output, relativeToDir, relativeDir, leftDelim, rightDelim)
		if err != nil {
			return nil, err
		}

		processed = append(processed, file)
	}

	return processed, nil
}

func copyFile(path string, output string) error {

	dirname := output + string(filepath.Separator) + path

	if err := os.MkdirAll(filepath.Dir(dirname), os.FileMode(0777)); err != nil {
		return err
	}

	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	out, err := os.Create(dirname)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = io.Copy(out, f)
	if err != nil {
		return err
	}

	return nil
}

func bundleFile(path string, output string, relativeToDir bool, relativeDir string, leftDelim string, rightDelim string) (*bundler.ProcessedFile, error) {

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	newFile, err := ioutil.TempFile("", filepath.Base(f.Name()))
	if err != nil {
		return nil, err
	}

	if err = bundler.Bundle(f, newFile, filepath.Dir(path), relativeToDir, relativeDir, leftDelim, rightDelim); err != nil {
		return nil, err
	}

	var newName string

	dirname, filename := filepath.Split(path)
	origDir := dirname
	ext := filepath.Ext(filename)
	filename = filepath.Base(filename)

	if output != "" {
		dirname = output + string(filepath.Separator) + dirname + string(filepath.Separator)
	}

	newName = dirname + filename[0:strings.LastIndex(filename, ext)]
	origDir += filename[0:strings.LastIndex(filename, ext)]

	b, err := ioutil.ReadFile(newFile.Name())
	if err != nil {
		return nil, err
	}

	h := md5.New()
	h.Write(b)
	hash := string(h.Sum(nil))

	hashName := "-" + fmt.Sprintf("%x", hash) + ext
	newName += hashName
	origDir += hashName

	if err = os.MkdirAll(filepath.Dir(newName), os.FileMode(0777)); err != nil {
		return nil, err
	}

	if err = os.Rename(newFile.Name(), newName); err != nil {
		return nil, err
	}

	return &bundler.ProcessedFile{OriginalFilename: path, NewFilename: origDir}, nil
}

// LoadManifestFiles reads the manifest file generated by the Generate() command
// in Production mode and returns template.FuncMap for the provided RunMode
func LoadManifestFiles(dirname string, mode RunMode, relativeToDir bool, leftDelim string, rightDelim string) (template.FuncMap, error) {

	var f *os.File
	var err error

	if mode == Production {
		f, err = os.Open(dirname + manifestFile)
		if err != nil {
			return nil, err
		}
		defer f.Close()
	}

	return ProcessManifestFiles(f, dirname, mode, relativeToDir, leftDelim, rightDelim)
}

// ProcessManifestFiles reads an existing manifest file generated by the Generate() command
// in Production mode and returns template.FuncMap for the provided RunMode
func ProcessManifestFiles(manifest io.Reader, dirname string, mode RunMode, relativeToDir bool, leftDelim string, rightDelim string) (template.FuncMap, error) {

	mapped := map[string]string{}
	dirname = filepath.Clean(dirname) + string(os.PathSeparator)

	if mode == Production {

		mapped := map[string]string{}
		var files []string

		scanner := bufio.NewScanner(manifest)

		for scanner.Scan() {
			files = strings.SplitN(scanner.Text(), oldNewSeparator, 2)
			mapped[files[0]] = files[1]
		}
	}

	return loadMapFuncs(dirname, mode, relativeToDir, leftDelim, rightDelim, mapped), nil
}

func loadMapFuncs(dirname string, mode RunMode, relativeToDir bool, leftDelim string, rightDelim string, mapped map[string]string) template.FuncMap {

	funcs := template.FuncMap{}

	if mode == Production {
		funcs[cssHTMLTag] = createProdCSSTemplateFunc(mapped)
		funcs[jsHTMLTag] = createProdJSTemplateFunc(mapped)

		return funcs
	}

	funcs[cssHTMLTag] = createDevCSSTemplateFunc(dirname, relativeToDir, leftDelim, rightDelim)
	funcs[jsHTMLTag] = createDevJSTemplateFunc(dirname, relativeToDir, leftDelim, rightDelim)

	return funcs
}

func createProdCSSTemplateFunc(mapped map[string]string) interface{} {
	return func(name string) template.HTML {
		return template.HTML(fmt.Sprintf(cssTag, mapped[name]))
	}
}

func createProdJSTemplateFunc(mapped map[string]string) interface{} {
	return func(name string) template.HTML {
		return template.HTML(fmt.Sprintf(jsTag, mapped[name]))
	}
}

func createDevCSSTemplateFunc(dirname string, relativeToDir bool, leftDelim string, rightDelim string) interface{} {
	// custom lexer, bytesBuffer

	return func(name string) template.HTML {
		buff := new(bytes.Buffer)

		files, err := loadFromDelims(dirname, name, relativeToDir, dirname, leftDelim, rightDelim)
		if err != nil {
			panic(err)
		}

		for _, file := range files {
			buff.WriteString(fmt.Sprintf(cssTag, "/"+filepath.Clean(dirname)+file))
		}

		buff.WriteString(fmt.Sprintf(cssTag, "/"+dirname+name))

		return template.HTML(buff.String())
	}
}

func createDevJSTemplateFunc(dirname string, relativeToDir bool, leftDelim string, rightDelim string) interface{} {
	// custom lexer, bytesBuffer

	return func(name string) template.HTML {
		buff := new(bytes.Buffer)

		files, err := loadFromDelims(dirname, name, relativeToDir, dirname, leftDelim, rightDelim)
		if err != nil {
			panic(err)
		}

		for _, file := range files {
			buff.WriteString(fmt.Sprintf(jsTag, "/"+filepath.Clean(dirname)+file))
		}

		buff.WriteString(fmt.Sprintf(jsTag, "/"+dirname+name))

		return template.HTML(buff.String())
	}
}

func loadFromDelims(dirname string, name string, relativeToDir bool, relativeDir string, leftDelim string, rightDelim string) ([]string, error) {
	var err error
	var files []string
	var ok bool
	var path string

	existing := map[string]struct{}{}

	f, err := os.Open(dirname + name)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	l, err := bundler.NewLexer("assets-bundle", f, leftDelim, rightDelim)
	if err != nil {
		return nil, err
	}

LOOP:
	for {
		itm := l.NextItem()

		switch itm.Type {

		case bundler.ItemFile:

			if relativeToDir {
				path = filepath.FromSlash("/" + itm.Val)
			} else {
				path = filepath.FromSlash("/" + dirname + itm.Val)
			}

			if _, ok = existing[path]; !ok {
				files = append(files, path)
				existing[path] = struct{}{}
			}

			fls, err := loadFromDelims(dirname, itm.Val, relativeToDir, relativeDir, leftDelim, rightDelim)
			if err != nil {
				return nil, err
			}

			files = append(files, fls...)

		case bundler.ItemEOF:
			break LOOP
		case bundler.ItemError:
			return nil, errors.New(itm.Val)
		}
	}

	return files, nil
}
<|endoftext|>