<commit_before>// Copyright 2018-2022 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build !plan9
// +build !plan9

package session

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"

	"github.com/hashicorp/go-multierror"
	"github.com/u-root/cpu/mount"
	"github.com/u-root/u-root/pkg/termios"
	"golang.org/x/sys/unix"
)

// Bind defines a bind mount. It records the Local directory,
// e.g. /bin, and the remote directory, e.g. /tmp/cpu/bin.
type Bind struct {
	Local  string
	Remote string
}

// Session is one instance of a cpu session, started by a cpud.
type Session struct {
	restorer *termios.Termios
	Stdin    io.Reader
	Stdout   io.Writer
	Stderr   io.Writer
	binds    []Bind
	// Any function can use fail to mark that something
	// went badly wrong in some step. At that point, if wtf is set,
	// cpud will start it. This is incredibly handy for debugging.
	fail   bool
	msize  int
	mopts  string
	port9p string
	cmd    string
	args   []string
}

var (
	v = func(string, ...interface{}) {}
	// To get debugging when Things Go Wrong, you can run as, e.g., -wtf /bbin/elvish
	// or change the value here to /bbin/elvish.
	// This way, when Things Go Wrong, you'll be dropped into a shell and look around.
	// This is sometimes your only way to debug if there is (e.g.) a Go runtime
	// bug around unsharing. Which has happened.
	// This is compile time only because I'm so uncertain of whether it's dangerous
	wtf string
)

// DropPrivs drops privileges to the level of os.Getuid / os.Getgid
func (s *Session) DropPrivs() error {
	uid := unix.Getuid()
	v("CPUD:dropPrives: uid is %v", uid)
	if uid == 0 {
		v("CPUD:dropPrivs: not dropping privs")
		return nil
	}
	gid := unix.Getgid()
	v("CPUD:dropPrivs: gid is %v", gid)
	if err := unix.Setreuid(-1, uid); err != nil {
		return err
	}
	return unix.Setregid(-1, gid)
}

// Terminal sets up an interactive terminal.
func (s *Session) Terminal() error {
	// for some reason echo is not set.
	t, err := termios.New()
	if err != nil {
		return fmt.Errorf("CPUD:termios.New(): %v", err)
	}
	term, err := t.Get()
	if err != nil {
		return fmt.Errorf("CPUD:t.Get():%v", err)
	}
	s.restorer = term
	old := term.Lflag
	term.Lflag |= unix.ECHO | unix.ECHONL
	if err := t.Set(term); err != nil {
		return fmt.Errorf("CPUD:t.Set(%#x)(i.e. %#x | unix.ECHO | unix.ECHONL): %v", term.Lflag, old, err)
	}
	return nil
}

// TmpMounts sets up directories, and bind mounts, in /tmp/cpu.
// N.B. the /tmp/cpu mount is private assuming this program
// was started correctly with the namespace unshared (on Linux and
// Plan 9; on *BSD or Windows no such guarantees can be made).
//
// See the longer comment (rant) in session_linux.go
func (s *Session) TmpMounts() error {
	// It's true we are making this directory while still root.
	// This ought to be safe as it is a private namespace mount.
	// (or we are started with a clean namespace in Plan 9).
	for _, n := range []string{"/tmp/cpu", "/tmp/local", "/tmp/merge", "/tmp/root", "/home"} {
		if err := os.MkdirAll(n, 0666); err != nil && !os.IsExist(err) {
			log.Println(err)
		}
	}

	if err := osMounts(); err != nil {
		log.Println(err)
	}
	return nil
}

// Run starts up a remote cpu session. It is started by a cpu
// daemon via a -remote switch.
//
// This code assumes that cpud is running as init, or that
// an init has started a cpud, and that the code is running
// with a private namespace (CLONE_NEWNS on Linux; RFNAMEG on Plan9).
// On Linux, it starts as uid 0, and once the mount/bind is done,
// calls DropPrivs.
func (s *Session) Run() error {
	var errors error

	if err := runSetup(); err != nil {
		return err
	}
	if err := s.TmpMounts(); err != nil {
		v("CPUD: TmpMounts error: %v", err)
		s.fail = true
		errors = multierror.Append(err)
	}

	v("CPUD: Set up a namespace")
	if b, ok := os.LookupEnv("CPU_NAMESPACE"); ok {
		binds, err := ParseBinds(b)
		if err != nil {
			v("CPUD: ParseBind failed: %v", err)
			s.fail = true
			errors = multierror.Append(errors, err)
		}

		s.binds = binds
	}
	v("CPUD: call s.NameSpace")
	w, err := s.Namespace()
	if err != nil {
		return fmt.Errorf("CPUD:Namespace: warnings %v, err %v", w, multierror.Append(errors, err))
	}
	v("CPUD:warning: %v", w)

	v("CPUD: bind mounts done")

	// The CPU_FSTAB environment variable is, literally, an fstab.
	// Why an environment variable and not a file? We do not
	// want to require any 9p mounts at all. People should be able
	// to do this:
	// CPU_NAMESPACE="" CPU_FSTAB=`cat fstab`
	// and get the mounts they want. The first uses of this will
	// be building namespaces with drive and virtiofs.
	if tab, ok := os.LookupEnv("CPU_FSTAB"); ok {
		if err := mount.Mount(tab); err != nil {
			v("CPUD: fstab mount failure: %v", err)
			// Should we die if the mounts fail? For now, we think not;
			// the user may be able to debug if they have a non-empty
			// CPU_NAMESPACE. Just record that it failed.
			s.fail = true
		}
	}

	if err := s.Terminal(); err != nil {
		s.fail = true
		errors = multierror.Append(err)
	}
	v("CPUD: Terminal ready")
	if s.fail && len(wtf) != 0 {
		c := exec.Command(wtf)
		// Tricky question: should wtf use the stdio files or the ones
		// in the Server ... hmm.
		c.Stdin, c.Stdout, c.Stderr, c.Dir = os.Stdin, os.Stdout, os.Stderr, "/"
		log.Printf("CPUD: WTF: try to run %v", c)
		if err := c.Run(); err != nil {
			log.Printf("CPUD: Running %q failed: %v", wtf, err)
		}
		log.Printf("CPUD: WTF done")
		return errors
	}

	// We don't want to run as the wrong uid.
	if err := s.DropPrivs(); err != nil {
		return multierror.Append(errors, err)
	}

	// While it is true that things have been mounted, we need not
	// worry about unmounting them once the command is done: the
	// unmount happens for free since we unshared.
	v("CPUD:runRemote: command is %q", s.args)
	c := exec.Command(s.cmd, s.args...)
	c.Stdin, c.Stdout, c.Stderr, c.Dir = s.Stdin, s.Stdout, s.Stderr, os.Getenv("PWD")
	err = c.Run()
	v("CPUD:Run %v returns %v", c, err)
	if err != nil {
		if s.fail && len(wtf) != 0 {
			c := exec.Command(wtf)
			c.Stdin, c.Stdout, c.Stderr, c.Dir = os.Stdin, os.Stdout, os.Stderr, "/"
			log.Printf("CPUD: WTF: try to run %v", c)
			if err := c.Run(); err != nil {
				log.Printf("CPUD: Running %q failed: %v", wtf, err)
			}
			log.Printf("CPUD: WTF done: %v", err)
		}
	}
	return err
}

// New returns a New session with defaults set. It requires a port for
// 9p (which can be the empty string, but is usually not) and a
// command name.
func New(port9p, cmd string, args ...string) *Session {
	return &Session{msize: 8192, Stdin: os.Stdin, Stdout: os.Stdout, Stderr: os.Stderr, port9p: port9p, cmd: cmd, args: args}
}
<commit_msg>Error out loudly if PWD is not a directory<commit_after>// Copyright 2018-2022 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build !plan9
// +build !plan9

package session

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"

	"github.com/hashicorp/go-multierror"
	"github.com/u-root/cpu/mount"
	"github.com/u-root/u-root/pkg/termios"
	"golang.org/x/sys/unix"
)

// Bind defines a bind mount. It records the Local directory,
// e.g. /bin, and the remote directory, e.g. /tmp/cpu/bin.
type Bind struct {
	Local  string
	Remote string
}

// Session is one instance of a cpu session, started by a cpud.
type Session struct {
	restorer *termios.Termios
	Stdin    io.Reader
	Stdout   io.Writer
	Stderr   io.Writer
	binds    []Bind
	// Any function can use fail to mark that something
	// went badly wrong in some step. At that point, if wtf is set,
	// cpud will start it. This is incredibly handy for debugging.
	fail   bool
	msize  int
	mopts  string
	port9p string
	cmd    string
	args   []string
}

var (
	v = func(string, ...interface{}) {}
	// To get debugging when Things Go Wrong, you can run as, e.g., -wtf /bbin/elvish
	// or change the value here to /bbin/elvish.
	// This way, when Things Go Wrong, you'll be dropped into a shell and look around.
	// This is sometimes your only way to debug if there is (e.g.) a Go runtime
	// bug around unsharing. Which has happened.
	// This is compile time only because I'm so uncertain of whether it's dangerous
	wtf string
)

// DropPrivs drops privileges to the level of os.Getuid / os.Getgid
func (s *Session) DropPrivs() error {
	uid := unix.Getuid()
	v("CPUD:dropPrives: uid is %v", uid)
	if uid == 0 {
		v("CPUD:dropPrivs: not dropping privs")
		return nil
	}
	gid := unix.Getgid()
	v("CPUD:dropPrivs: gid is %v", gid)
	if err := unix.Setreuid(-1, uid); err != nil {
		return err
	}
	return unix.Setregid(-1, gid)
}

// Terminal sets up an interactive terminal.
func (s *Session) Terminal() error {
	// for some reason echo is not set.
	t, err := termios.New()
	if err != nil {
		return fmt.Errorf("CPUD:termios.New(): %v", err)
	}
	term, err := t.Get()
	if err != nil {
		return fmt.Errorf("CPUD:t.Get():%v", err)
	}
	s.restorer = term
	old := term.Lflag
	term.Lflag |= unix.ECHO | unix.ECHONL
	if err := t.Set(term); err != nil {
		return fmt.Errorf("CPUD:t.Set(%#x)(i.e. %#x | unix.ECHO | unix.ECHONL): %v", term.Lflag, old, err)
	}
	return nil
}

// TmpMounts sets up directories, and bind mounts, in /tmp/cpu.
// N.B. the /tmp/cpu mount is private assuming this program
// was started correctly with the namespace unshared (on Linux and
// Plan 9; on *BSD or Windows no such guarantees can be made).
//
// See the longer comment (rant) in session_linux.go
func (s *Session) TmpMounts() error {
	// It's true we are making this directory while still root.
	// This ought to be safe as it is a private namespace mount.
	// (or we are started with a clean namespace in Plan 9).
	for _, n := range []string{"/tmp/cpu", "/tmp/local", "/tmp/merge", "/tmp/root", "/home"} {
		if err := os.MkdirAll(n, 0666); err != nil && !os.IsExist(err) {
			log.Println(err)
		}
	}

	if err := osMounts(); err != nil {
		log.Println(err)
	}
	return nil
}

// Run starts up a remote cpu session. It is started by a cpu
// daemon via a -remote switch.
//
// This code assumes that cpud is running as init, or that
// an init has started a cpud, and that the code is running
// with a private namespace (CLONE_NEWNS on Linux; RFNAMEG on Plan9).
// On Linux, it starts as uid 0, and once the mount/bind is done,
// calls DropPrivs.
func (s *Session) Run() error {
	var errors error

	if err := runSetup(); err != nil {
		return err
	}
	if err := s.TmpMounts(); err != nil {
		v("CPUD: TmpMounts error: %v", err)
		s.fail = true
		errors = multierror.Append(err)
	}

	v("CPUD: Set up a namespace")
	if b, ok := os.LookupEnv("CPU_NAMESPACE"); ok {
		binds, err := ParseBinds(b)
		if err != nil {
			v("CPUD: ParseBind failed: %v", err)
			s.fail = true
			errors = multierror.Append(errors, err)
		}

		s.binds = binds
	}
	v("CPUD: call s.NameSpace")
	w, err := s.Namespace()
	if err != nil {
		return fmt.Errorf("CPUD:Namespace: warnings %v, err %v", w, multierror.Append(errors, err))
	}
	v("CPUD:warning: %v", w)

	v("CPUD: bind mounts done")

	// The CPU_FSTAB environment variable is, literally, an fstab.
	// Why an environment variable and not a file? We do not
	// want to require any 9p mounts at all. People should be able
	// to do this:
	// CPU_NAMESPACE="" CPU_FSTAB=`cat fstab`
	// and get the mounts they want. The first uses of this will
	// be building namespaces with drive and virtiofs.
	if tab, ok := os.LookupEnv("CPU_FSTAB"); ok {
		if err := mount.Mount(tab); err != nil {
			v("CPUD: fstab mount failure: %v", err)
			// Should we die if the mounts fail? For now, we think not;
			// the user may be able to debug if they have a non-empty
			// CPU_NAMESPACE. Just record that it failed.
			s.fail = true
		}
	}

	if err := s.Terminal(); err != nil {
		s.fail = true
		errors = multierror.Append(err)
	}
	v("CPUD: Terminal ready")
	if s.fail && len(wtf) != 0 {
		c := exec.Command(wtf)
		// Tricky question: should wtf use the stdio files or the ones
		// in the Server ... hmm.
		c.Stdin, c.Stdout, c.Stderr, c.Dir = os.Stdin, os.Stdout, os.Stderr, "/"
		log.Printf("CPUD: WTF: try to run %v", c)
		if err := c.Run(); err != nil {
			log.Printf("CPUD: Running %q failed: %v", wtf, err)
		}
		log.Printf("CPUD: WTF done")
		return errors
	}

	// We don't want to run as the wrong uid.
	if err := s.DropPrivs(); err != nil {
		return multierror.Append(errors, err)
	}

	// While it is true that things have been mounted, we need not
	// worry about unmounting them once the command is done: the
	// unmount happens for free since we unshared.
	v("CPUD:runRemote: command is %q", s.args)
	c := exec.Command(s.cmd, s.args...)
	c.Stdin, c.Stdout, c.Stderr, c.Dir = s.Stdin, s.Stdout, s.Stderr, os.Getenv("PWD")
	dirInfo, err := os.Stat(c.Dir)
	if err != nil || !dirInfo.IsDir() {
		log.Printf("CPUD: your $PWD %s is not in the remote namespace", c.Dir)
		return os.ErrNotExist
	}
	err = c.Run()
	v("CPUD:Run %v returns %v", c, err)
	if err != nil {
		if s.fail && len(wtf) != 0 {
			c := exec.Command(wtf)
			c.Stdin, c.Stdout, c.Stderr, c.Dir = os.Stdin, os.Stdout, os.Stderr, "/"
			log.Printf("CPUD: WTF: try to run %v", c)
			if err := c.Run(); err != nil {
				log.Printf("CPUD: Running %q failed: %v", wtf, err)
			}
			log.Printf("CPUD: WTF done: %v", err)
		}
	}
	return err
}

// New returns a New session with defaults set. It requires a port for
// 9p (which can be the empty string, but is usually not) and a
// command name.
func New(port9p, cmd string, args ...string) *Session {
	return &Session{msize: 8192, Stdin: os.Stdin, Stdout: os.Stdout, Stderr: os.Stderr, port9p: port9p, cmd: cmd, args: args}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
)

const (
	errorMsg = "Path parameter must be a number from 1 to 100."
)

func handler(w http.ResponseWriter, r *http.Request) {
	if s, ok := cache[r.URL.Path[1:]]; ok {
		fmt.Fprint(w, s)
		return
	}
	fmt.Fprint(w, errorMsg)
}

var cache map[string]string

func main() {
	args := struct{ port string }{}
	flag.StringVar(&args.port, "port", "80", "The port to serve on.")
	flag.Parse()
	cache = make(map[string]string, 100)
	for i := 0; i < 101; i++ {
		s := strconv.Itoa(i)
		t := s
		f, b := i%3 == 0, i%5 == 0
		if f || b {
			t = ""
		}
		if f {
			t = "Fizz"
		}
		if b {
			t += "Buzz"
		}
		cache[s] = t
	}
	http.HandleFunc("/", handler)
	if err := http.ListenAndServe(args.port, nil); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Add fast CGI support<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/fcgi"
	"strconv"
)

const (
	errorMsg = "Path parameter must be a number from 1 to 100."
)

func handler(w http.ResponseWriter, r *http.Request) {
	if s, ok := cache[r.URL.Path[1:]]; ok {
		fmt.Fprint(w, s)
		return
	}
	fmt.Fprint(w, errorMsg)
}

var cache map[string]string

func main() {
	args := struct{ port string }{}
	flag.StringVar(&args.port, "port", "", "The port to serve on.")
	flag.Parse()
	cache = make(map[string]string, 100)
	for i := 0; i < 101; i++ {
		s := strconv.Itoa(i)
		t := s
		f, b := i%3 == 0, i%5 == 0
		if f || b {
			t = ""
		}
		if f {
			t = "Fizz"
		}
		if b {
			t += "Buzz"
		}
		cache[s] = t
	}
	http.HandleFunc("/", handler)
	switch {
	case args.port != "":
		if err := http.ListenAndServe(args.port, nil); err != nil {
			log.Fatal(err)
		}
	default:
		if err := fcgi.Serve(nil, nil); err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>/****************************************************************************
 * This file is part of Builder.
 *
 * Copyright (C) 2015 Pier Luigi Fiorini
 *
 * Author(s):
 *    Pier Luigi Fiorini <pierluigi.fiorini@gmail.com>
 *
 * $BEGIN_LICENSE:AGPL3+$
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * $END_LICENSE$
 ***************************************************************************/

package database

import (
	"bytes"
	"encoding/json"
	"github.com/boltdb/bolt"
)

type Package struct {
	Name          string   `json:"name"`
	Architectures []string `json:"architectures"`
	Ci            bool     `json:"ci"`
	Vcs           VcsInfo  `json:"vcs"`
	UpstreamVcs   VcsInfo  `json:"upstream_vcs"`
}

// Return whether the package was stored into the db.
func (db *Database) HasPackage(name string) bool {
	var found bool = false
	db.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte("package")).Cursor()
		for k, _ := c.Seek([]byte(name)); bytes.Equal(k, []byte(name)); k, _ = c.Next() {
			found = true
			return nil
		}
		return nil
	})
	return found
}

// Return a list of package names.
func (db *Database) GetPackageNames() []string {
	var list = []string{}
	db.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		bucket.ForEach(func(k, v []byte) error {
			list = append(list, string(k))
			return nil
		})
		return nil
	})
	return list
}

// Return a list of all packages.
func (db *Database) ListAllPackages() []*Package {
	var list []*Package
	db.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		bucket.ForEach(func(k, v []byte) error {
			pkg := &Package{}
			json.Unmarshal(v, &pkg)
			list = append(list, pkg)
			return nil
		})
		return nil
	})
	return list
}

// Return a package from the database.
func (db *Database) GetPackage(name string) *Package {
	var pkg *Package = nil
	db.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte("package")).Cursor()
		for k, v := c.Seek([]byte(name)); bytes.Equal(k, []byte(name)); k, v = c.Next() {
			json.Unmarshal(v, &pkg)
			return nil
		}
		return nil
	})
	return pkg
}

// Add a package to the database.
func (db *Database) AddPackage(pkg *Package) error {
	encoded, err := json.Marshal(pkg)
	if err != nil {
		return nil
	}
	return db.db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte("package"))
		if err != nil {
			return err
		}

		err = bucket.Put([]byte(pkg.Name), encoded)
		if err != nil {
			return err
		}

		return nil
	})
}

// Remove a package from the database.
func (db *Database) RemovePackage(name string) error {
	return db.db.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		err := bucket.Delete([]byte(name))
		if err != nil {
			return err
		}

		return nil
	})
}
<commit_msg>Do not crash if "package" bucket was not created<commit_after>/****************************************************************************
 * This file is part of Builder.
 *
 * Copyright (C) 2015 Pier Luigi Fiorini
 *
 * Author(s):
 *    Pier Luigi Fiorini <pierluigi.fiorini@gmail.com>
 *
 * $BEGIN_LICENSE:AGPL3+$
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * $END_LICENSE$
 ***************************************************************************/

package database

import (
	"bytes"
	"encoding/json"
	"github.com/boltdb/bolt"
)

type Package struct {
	Name          string   `json:"name"`
	Architectures []string `json:"architectures"`
	Ci            bool     `json:"ci"`
	Vcs           VcsInfo  `json:"vcs"`
	UpstreamVcs   VcsInfo  `json:"upstream_vcs"`
}

// Return whether the package was stored into the db.
func (db *Database) HasPackage(name string) bool {
	var found bool = false
	db.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		if bucket == nil {
			return nil
		}

		c := bucket.Cursor()
		for k, _ := c.Seek([]byte(name)); bytes.Equal(k, []byte(name)); k, _ = c.Next() {
			found = true
			return nil
		}
		return nil
	})
	return found
}

// Return a list of package names.
func (db *Database) GetPackageNames() []string {
	var list = []string{}
	db.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		if bucket == nil {
			return nil
		}

		bucket.ForEach(func(k, v []byte) error {
			list = append(list, string(k))
			return nil
		})
		return nil
	})
	return list
}

// Return a list of all packages.
func (db *Database) ListAllPackages() []*Package {
	var list []*Package
	db.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		if bucket == nil {
			return nil
		}

		bucket.ForEach(func(k, v []byte) error {
			pkg := &Package{}
			json.Unmarshal(v, &pkg)
			list = append(list, pkg)
			return nil
		})
		return nil
	})
	return list
}

// Return a package from the database.
func (db *Database) GetPackage(name string) *Package {
	var pkg *Package = nil
	db.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		if bucket == nil {
			return nil
		}

		c := bucket.Cursor()
		for k, v := c.Seek([]byte(name)); bytes.Equal(k, []byte(name)); k, v = c.Next() {
			json.Unmarshal(v, &pkg)
			return nil
		}
		return nil
	})
	return pkg
}

// Add a package to the database.
func (db *Database) AddPackage(pkg *Package) error {
	encoded, err := json.Marshal(pkg)
	if err != nil {
		return nil
	}
	return db.db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte("package"))
		if err != nil {
			return err
		}

		err = bucket.Put([]byte(pkg.Name), encoded)
		if err != nil {
			return err
		}

		return nil
	})
}

// Remove a package from the database.
func (db *Database) RemovePackage(name string) error {
	return db.db.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("package"))
		if bucket == nil {
			return nil
		}

		err := bucket.Delete([]byte(name))
		if err != nil {
			return err
		}

		return nil
	})
}
<|endoftext|>


<commit_before>package command_registry

import (
	"fmt"
	"os"
	"time"

	"github.com/cloudfoundry/cli/cf/actors"
	"github.com/cloudfoundry/cli/cf/actors/broker_builder"
	"github.com/cloudfoundry/cli/cf/actors/plan_builder"
	"github.com/cloudfoundry/cli/cf/actors/plugin_repo"
	"github.com/cloudfoundry/cli/cf/actors/service_builder"
	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/app_files"
	"github.com/cloudfoundry/cli/cf/configuration/config_helpers"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	"github.com/cloudfoundry/cli/cf/manifest"
	"github.com/cloudfoundry/cli/cf/net"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/cli/cf/trace"
	"github.com/cloudfoundry/cli/plugin/models"
	"github.com/cloudfoundry/cli/utils"
	"github.com/cloudfoundry/cli/words/generator"
)

type Dependency struct {
	Ui                 terminal.UI
	Config             core_config.Repository
	RepoLocator        api.RepositoryLocator
	PluginConfig       plugin_config.PluginConfiguration
	ManifestRepo       manifest.ManifestRepository
	AppManifest        manifest.AppManifest
	Gateways           map[string]net.Gateway
	TeePrinter         *terminal.TeePrinter
	PluginRepo         plugin_repo.PluginRepo
	PluginModels       *pluginModels
	ServiceBuilder     service_builder.ServiceBuilder
	BrokerBuilder      broker_builder.Builder
	PlanBuilder        plan_builder.PlanBuilder
	ServiceHandler     actors.ServiceActor
	ServicePlanHandler actors.ServicePlanActor
	WordGenerator      generator.WordGenerator
	AppZipper          app_files.Zipper
	AppFiles           app_files.AppFiles
	PushActor          actors.PushActor
	ChecksumUtil       utils.Sha1Checksum
	WilecardDependency interface{} //use for injecting fakes
}

type pluginModels struct {
	Application   *plugin_models.GetAppModel
	AppsSummary   *[]plugin_models.GetAppsModel
	Organizations *[]plugin_models.GetOrgs_Model
	Organization  *plugin_models.GetOrg_Model
	Spaces        *[]plugin_models.GetSpaces_Model
	Space         *plugin_models.GetSpace_Model
	OrgUsers      *[]plugin_models.GetOrgUsers_Model
	SpaceUsers    *[]plugin_models.GetSpaceUsers_Model
	Services      *[]plugin_models.GetServices_Model
	Service       *plugin_models.GetService_Model
	OauthToken    *plugin_models.GetOauthToken_Model
}

func NewDependency() Dependency {
	deps := Dependency{}
	deps.TeePrinter = terminal.NewTeePrinter()
	deps.Ui = terminal.NewUI(os.Stdin, deps.TeePrinter)
	deps.ManifestRepo = manifest.NewManifestDiskRepository()
	deps.AppManifest = manifest.NewGenerator()

	errorHandler := func(err error) {
		if err != nil {
			deps.Ui.Failed(fmt.Sprintf("Config error: %s", err))
		}
	}
	deps.Config = core_config.NewRepositoryFromFilepath(config_helpers.DefaultFilePath(), errorHandler)
	deps.PluginConfig = plugin_config.NewPluginConfig(errorHandler)

	terminal.UserAskedForColors = deps.Config.ColorEnabled()
	terminal.InitColorSupport()

	if os.Getenv("CF_TRACE") != "" {
		trace.Logger = trace.NewLogger(os.Getenv("CF_TRACE"))
	} else {
		trace.Logger = trace.NewLogger(deps.Config.Trace())
	}

	deps.Gateways = map[string]net.Gateway{
		"cloud-controller": net.NewCloudControllerGateway(deps.Config, time.Now, deps.Ui),
		"uaa":              net.NewUAAGateway(deps.Config, deps.Ui),
		"routing-api":      net.NewRoutingApiGateway(deps.Config, time.Now, deps.Ui),
	}
	deps.RepoLocator = api.NewRepositoryLocator(deps.Config, deps.Gateways)

	deps.PluginModels = &pluginModels{Application: nil}

	deps.PlanBuilder = plan_builder.NewBuilder(
		deps.RepoLocator.GetServicePlanRepository(),
		deps.RepoLocator.GetServicePlanVisibilityRepository(),
		deps.RepoLocator.GetOrganizationRepository(),
	)

	deps.ServiceBuilder = service_builder.NewBuilder(
		deps.RepoLocator.GetServiceRepository(),
		deps.PlanBuilder,
	)

	deps.BrokerBuilder = broker_builder.NewBuilder(
		deps.RepoLocator.GetServiceBrokerRepository(),
		deps.ServiceBuilder,
	)

	deps.PluginRepo = plugin_repo.NewPluginRepo()

	deps.ServiceHandler = actors.NewServiceHandler(
		deps.RepoLocator.GetOrganizationRepository(),
		deps.BrokerBuilder,
		deps.ServiceBuilder,
	)

	deps.ServicePlanHandler = actors.NewServicePlanHandler(
		deps.RepoLocator.GetServicePlanRepository(),
		deps.RepoLocator.GetServicePlanVisibilityRepository(),
		deps.RepoLocator.GetOrganizationRepository(),
		deps.PlanBuilder,
		deps.ServiceBuilder,
	)

	deps.WordGenerator = generator.NewWordGenerator()

	deps.AppZipper = app_files.ApplicationZipper{}
	deps.AppFiles = app_files.ApplicationFiles{}

	deps.PushActor = actors.NewPushActor(deps.RepoLocator.GetApplicationBitsRepository(), deps.AppZipper, deps.AppFiles)

	deps.ChecksumUtil = utils.NewSha1Checksum("")

	return deps
}
<commit_msg>Export PluginModels<commit_after>package command_registry

import (
	"fmt"
	"os"
	"time"

	"github.com/cloudfoundry/cli/cf/actors"
	"github.com/cloudfoundry/cli/cf/actors/broker_builder"
	"github.com/cloudfoundry/cli/cf/actors/plan_builder"
	"github.com/cloudfoundry/cli/cf/actors/plugin_repo"
	"github.com/cloudfoundry/cli/cf/actors/service_builder"
	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/app_files"
	"github.com/cloudfoundry/cli/cf/configuration/config_helpers"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	"github.com/cloudfoundry/cli/cf/manifest"
	"github.com/cloudfoundry/cli/cf/net"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/cli/cf/trace"
	"github.com/cloudfoundry/cli/plugin/models"
	"github.com/cloudfoundry/cli/utils"
	"github.com/cloudfoundry/cli/words/generator"
)

type Dependency struct {
	Ui                 terminal.UI
	Config             core_config.Repository
	RepoLocator        api.RepositoryLocator
	PluginConfig       plugin_config.PluginConfiguration
	ManifestRepo       manifest.ManifestRepository
	AppManifest        manifest.AppManifest
	Gateways           map[string]net.Gateway
	TeePrinter         *terminal.TeePrinter
	PluginRepo         plugin_repo.PluginRepo
	PluginModels       *PluginModels
	ServiceBuilder     service_builder.ServiceBuilder
	BrokerBuilder      broker_builder.Builder
	PlanBuilder        plan_builder.PlanBuilder
	ServiceHandler     actors.ServiceActor
	ServicePlanHandler actors.ServicePlanActor
	WordGenerator      generator.WordGenerator
	AppZipper          app_files.Zipper
	AppFiles           app_files.AppFiles
	PushActor          actors.PushActor
	ChecksumUtil       utils.Sha1Checksum
	WilecardDependency interface{} //use for injecting fakes
}

type PluginModels struct {
	Application   *plugin_models.GetAppModel
	AppsSummary   *[]plugin_models.GetAppsModel
	Organizations *[]plugin_models.GetOrgs_Model
	Organization  *plugin_models.GetOrg_Model
	Spaces        *[]plugin_models.GetSpaces_Model
	Space         *plugin_models.GetSpace_Model
	OrgUsers      *[]plugin_models.GetOrgUsers_Model
	SpaceUsers    *[]plugin_models.GetSpaceUsers_Model
	Services      *[]plugin_models.GetServices_Model
	Service       *plugin_models.GetService_Model
	OauthToken    *plugin_models.GetOauthToken_Model
}

func NewDependency() Dependency {
	deps := Dependency{}
	deps.TeePrinter = terminal.NewTeePrinter()
	deps.Ui = terminal.NewUI(os.Stdin, deps.TeePrinter)
	deps.ManifestRepo = manifest.NewManifestDiskRepository()
	deps.AppManifest = manifest.NewGenerator()

	errorHandler := func(err error) {
		if err != nil {
			deps.Ui.Failed(fmt.Sprintf("Config error: %s", err))
		}
	}
	deps.Config = core_config.NewRepositoryFromFilepath(config_helpers.DefaultFilePath(), errorHandler)
	deps.PluginConfig = plugin_config.NewPluginConfig(errorHandler)

	terminal.UserAskedForColors = deps.Config.ColorEnabled()
	terminal.InitColorSupport()

	if os.Getenv("CF_TRACE") != "" {
		trace.Logger = trace.NewLogger(os.Getenv("CF_TRACE"))
	} else {
		trace.Logger = trace.NewLogger(deps.Config.Trace())
	}

	deps.Gateways = map[string]net.Gateway{
		"cloud-controller": net.NewCloudControllerGateway(deps.Config, time.Now, deps.Ui),
		"uaa":              net.NewUAAGateway(deps.Config, deps.Ui),
		"routing-api":      net.NewRoutingApiGateway(deps.Config, time.Now, deps.Ui),
	}
	deps.RepoLocator = api.NewRepositoryLocator(deps.Config, deps.Gateways)

	deps.PluginModels = &PluginModels{Application: nil}

	deps.PlanBuilder = plan_builder.NewBuilder(
		deps.RepoLocator.GetServicePlanRepository(),
		deps.RepoLocator.GetServicePlanVisibilityRepository(),
		deps.RepoLocator.GetOrganizationRepository(),
	)

	deps.ServiceBuilder = service_builder.NewBuilder(
		deps.RepoLocator.GetServiceRepository(),
		deps.PlanBuilder,
	)

	deps.BrokerBuilder = broker_builder.NewBuilder(
		deps.RepoLocator.GetServiceBrokerRepository(),
		deps.ServiceBuilder,
	)

	deps.PluginRepo = plugin_repo.NewPluginRepo()

	deps.ServiceHandler = actors.NewServiceHandler(
		deps.RepoLocator.GetOrganizationRepository(),
		deps.BrokerBuilder,
		deps.ServiceBuilder,
	)

	deps.ServicePlanHandler = actors.NewServicePlanHandler(
		deps.RepoLocator.GetServicePlanRepository(),
		deps.RepoLocator.GetServicePlanVisibilityRepository(),
		deps.RepoLocator.GetOrganizationRepository(),
		deps.PlanBuilder,
		deps.ServiceBuilder,
	)

	deps.WordGenerator = generator.NewWordGenerator()

	deps.AppZipper = app_files.ApplicationZipper{}
	deps.AppFiles = app_files.ApplicationFiles{}

	deps.PushActor = actors.NewPushActor(deps.RepoLocator.GetApplicationBitsRepository(), deps.AppZipper, deps.AppFiles)

	deps.ChecksumUtil = utils.NewSha1Checksum("")

	return deps
}
<|endoftext|>


<commit_before>package server

import (
	"bufio"
	"bytes"
	"fmt"
	"net"
	"os"
	"time"

	"github.com/cashshuffle/cashshuffle/message"
	"github.com/golang/protobuf/proto"
)

const (
	maxMessageLength = 64 * 1024
)

var (
	// breakBytes are the bytes that delimit each protobuf message
	// This represents the character ⏎
	breakBytes = []byte{226, 143, 142}
)

// startSignedChan starts a loop reading messages.
func startSignedChan(c chan *signedConn) {
	for {
		sc := <-c
		err := sc.processReceivedMessage()
		if err != nil {
			sc.conn.Close()
			fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
		}
	}
}

// processReceivedMessage reads the message and processes it.
func (sc *signedConn) processReceivedMessage() error {
	// If we are not tracking the connection yet, the user must be
	// registering with the server.
	if sc.tracker.getTrackerData(sc.conn) == nil {
		err := sc.registerClient()
		if err != nil {
			return err
		}

		playerData := sc.tracker.getTrackerData(sc.conn)

		if sc.tracker.getPoolSize(playerData.pool) == sc.tracker.poolSize {
			time.Sleep(50 * time.Millisecond)
			sc.announceStart()
		}

		return nil
	}

	if err := sc.verifyMessage(); err != nil {
		return err
	}

	err := sc.broadcastMessage()
	return err
}

// processMessages reads messages from the connection and begins processing.
func processMessages(conn net.Conn, c chan *signedConn, t *tracker) {
	scanner := bufio.NewScanner(conn)
	scanner.Split(bufio.ScanRunes)

	var b bytes.Buffer

	for {
		for scanner.Scan() {
			scanBytes := scanner.Bytes()

			if breakScan(scanBytes) {
				break
			}

			if len(b.String()) > maxMessageLength {
				fmt.Fprintln(os.Stderr, "[Error] message too long")
				return
			}

			b.Write(scanBytes)
		}

		if err := scanner.Err(); err != nil {
			fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
			break
		}

		// We should not receive empty messages.
		if b.String() == "" {
			break
		}

		if err := sendToSignedChan(&b, conn, c, t); err != nil {
			fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
			break
		}
	}
}

// sendToSignedChannel takes a byte buffer containing a protobuf message,
// converts it to message.Signed and sends it over signedChan.
func sendToSignedChan(b *bytes.Buffer, conn net.Conn, c chan *signedConn, t *tracker) error {
	defer b.Reset()

	pdata := new(message.Packets)

	err := proto.Unmarshal(b.Bytes(), pdata)
	if err != nil {
		return err
	}

	if debugMode {
		fmt.Println("RECEIVED", pdata)
	}

	for _, signed := range pdata.Packet {
		data := &signedConn{
			message: signed,
			conn:    conn,
			tracker: t,
		}

		c <- data
	}

	return nil
}

// breakScan checks if a byte sequence is the break point on the scanner.
func breakScan(bs []byte) bool {
	if len(bs) == 3 {
		for i := range bs {
			if bs[i] != breakBytes[i] {
				return false
			}
		}

		return true
	}

	return false
}
<commit_msg>Remove sleep<commit_after>package server

import (
	"bufio"
	"bytes"
	"fmt"
	"net"
	"os"

	"github.com/cashshuffle/cashshuffle/message"
	"github.com/golang/protobuf/proto"
)

const (
	maxMessageLength = 64 * 1024
)

var (
	// breakBytes are the bytes that delimit each protobuf message
	// This represents the character ⏎
	breakBytes = []byte{226, 143, 142}
)

// startSignedChan starts a loop reading messages.
func startSignedChan(c chan *signedConn) {
	for {
		sc := <-c
		err := sc.processReceivedMessage()
		if err != nil {
			sc.conn.Close()
			fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
		}
	}
}

// processReceivedMessage reads the message and processes it.
func (sc *signedConn) processReceivedMessage() error {
	// If we are not tracking the connection yet, the user must be
	// registering with the server.
	if sc.tracker.getTrackerData(sc.conn) == nil {
		err := sc.registerClient()
		if err != nil {
			return err
		}

		playerData := sc.tracker.getTrackerData(sc.conn)

		if sc.tracker.getPoolSize(playerData.pool) == sc.tracker.poolSize {
			sc.announceStart()
		}

		return nil
	}

	if err := sc.verifyMessage(); err != nil {
		return err
	}

	err := sc.broadcastMessage()
	return err
}

// processMessages reads messages from the connection and begins processing.
func processMessages(conn net.Conn, c chan *signedConn, t *tracker) {
	scanner := bufio.NewScanner(conn)
	scanner.Split(bufio.ScanRunes)

	var b bytes.Buffer

	for {
		for scanner.Scan() {
			scanBytes := scanner.Bytes()

			if breakScan(scanBytes) {
				break
			}

			if len(b.String()) > maxMessageLength {
				fmt.Fprintln(os.Stderr, "[Error] message too long")
				return
			}

			b.Write(scanBytes)
		}

		if err := scanner.Err(); err != nil {
			fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
			break
		}

		// We should not receive empty messages.
		if b.String() == "" {
			break
		}

		if err := sendToSignedChan(&b, conn, c, t); err != nil {
			fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
			break
		}
	}
}

// sendToSignedChannel takes a byte buffer containing a protobuf message,
// converts it to message.Signed and sends it over signedChan.
func sendToSignedChan(b *bytes.Buffer, conn net.Conn, c chan *signedConn, t *tracker) error {
	defer b.Reset()

	pdata := new(message.Packets)

	err := proto.Unmarshal(b.Bytes(), pdata)
	if err != nil {
		return err
	}

	if debugMode {
		fmt.Println("RECEIVED", pdata)
	}

	for _, signed := range pdata.Packet {
		data := &signedConn{
			message: signed,
			conn:    conn,
			tracker: t,
		}

		c <- data
	}

	return nil
}

// breakScan checks if a byte sequence is the break point on the scanner.
func breakScan(bs []byte) bool {
	if len(bs) == 3 {
		for i := range bs {
			if bs[i] != breakBytes[i] {
				return false
			}
		}

		return true
	}

	return false
}
<|endoftext|>


<commit_before>package gforms

import (
	"bytes"
	"reflect"
)

type Field interface {
	Clean(Data) (*V, error)
	Validate(*V, CleanedData) error
	Html(RawData) string
	html(...string) string
	GetName() string
	GetWigdet() Widget
}

type ValidationError interface {
	Error() string
}

type BaseField struct {
	name       string
	validators Validators
	Widget     Widget
	Field
}

func (self *BaseField) GetName() string {
	return self.name
}

func (self *BaseField) GetWigdet() Widget {
	return self.Widget
}

func (self *BaseField) Clean(data Data) (*V, error) {
	m, hasField := data[self.GetName()]
	if hasField {
		v := m.rawValueAsString()
		m.Kind = reflect.String
		if v != nil {
			m.Value = *v
			m.IsNil = false
			return m, nil
		}
	}
	return nilV(), nil
}

func (self *BaseField) Validate(value *V, cleanedData CleanedData) error {
	if self.validators == nil {
		return nil
	}
	for _, v := range self.validators {
		err := v.Validate(value, cleanedData)
		if err != nil {
			return err
		}
	}
	return nil
}

func fieldToHtml(field Field, rd RawData) string {
	v, hasField := rd[field.GetName()]
	if field.GetWigdet() == nil {
		if hasField {
			return field.html(v)
		} else {
			return field.html()
		}
	} else {
		if hasField {
			return field.GetWigdet().html(field, v)
		} else {
			return field.GetWigdet().html(field)
		}
	}
}

type templateContext struct {
	Field Field
	Value string
}

func newTemplateContext(field Field, vs ...string) templateContext {
	ctx := templateContext{
		Field: field,
	}
	if len(vs) > 0 {
		ctx.Value = vs[0]
	}
	return ctx
}

func renderTemplate(name string, ctx interface{}) string {
	var buffer bytes.Buffer
	Template.ExecuteTemplate(&buffer, name, ctx)
	return buffer.String()
}
<commit_msg>Handle parsing error for field templates.<commit_after>package gforms

import (
	"bytes"
	"reflect"
)

type Field interface {
	Clean(Data) (*V, error)
	Validate(*V, CleanedData) error
	Html(RawData) string
	html(...string) string
	GetName() string
	GetWigdet() Widget
}

type ValidationError interface {
	Error() string
}

type BaseField struct {
	name       string
	validators Validators
	Widget     Widget
	Field
}

func (self *BaseField) GetName() string {
	return self.name
}

func (self *BaseField) GetWigdet() Widget {
	return self.Widget
}

func (self *BaseField) Clean(data Data) (*V, error) {
	m, hasField := data[self.GetName()]
	if hasField {
		v := m.rawValueAsString()
		m.Kind = reflect.String
		if v != nil {
			m.Value = *v
			m.IsNil = false
			return m, nil
		}
	}
	return nilV(), nil
}

func (self *BaseField) Validate(value *V, cleanedData CleanedData) error {
	if self.validators == nil {
		return nil
	}
	for _, v := range self.validators {
		err := v.Validate(value, cleanedData)
		if err != nil {
			return err
		}
	}
	return nil
}

func fieldToHtml(field Field, rd RawData) string {
	v, hasField := rd[field.GetName()]
	if field.GetWigdet() == nil {
		if hasField {
			return field.html(v)
		} else {
			return field.html()
		}
	} else {
		if hasField {
			return field.GetWigdet().html(field, v)
		} else {
			return field.GetWigdet().html(field)
		}
	}
}

type templateContext struct {
	Field Field
	Value string
}

func newTemplateContext(field Field, vs ...string) templateContext {
	ctx := templateContext{
		Field: field,
	}
	if len(vs) > 0 {
		ctx.Value = vs[0]
	}
	return ctx
}

func renderTemplate(name string, ctx interface{}) string {
	var buffer bytes.Buffer
	err := Template.ExecuteTemplate(&buffer, name, ctx)
	if err != nil {
		panic(err)
	}
	return buffer.String()
}
<|endoftext|>


<commit_before>package scroll

import (
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mailgun/log"
)

// Retrieve a POST request field as a string.
// Returns `MissingFieldError` if requested field is missing.
func GetStringField(r *http.Request, fieldName string) (string, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return "", MissingFieldError{fieldName}
	}
	return r.FormValue(fieldName), nil
}

// Retrieve a POST request field as a string.
// If the requested field is missing, returns provided default value.
func GetStringFieldWithDefault(r *http.Request, fieldName, defaultValue string) string {
	if fieldValue, err := GetStringField(r, fieldName); err == nil {
		return fieldValue
	}
	return defaultValue
}

// A multiParamRegex is used to convert Ruby and PHP style array params.
// PHP uses ["param[0]", "param[1]",..] instead of ["param", "param",..]
// Ruby uses ["param[]", "param[]",..]
var multiParamRegex *regexp.Regexp

func init() {
	multiParamRegex = regexp.MustCompile(`^([a-z:]*)\[\d*\]$`)
}

// Retrieve fields with the same name as an array of strings.
func GetMultipleFields(r *http.Request, fieldName string) ([]string, error) {
	var values = []string{}

	for field, value := range r.Form {
		// Strip the square brackets.
		if multiParamRegex.ReplaceAllString(field, "$1") == fieldName {
			values = append(values, value...)
		}
	}

	if len(values) == 0 {
		return []string{}, MissingFieldError{fieldName}
	}

	return values, nil
}

// Retrieve a POST request field as an integer.
// Returns `MissingFieldError` if requested field is missing.
func GetIntField(r *http.Request, fieldName string) (int, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	intField, err := strconv.Atoi(stringField)
	if err != nil {
		return 0, InvalidFormatError{fieldName, stringField}
	}
	return intField, nil
}

// Retrieve a request field as a float.
// Returns `MissingFieldError` if requested field is missing.
func GetFloatField(r *http.Request, fieldName string) (float64, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return float64(0), err
	}
	floatField, err := strconv.ParseFloat(stringField, 64)
	if err != nil {
		return float64(0), InvalidFormatError{fieldName, stringField}
	}
	return floatField, nil
}

// Helper method to retrieve an optional timestamp from POST request field.
// If no timestamp provided, returns current time.
// Returns `InvalidFormatError` if provided timestamp can't be parsed.
func GetTimestampField(r *http.Request, fieldName string) (time.Time, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return time.Now(), MissingFieldError{fieldName}
	}
	parsedTime, err := time.Parse(time.RFC1123, r.FormValue(fieldName))
	if err != nil {
		log.Infof("Failed to convert timestamp %v: %v", r.FormValue(fieldName), err)
		return time.Now(), InvalidFormatError{fieldName, r.FormValue(fieldName)}
	}
	return parsedTime, nil
}

// GetDurationField retrieves a request field as a time.Duration, which is not allowed to be negative.
// Returns `MissingFieldError` if requested field is missing.
func GetDurationField(r *http.Request, fieldName string) (time.Duration, error) {
	s, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	d, err := time.ParseDuration(s)
	if err != nil || d < 0 {
		return 0, InvalidFormatError{fieldName, s}
	}
	return d, nil
}
<commit_msg>Address code review comments<commit_after>package scroll

import (
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mailgun/log"
)

// Retrieve a POST request field as a string.
// Returns `MissingFieldError` if requested field is missing.
func GetStringField(r *http.Request, fieldName string) (string, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return "", MissingFieldError{fieldName}
	}
	return r.FormValue(fieldName), nil
}

// Retrieve a POST request field as a string.
// If the requested field is missing, returns provided default value.
func GetStringFieldWithDefault(r *http.Request, fieldName, defaultValue string) string {
	if fieldValue, err := GetStringField(r, fieldName); err == nil {
		return fieldValue
	}
	return defaultValue
}

// A multiParamRegex is used to convert Ruby and PHP style array params.
// PHP uses ["param[0]", "param[1]",..] instead of ["param", "param",..]
// Ruby uses ["param[]", "param[]",..]
var multiParamRegex *regexp.Regexp

func init() {
	multiParamRegex = regexp.MustCompile(`^([a-z:]*)\[\d*\]$`)
}

// Retrieve fields with the same name as an array of strings.
func GetMultipleFields(r *http.Request, fieldName string) ([]string, error) {
	var values = []string{}

	for field, value := range r.Form {
		// Strip the square brackets.
		if multiParamRegex.ReplaceAllString(field, "$1") == fieldName {
			values = append(values, value...)
		}
	}

	if len(values) == 0 {
		return []string{}, MissingFieldError{fieldName}
	}

	return values, nil
}

// Retrieve a POST request field as an integer.
// Returns `MissingFieldError` if requested field is missing.
func GetIntField(r *http.Request, fieldName string) (int, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	intField, err := strconv.Atoi(stringField)
	if err != nil {
		return 0, InvalidFormatError{fieldName, stringField}
	}
	return intField, nil
}

// Retrieve a request field as a float.
// Returns `MissingFieldError` if requested field is missing.
func GetFloatField(r *http.Request, fieldName string) (float64, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	floatField, err := strconv.ParseFloat(stringField, 64)
	if err != nil {
		return 0, InvalidFormatError{fieldName, stringField}
	}
	return floatField, nil
}

// Helper method to retrieve an optional timestamp from POST request field.
// If no timestamp provided, returns current time.
// Returns `InvalidFormatError` if provided timestamp can't be parsed.
func GetTimestampField(r *http.Request, fieldName string) (time.Time, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return time.Now(), MissingFieldError{fieldName}
	}
	parsedTime, err := time.Parse(time.RFC1123, r.FormValue(fieldName))
	if err != nil {
		log.Infof("Failed to convert timestamp %v: %v", r.FormValue(fieldName), err)
		return time.Now(), InvalidFormatError{fieldName, r.FormValue(fieldName)}
	}
	return parsedTime, nil
}

// GetDurationField retrieves a request field as a time.Duration, which is not allowed to be negative.
// Returns `MissingFieldError` if requested field is missing.
func GetDurationField(r *http.Request, fieldName string) (time.Duration, error) {
	s, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	d, err := time.ParseDuration(s)
	if err != nil || d < 0 {
		return 0, InvalidFormatError{fieldName, s}
	}
	return d, nil
}
<|endoftext|>


<commit_before>package fluent

import (
	"container/list"
	"io"
	"math"
	"net"
	"time"

	"github.com/ugorji/go/codec"
)

type ErrorHandler interface {
	HandleError(error)
}

type ErrorHandlerFunc func(error)

func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}

type pending struct {
	list  *list.List
	limit int
}

func newPending(limit int) *pending {
	return &pending{
		list:  list.New(),
		limit: limit,
	}
}

func (p *pending) Add(b []byte) {
	// trim the pending if limit exceeded
	for i := p.list.Len() - p.limit; i >= 0; i-- {
		p.list.Remove(p.list.Front())
	}

	p.list.PushBack(b)
}

func (p *pending) Flush(w io.Writer) error {
	for e := p.list.Front(); e != nil; e = p.list.Front() {
		if _, err := w.Write(e.Value.([]byte)); err != nil {
			return err
		}
		p.list.Remove(e)
	}
	return nil
}

type Options struct {
	Addr           string
	TagPrefix      string
	SendBufSize    int
	ErrorBufSize   int
	ErrorHandler   ErrorHandler
	DialTimeout    time.Duration
	CloseTimeout   time.Duration
	RetryInterval  time.Duration
	MaxBackoff     int
	MaxPendingSize int
	IsFluxion      bool
}

func (o *Options) Default() {
	if o.SendBufSize < 0 {
		o.SendBufSize = 100
	}
	if o.ErrorBufSize < 0 {
		o.ErrorBufSize = 100
	}
	if o.DialTimeout == 0 {
		o.DialTimeout = 5 * time.Second
	}
	if o.CloseTimeout == 0 {
		o.CloseTimeout = 5 * time.Second
	}
	if o.RetryInterval == 0 {
		o.RetryInterval = 100 * time.Millisecond
	}
	if o.MaxBackoff == 0 {
		o.MaxBackoff = 8
	}
	if o.MaxPendingSize == 0 {
		o.MaxPendingSize = 1000 * 1000
	}
}

type Client struct {
	conn    net.Conn
	inputCh chan interface{}
	errorCh chan error
	closeCh chan bool
	opts    *Options
	pending *pending
	mh      *codec.MsgpackHandle
}

func NewClient(opts Options) (*Client, error) {
	opts.Default()
	c := &Client{
		opts:    &opts,
		inputCh: make(chan interface{}, opts.SendBufSize),
		closeCh: make(chan bool),
		pending: newPending(opts.MaxPendingSize),
	}
	if !opts.IsFluxion {
		c.mh = &codec.MsgpackHandle{}
	} else {
		c.mh = &codec.MsgpackHandle{RawToString: true, WriteExt: true}
	}

	conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout)
	if err != nil {
		return nil, err
	}
	c.conn = conn

	if c.opts.ErrorHandler != nil {
		c.errorCh = make(chan error, c.opts.ErrorBufSize)
		go c.errorWorker()
	}

	go c.worker()
	return c, nil
}

func (c *Client) Send(tag string, v interface{}) {
	c.SendWithTime(tag, time.Now(), v)
}

func (c *Client) SendWithTime(tag string, t time.Time, v interface{}) {
	if c.opts.TagPrefix != "" {
		if tag != "" {
			tag = c.opts.TagPrefix + "." + tag
		} else {
			tag = c.opts.TagPrefix
		}
	}

	var tt interface{}
	if !c.opts.IsFluxion {
		tt = t.Unix()
	} else {
		tt = t
	}

	val := []interface{}{tag, tt, v}
	c.inputCh <- val
}

func (c *Client) Close() {
	close(c.inputCh)

	select {
	case <-c.closeCh:
	case <-time.After(c.opts.CloseTimeout):
	}
}

func (c *Client) worker() {
	defer func() {
		if c.errorCh != nil {
			close(c.errorCh)
		} else {
			close(c.closeCh)
		}
	}()

	for v := range c.inputCh {
		var b []byte
		if err := codec.NewEncoderBytes(&b, c.mh).Encode(v); err != nil {
			c.pushError(err)
			continue
		}
		if _, err := c.conn.Write(b); err != nil {
			c.pending.Add(b)
			c.pushError(err)
			c.reconnect()
		}
	}
}

func (c *Client) reconnect() {
	attempts := 0
	retry := time.After(0)

	for {
		select {
		case v, ok := <-c.inputCh:
			if !ok {
				return
			}

			var b []byte
			if err := codec.NewEncoderBytes(&b, c.mh).Encode(v); err != nil {
				c.pushError(err)
				continue
			}
			c.pending.Add(b)
		case <-retry:
			if conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout); err == nil {
				if err := c.pending.Flush(conn); err == nil {
					c.conn = conn
					return
				}
			}
			retry = time.After(backoff(c.opts.RetryInterval, attempts, c.opts.MaxBackoff))
			attempts++
		}
	}
}

func (c *Client) errorWorker() {
	defer close(c.closeCh)

	for err := range c.errorCh {
		c.opts.ErrorHandler.HandleError(err)
	}
}

func (c *Client) pushError(err error) {
	if c.errorCh == nil {
		return
	}

	select {
	case c.errorCh <- err:
	default:
	}
}

func backoff(interval time.Duration, count, limit int) time.Duration {
	if count > limit {
		count = limit
	}
	return interval * time.Duration(math.Exp2(float64(count)))
}
<commit_msg>Rewrite reconnecting logic to eliminate unexpected blocking<commit_after>package fluent

import (
	"container/list"
	"math"
	"net"
	"sync"
	"time"

	"github.com/ugorji/go/codec"
)

type ErrorHandler interface {
	HandleError(error)
}

type ErrorHandlerFunc func(error)

func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}

type pending struct {
	list     *list.List
	limit    int
	m        sync.Mutex
	flushing bool
}

func newPending(limit int) *pending {
	return &pending{
		list:  list.New(),
		limit: limit,
	}
}

func (p *pending) Add(b []byte) {
	p.m.Lock()
	defer p.m.Unlock()

	// If not flushing now, trim the pending if limit exceeded.
	// Otherwise, to avoid conflict, trimming is temporary disabled.
	if !p.flushing {
		for i := p.list.Len() - p.limit; i >= 0; i-- {
			p.list.Remove(p.list.Front())
		}
	}

	p.list.PushBack(b)
}

func (p *pending) Flush(conn net.Conn, timeout time.Duration) error {
	p.m.Lock()
	n := p.list.Len()
	p.flushing = true
	p.m.Unlock()

	defer func() {
		p.flushing = false
	}()

	for i, e := 0, p.list.Front(); i < n; i, e = i+1, p.list.Front() {
		conn.SetDeadline(time.Now().Add(timeout))
		if _, err := conn.Write(e.Value.([]byte)); err != nil {
			return err
		}

		p.m.Lock()
		p.list.Remove(e)
		p.m.Unlock()
	}
	conn.SetDeadline(time.Time{})
	return nil
}

type Options struct {
	Addr           string
	TagPrefix      string
	SendBufSize    int
	ErrorBufSize   int
	ErrorHandler   ErrorHandler
	LogHandler     func(string, ...interface{})
	DialTimeout    time.Duration
	CloseTimeout   time.Duration
	Timeout        time.Duration
	RetryWait      time.Duration
	MaxBackoff     int
	MaxPendingSize int
	IsFluxion      bool

	// If NonBlocking is true and send buffer is full, further events will be dropped.
	NonBlocking bool
}

func (o *Options) Default() {
	if o.SendBufSize == 0 {
		o.SendBufSize = 50000
	}
	if o.ErrorBufSize == 0 {
		o.ErrorBufSize = 100
	}
	if o.DialTimeout == 0 {
		o.DialTimeout = 5 * time.Second
	}
	if o.CloseTimeout == 0 {
		o.CloseTimeout = 5 * time.Second
	}
	if o.Timeout == 0 {
		o.Timeout = 5 * time.Second
	}
	if o.RetryWait == 0 {
		o.RetryWait = 100 * time.Millisecond
	}
	if o.MaxBackoff == 0 {
		o.MaxBackoff = 8
	}
	if o.MaxPendingSize == 0 {
		o.MaxPendingSize = 1000 * 1000
	}
}

type Client struct {
	conn    net.Conn
	inputCh chan interface{}
	errorCh chan error
	closeCh chan bool
	opts    *Options
	pending *pending
	mh      *codec.MsgpackHandle
}

func NewClient(opts Options) (*Client, error) {
	opts.Default()
	c := &Client{
		opts:    &opts,
		inputCh: make(chan interface{}, opts.SendBufSize),
		closeCh: make(chan bool),
		pending: newPending(opts.MaxPendingSize),
	}
	c.log("Fluent options: %+v", opts)
	if !opts.IsFluxion {
		c.mh = &codec.MsgpackHandle{}
	} else {
		c.mh = &codec.MsgpackHandle{RawToString: true, WriteExt: true}
	}

	conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout)
	if err != nil {
		return nil, err
	}
	c.conn = conn

	if c.opts.ErrorHandler != nil {
		c.errorCh = make(chan error, c.opts.ErrorBufSize)
		go c.errorWorker()
	}

	go c.sender()
	return c, nil
}

// Send sends record v with given tag. v can be any type which can be encoded
// to msgpack. Specially, if type of v is []byte, it will be sent without any
// modification.
func (c *Client) Send(tag string, v interface{}) {
	c.SendWithTime(tag, time.Now(), v)
}

// SendWithTime sends record v with given tag and time. See Send.
func (c *Client) SendWithTime(tag string, t time.Time, v interface{}) {
	if c.opts.TagPrefix != "" {
		if tag != "" {
			tag = c.opts.TagPrefix + "." + tag
		} else {
			tag = c.opts.TagPrefix
		}
	}

	var tt interface{}
	if !c.opts.IsFluxion {
		tt = t.Unix()
	} else {
		tt = t
	}

	val := []interface{}{tag, tt, v}
	if c.opts.NonBlocking {
		select {
		case c.inputCh <- val:
		default:
		}
	} else {
		c.inputCh <- val
	}
}

func (c *Client) Close() {
	close(c.inputCh)

	select {
	case <-c.closeCh:
	case <-time.After(c.opts.CloseTimeout):
	}
}

func (c *Client) sender() {
	defer func() {
		if c.errorCh != nil {
			close(c.errorCh)
		} else {
			close(c.closeCh)
		}
	}()

	for v := range c.inputCh {
		b, err := c.encode(v)
		if err != nil {
			c.pushError(err)
			continue
		}
		c.conn.SetDeadline(time.Now().Add(c.opts.Timeout))
		if _, err = c.conn.Write(b); err != nil {
			c.pending.Add(b)
			c.pushError(err)
			c.conn.Close()
			c.reconnect()
		}
		// Cancel deadline setting
		c.conn.SetDeadline(time.Time{})
	}
}

func (c *Client) encode(v interface{}) ([]byte, error) {
	if b, ok := v.([]byte); ok {
		return b, nil
	}
	var b []byte
	err := codec.NewEncoderBytes(&b, c.mh).Encode(v)
	return b, err
}

func (c *Client) reconnect() {
	c.log("Connection failed, enter reconnection loop")
	stop := c.poll()
	for {
		for attempts := 0; ; attempts++ {
			time.Sleep(backoff(c.opts.RetryWait, attempts, c.opts.MaxBackoff))
			c.log("Reconnect attempt %d", attempts+1)
			if conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout); err == nil {
				c.log("Reconnected! try to flush pendings")
				err, f := c.flush(conn, stop)
				if err == nil {
					c.conn = conn
					c.log("Reconnection process completed! enter normal loop")
					return
				}
				conn.Close()
				if f != nil {
					stop = f
				}
			}
		}
	}
}

func (c *Client) flush(conn net.Conn, stop func()) (error, func()) {
	for attempts := 1; ; attempts++ {
		t := time.Now()
		if err := c.pending.Flush(conn, c.opts.Timeout); err != nil {
			c.log("Failed to flush pendings, retrying...")
			return err, nil
		}
		if took := time.Since(t); took < c.opts.Timeout {
			c.log("Flushing almost completed in an acceptable time (%v)", took)
			stop()
			break
		} else {
			c.log("Flush attempts %d in %v", attempts, took)
		}
	}
	if err := c.pending.Flush(conn, c.opts.Timeout); err != nil {
		c.log("Failed to flush last piece of pendings")
		f := c.poll()
		return err, f
	}
	c.log("All flushing process succeeded!")
	return nil, nil
}

func (c *Client) poll() (stop func()) {
	closeC, doneC := make(chan bool, 1), make(chan bool)
	go func() {
		c.log("Start pending loop")
		defer c.log("Stop pending loop")
		for {
			select {
			case <-closeC:
				doneC <- true
				return
			case v, ok := <-c.inputCh:
				if !ok {
					doneC <- true
					return
				}

				b, err := c.encode(v)
				if err != nil {
					c.pushError(err)
				} else {
					c.pending.Add(b)
				}
			}
		}
	}()
	return func() {
		closeC <- true
		<-doneC
	}
}

func (c *Client) log(s string, args ...interface{}) {
	if c.opts.LogHandler != nil {
		c.opts.LogHandler(s, args...)
	}
}

func (c *Client) errorWorker() {
	defer close(c.closeCh)

	for err := range c.errorCh {
		c.opts.ErrorHandler.HandleError(err)
	}
}

func (c *Client) pushError(err error) {
	if c.errorCh == nil {
		return
	}

	select {
	case c.errorCh <- err:
	default:
	}
}

func backoff(interval time.Duration, count, limit int) time.Duration {
	if count > limit {
		count = limit
	}
	return interval * time.Duration(math.Exp2(float64(count)))
}
<|endoftext|>


<commit_before>package fluent

import (
	"container/list"
	"math"
	"net"
	"time"

	"github.com/ugorji/go/codec"
)

type ErrorHandler interface {
	HandleError(error)
}

type ErrorHandlerFunc func(error)

func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}

type Options struct {
	Addr          string
	SendBufSize   int
	ErrorBufSize  int
	ErrorHandler  ErrorHandler
	DialTimeout   time.Duration
	CloseTimeout  time.Duration
	RetryInterval time.Duration
	MaxBackoff    int
}

func (o *Options) Default() {
	if o.SendBufSize < 0 {
		o.SendBufSize = 100
	}
	if o.ErrorBufSize < 0 {
		o.ErrorBufSize = 100
	}
	if o.DialTimeout == 0 {
		o.DialTimeout = 5 * time.Second
	}
	if o.CloseTimeout == 0 {
		o.CloseTimeout = 5 * time.Second
	}
	if o.RetryInterval == 0 {
		o.RetryInterval = 100 * time.Millisecond
	}
	if o.MaxBackoff == 0 {
		o.MaxBackoff = 8
	}
}

type Client struct {
	conn    net.Conn
	inputCh chan interface{}
	errorCh chan error
	closeCh chan bool
	opts    *Options
	pending *list.List
}

func NewClient(opts Options) (*Client, error) {
	opts.Default()
	c := &Client{
		opts:    &opts,
		inputCh: make(chan interface{}, opts.SendBufSize),
		closeCh: make(chan bool),
		pending: list.New(),
	}

	conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout)
	if err != nil {
		return nil, err
	}
	c.conn = conn

	if c.opts.ErrorHandler != nil {
		c.errorCh = make(chan error, c.opts.ErrorBufSize)
		go c.errorWorker()
	}

	go c.worker()
	return c, nil
}

func (c *Client) Send(tag string, v interface{}) {
	now := time.Now().Unix()
	val := []interface{}{tag, now, v}
	c.inputCh <- val
}

func (c *Client) Close() {
	close(c.inputCh)

	select {
	case <-c.closeCh:
	case <-time.After(c.opts.CloseTimeout):
	}
}

func (c *Client) worker() {
	defer func() {
		if c.errorCh != nil {
			close(c.errorCh)
		} else {
			close(c.closeCh)
		}
	}()

	for v := range c.inputCh {
		var b []byte
		if err := codec.NewEncoderBytes(&b, &codec.MsgpackHandle{}).Encode(v); err != nil {
			c.pushError(err)
			continue
		}

		if _, err := c.conn.Write(b); err != nil {
			c.pending.PushBack(b)
			c.pushError(err)
			c.reconnect()
		}
	}
}

func (c *Client) reconnect() {
	attempts := 0
	retry := time.After(0)

	for {
		select {
		case v, ok := <-c.inputCh:
			if !ok {
				return
			}

			var b []byte
			if err := codec.NewEncoderBytes(&b, &codec.MsgpackHandle{}).Encode(v); err != nil {
				c.pushError(err)
				continue
			}
			c.pending.PushBack(b)
		case <-retry:
			if conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout); err == nil {
				c.conn = conn
				if err := c.sendPending(); err == nil {
					return
				}
			}
			retry = time.After(backoff(c.opts.RetryInterval, attempts, c.opts.MaxBackoff))
			attempts++
		}
	}
}

func (c *Client) sendPending() error {
	for e := c.pending.Front(); e != nil; e = c.pending.Front() {
		if _, err := c.conn.Write(e.Value.([]byte)); err != nil {
			return err
		}
		c.pending.Remove(e)
	}
	return nil
}

func (c *Client) errorWorker() {
	defer close(c.closeCh)

	for err := range c.errorCh {
		c.opts.ErrorHandler.HandleError(err)
	}
}

func (c *Client) pushError(err error) {
	if c.errorCh == nil {
		return
	}

	select {
	case c.errorCh <- err:
	default:
	}
}

func backoff(interval time.Duration, count, limit int) time.Duration {
	if count > limit {
		count = limit
	}
	return interval * time.Duration(math.Exp2(float64(count)))
}
<commit_msg>add an option for pending size limit<commit_after>package fluent

import (
	"container/list"
	"io"
	"math"
	"net"
	"time"

	"github.com/ugorji/go/codec"
)

type ErrorHandler interface {
	HandleError(error)
}

type ErrorHandlerFunc func(error)

func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}

type pending struct {
	list  *list.List
	limit int
}

func newPending(limit int) *pending {
	return &pending{
		list:  list.New(),
		limit: limit,
	}
}

func (p *pending) Add(b []byte) {
	// trim the pending if limit exceeded
	for i := p.list.Len() - p.limit; i >= 0; i-- {
		p.list.Remove(p.list.Front())
	}

	p.list.PushBack(b)
}

func (p *pending) Flush(w io.Writer) error {
	for e := p.list.Front(); e != nil; e = p.list.Front() {
		if _, err := w.Write(e.Value.([]byte)); err != nil {
			return err
		}
		p.list.Remove(e)
	}
	return nil
}

type Options struct {
	Addr           string
	SendBufSize    int
	ErrorBufSize   int
	ErrorHandler   ErrorHandler
	DialTimeout    time.Duration
	CloseTimeout   time.Duration
	RetryInterval  time.Duration
	MaxBackoff     int
	MaxPendingSize int
}

func (o *Options) Default() {
	if o.SendBufSize < 0 {
		o.SendBufSize = 100
	}
	if o.ErrorBufSize < 0 {
		o.ErrorBufSize = 100
	}
	if o.DialTimeout == 0 {
		o.DialTimeout = 5 * time.Second
	}
	if o.CloseTimeout == 0 {
		o.CloseTimeout = 5 * time.Second
	}
	if o.RetryInterval == 0 {
		o.RetryInterval = 100 * time.Millisecond
	}
	if o.MaxBackoff == 0 {
		o.MaxBackoff = 8
	}
	if o.MaxPendingSize == 0 {
		o.MaxPendingSize = 1000 * 1000
	}
}

type Client struct {
	conn    net.Conn
	inputCh chan interface{}
	errorCh chan error
	closeCh chan bool
	opts    *Options
	pending *pending
}

func NewClient(opts Options) (*Client, error) {
	opts.Default()
	c := &Client{
		opts:    &opts,
		inputCh: make(chan interface{}, opts.SendBufSize),
		closeCh: make(chan bool),
		pending: newPending(opts.MaxPendingSize),
	}

	conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout)
	if err != nil {
		return nil, err
	}
	c.conn = conn

	if c.opts.ErrorHandler != nil {
		c.errorCh = make(chan error, c.opts.ErrorBufSize)
		go c.errorWorker()
	}

	go c.worker()
	return c, nil
}

func (c *Client) Send(tag string, v interface{}) {
	now := time.Now().Unix()
	val := []interface{}{tag, now, v}
	c.inputCh <- val
}

func (c *Client) Close() {
	close(c.inputCh)

	select {
	case <-c.closeCh:
	case <-time.After(c.opts.CloseTimeout):
	}
}

func (c *Client) worker() {
	defer func() {
		if c.errorCh != nil {
			close(c.errorCh)
		} else {
			close(c.closeCh)
		}
	}()

	for v := range c.inputCh {
		var b []byte
		if err := codec.NewEncoderBytes(&b, &codec.MsgpackHandle{}).Encode(v); err != nil {
			c.pushError(err)
			continue
		}

		if _, err := c.conn.Write(b); err != nil {
			c.pending.Add(b)
			c.pushError(err)
			c.reconnect()
		}
	}
}

func (c *Client) reconnect() {
	attempts := 0
	retry := time.After(0)

	for {
		select {
		case v, ok := <-c.inputCh:
			if !ok {
				return
			}

			var b []byte
			if err := codec.NewEncoderBytes(&b, &codec.MsgpackHandle{}).Encode(v); err != nil {
				c.pushError(err)
				continue
			}
			c.pending.Add(b)
		case <-retry:
			if conn, err := net.DialTimeout("tcp", c.opts.Addr, c.opts.DialTimeout); err == nil {
				if err := c.pending.Flush(conn); err == nil {
					c.conn = conn
					return
				}
			}
			retry = time.After(backoff(c.opts.RetryInterval, attempts, c.opts.MaxBackoff))
			attempts++
		}
	}
}

func (c *Client) errorWorker() {
	defer close(c.closeCh)

	for err := range c.errorCh {
		c.opts.ErrorHandler.HandleError(err)
	}
}

func (c *Client) pushError(err error) {
	if c.errorCh == nil {
		return
	}

	select {
	case c.errorCh <- err:
	default:
	}
}

func backoff(interval time.Duration, count, limit int) time.Duration {
	if count > limit {
		count = limit
	}
	return interval * time.Duration(math.Exp2(float64(count)))
}
<|endoftext|>


<commit_before>package service

import (
	"encoding/json"
	"fmt"
	memcache "github.com/patrickmn/go-cache"
	"github.com/zwirec/TGChatScanner/TGBotApi"
	"github.com/zwirec/TGChatScanner/clarifaiApi"
	"github.com/zwirec/TGChatScanner/modelManager"
	"github.com/zwirec/TGChatScanner/requestHandler"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"os/user"
	"sync"
	"syscall"
	"time"
)

var (
	home      = os.Getenv("HOME")
	configUrl = os.Getenv("TGCHATSCANNER_REMOTE_CONFIG")
)

func init() {
	if home == "" {
		u, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}
		home := u.HomeDir
		fmt.Fprint(ioutil.Discard, home)
	}
	if configUrl == "" {
		configUrl = home + "/.config/tgchatscanner/config.json"
	}
}

type Config map[string]map[string]interface{}

//Service s
type Service struct {
	sock         net.Listener
	mux          *http.ServeMux
	srv          *http.Server
	rAPIHandler  *requestHandler.RequestHandler
	config       Config
	sysLogger    *log.Logger
	accessLogger *log.Logger
	notifier     chan os.Signal
	poolsWG      sync.WaitGroup
	poolsDone    chan struct{}
}

func NewService() *Service {
	return &Service{
		rAPIHandler: requestHandler.NewRequestHandler(),
		mux:         http.NewServeMux(),
		notifier:    make(chan os.Signal),
		poolsDone:   make(chan struct{}),
	}
}

func (s *Service) Run() error {

	//_, err := os.OpenFile("error.log", os.O_CREATE|os.O_RDWR, 0777)
	//if err != nil {
	//	errorlog = os.Stderr
	//}
	//_, err := os.OpenFile("access.log", os.O_CREATE|os.O_RDWR, 0777)
	//if err != nil {
	//	accesslog = os.Stdout
	//}

	s.sysLogger = log.New(os.Stdout, "", log.LstdFlags|log.Llongfile)
	s.accessLogger = log.New(os.Stderr, "", log.LstdFlags)

	if err := s.parseConfig(configUrl); err != nil {
		s.sysLogger.Println(err)
		return err
	}

	s.signalProcessing()

	db, err := modelManager.ConnectToDB(s.config["db"])

	if err != nil {
		s.sysLogger.Println(err)
		return err
	}

	if err := modelManager.InitDB(db); err != nil {
		s.sysLogger.Println(err)
	}

	clApi := clarifaiApi.NewClarifaiApi(s.config["clarifai"]["api_key"].(string))

	botApi := TGBotApi.NewBotApi(s.config["tg_bot_api"]["token"].(string))

	workers_n, ok := s.config["server"]["workers"].(int)

	if !ok {
		workers_n = 10
	}

	dr := make(chan *requestHandler.FileBasic, workers_n*2)

	poolStopper := make(chan struct{})

	fp := &requestHandler.FilePreparatorsPool{In: dr, Done: poolStopper, WorkersNumber: workers_n}
	fpOut := fp.Run(workers_n*2, s.poolsWG)

	forker := &requestHandler.ForkersPool{
		In:             fpOut,
		Done:           poolStopper,
		WorkersNumber:  workers_n,
		ForkToFileInfo: requestHandler.CastToFileInfo,
		ForkToFileLink: requestHandler.CastToFileLink,
	}

	fdIn, prIn := forker.Run(workers_n, workers_n, s.poolsWG)

	fd := &requestHandler.FileDownloadersPool{In: fdIn, Done: poolStopper, WorkersNumber: workers_n}
	fdOut := fd.Run(workers_n, s.poolsWG)

	pr := &requestHandler.PhotoRecognizersPool{In: prIn, Done: poolStopper, WorkersNumber: workers_n}
	prOut := pr.Run(workers_n, s.poolsWG)

	deforker := &requestHandler.DeforkersPool{
		In1:              fdOut,
		In2:              prOut,
		WorkersNumber:    workers_n,
		DeforkDownloaded: requestHandler.CastFromDownloadedFile,
		DeforkRecognized: requestHandler.CastFromRecognizedPhoto,
	}

	dbsIn := deforker.Run(workers_n*2, s.poolsWG)

	dbs := &requestHandler.DbStoragersPool{In: dbsIn, WorkersNumber: workers_n}
	dbs.Run(s.poolsWG)

	cache := memcache.New(5*time.Minute, 10*time.Minute)

	imgPath, ok := s.config["chatscanner"]["images_path"].(string)

	if err := os.MkdirAll(imgPath, os.ModePerm); err != nil {
		s.sysLogger.Println(err)
	}

	hostname, ok := s.config["chatscanner"]["host"].(string)

	_, err = url.Parse(hostname)

	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			s.sysLogger.Println(err)
			hostname = "localhost"
		}
	}

	context := requestHandler.AppContext{
		Db:               db,
		DownloadRequests: dr,
		BotApi:           botApi,
		CfApi:            clApi,
		Cache:            cache,
		SysLogger:        s.sysLogger,
		AccessLogger:     s.accessLogger,
		ImagesPath:       imgPath,
		Hostname:         hostname,
	}

	s.rAPIHandler.SetAppContext(&context)
	s.rAPIHandler.RegisterHandlers()

	s.srv = &http.Server{Handler: s.rAPIHandler}

	defer close(poolStopper)

	var wg sync.WaitGroup
	wg.Add(1)

	go s.endpoint()

	wg.Wait()
	return nil
}

func (s *Service) endpoint() (err error) {

	s.sock, err = net.Listen("unix", s.config["server"]["socket"].(string))

	if err != nil {
		s.sysLogger.Println(err)
		s.notifier <- syscall.SIGINT
	}
	if err := os.Chmod(s.config["server"]["socket"].(string), 0777); err != nil {
		s.sysLogger.Println(err)
		s.notifier <- syscall.SIGINT
	}

	s.sysLogger.Println("Socket opened")
	s.sysLogger.Println("Server started")

	if err := s.srv.Serve(s.sock); err != nil {
		s.sysLogger.Println(err)
		s.notifier <- syscall.SIGINT
	}
	return nil
}

func (s *Service) parseConfig(_url string) error {
	var configRaw []byte

	_, err := url.Parse(_url)

	if err == nil {
		res, err := http.Get(_url)
		if err != nil {
			s.sysLogger.Println(err)
			return err
		}

		configRaw, err = ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			s.sysLogger.Println(err)
			return err
		}

	} else {
		var err error
		configRaw, err = ioutil.ReadFile(_url)
		if err != nil {
			s.sysLogger.Println(err)
			return err
		}
	}

	if err := json.Unmarshal(configRaw, &s.config); err != nil {
		s.sysLogger.Println(err)
		return err
	}

	return nil

}

func (s *Service) signalProcessing() {
	signal.Notify(s.notifier, syscall.SIGINT)
	go s.handler(s.notifier)
}

func (s *Service) handler(c chan os.Signal) {
	for {
		<-c
		s.sysLogger.Println("Gracefully stopping...")
		close(s.poolsDone)
		s.poolsWG.Wait()
		if err := s.srv.Shutdown(nil); err != nil {
			s.sysLogger.Println(err)
			return
		}
		if err := s.sock.Close(); err != nil {
			s.sysLogger.Println(err)
		}
		if err := os.Remove(s.config["server"]["socket"].(string)); err != nil {
			s.sysLogger.Println(err)
			return
		}
		os.Exit(0)
	}
}
<commit_msg>fix logs fix connection<commit_after>package service

import (
	"encoding/json"
	"fmt"
	memcache "github.com/patrickmn/go-cache"
	"github.com/zwirec/TGChatScanner/TGBotApi"
	"github.com/zwirec/TGChatScanner/clarifaiApi"
	"github.com/zwirec/TGChatScanner/modelManager"
	"github.com/zwirec/TGChatScanner/requestHandler"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"os/user"
	"sync"
	"syscall"
	"time"
)

var (
	home      = os.Getenv("HOME")
	configUrl = os.Getenv("TGCHATSCANNER_REMOTE_CONFIG")
)

func init() {
	if home == "" {
		u, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}
		home := u.HomeDir
		fmt.Fprint(ioutil.Discard, home)
	}
	if configUrl == "" {
		configUrl = home + "/.config/tgchatscanner/config.json"
	}
}

type Config map[string]map[string]interface{}

//Service s
type Service struct {
	sock         net.Listener
	mux          *http.ServeMux
	srv          *http.Server
	rAPIHandler  *requestHandler.RequestHandler
	config       Config
	sysLogger    *log.Logger
	accessLogger *log.Logger
	notifier     chan os.Signal
	poolsWG      sync.WaitGroup
	poolsDone    chan struct{}
}

func NewService() *Service {
	return &Service{
		rAPIHandler: requestHandler.NewRequestHandler(),
		mux:         http.NewServeMux(),
		notifier:    make(chan os.Signal),
		poolsDone:   make(chan struct{}),
	}
}

func (s *Service) Run() error {

	errorlog, err := os.OpenFile("error.log", os.O_CREATE|os.O_RDWR, 0777)
	if err != nil {
		errorlog = os.Stderr
	}
	accesslog, err := os.OpenFile("access.log", os.O_CREATE|os.O_RDWR, 0777)
	if err != nil {
		accesslog = os.Stdout
	}

	s.sysLogger = log.New(errorlog, "", log.LstdFlags|log.Llongfile)
	s.accessLogger = log.New(accesslog, "", log.LstdFlags)

	if err := s.parseConfig(configUrl); err != nil {
		s.sysLogger.Println(err)
		return err
	}

	s.signalProcessing()

	db, err := modelManager.ConnectToDB(s.config["db"])

	if err != nil {
		s.sysLogger.Println(err)
		return err
	}

	if err := modelManager.InitDB(db); err != nil {
		s.sysLogger.Println(err)
	}

	clApi := clarifaiApi.NewClarifaiApi(s.config["clarifai"]["api_key"].(string))

	botApi := TGBotApi.NewBotApi(s.config["tg_bot_api"]["token"].(string))

	workers_n, ok := s.config["server"]["workers"].(int)

	if !ok {
		workers_n = 10
	}

	dr := make(chan *requestHandler.FileBasic, workers_n*2)

	poolStopper := make(chan struct{})

	fp := &requestHandler.FilePreparatorsPool{In: dr, Done: poolStopper, WorkersNumber: workers_n}
	fpOut := fp.Run(workers_n*2, s.poolsWG)

	forker := &requestHandler.ForkersPool{
		In:             fpOut,
		Done:           poolStopper,
		WorkersNumber:  workers_n,
		ForkToFileInfo: requestHandler.CastToFileInfo,
		ForkToFileLink: requestHandler.CastToFileLink,
	}

	fdIn, prIn := forker.Run(workers_n, workers_n, s.poolsWG)

	fd := &requestHandler.FileDownloadersPool{In: fdIn, Done: poolStopper, WorkersNumber: workers_n}
	fdOut := fd.Run(workers_n, s.poolsWG)

	pr := &requestHandler.PhotoRecognizersPool{In: prIn, Done: poolStopper, WorkersNumber: workers_n}
	prOut := pr.Run(workers_n, s.poolsWG)

	deforker := &requestHandler.DeforkersPool{
		In1:              fdOut,
		In2:              prOut,
		WorkersNumber:    workers_n,
		DeforkDownloaded: requestHandler.CastFromDownloadedFile,
		DeforkRecognized: requestHandler.CastFromRecognizedPhoto,
	}

	dbsIn := deforker.Run(workers_n*2, s.poolsWG)

	dbs := &requestHandler.DbStoragersPool{In: dbsIn, WorkersNumber: workers_n}
	dbs.Run(s.poolsWG)

	cache := memcache.New(5*time.Minute, 10*time.Minute)

	imgPath, ok := s.config["chatscanner"]["images_path"].(string)

	if err := os.MkdirAll(imgPath, os.ModePerm); err != nil {
		s.sysLogger.Println(err)
	}

	hostname, ok := s.config["chatscanner"]["host"].(string)

	_, err = url.Parse(hostname)

	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			s.sysLogger.Println(err)
			hostname = "localhost"
		}
	}

	context := requestHandler.AppContext{
		Db:               db,
		DownloadRequests: dr,
		BotApi:           botApi,
		CfApi:            clApi,
		Cache:            cache,
		SysLogger:        s.sysLogger,
		AccessLogger:     s.accessLogger,
		ImagesPath:       imgPath,
		Hostname:         hostname,
	}

	s.rAPIHandler.SetAppContext(&context)
	s.rAPIHandler.RegisterHandlers()

	s.srv = &http.Server{Handler: s.rAPIHandler}

	defer close(poolStopper)

	var wg sync.WaitGroup
	wg.Add(1)

	go s.endpoint()

	wg.Wait()
	return nil
}

func (s *Service) endpoint() (err error) {

	s.sock, err = net.Listen("unix", s.config["server"]["socket"].(string))

	if err != nil {
		s.sysLogger.Println(err)
		s.notifier <- syscall.SIGINT
	}
	if err := os.Chmod(s.config["server"]["socket"].(string), 0777); err != nil {
		s.sysLogger.Println(err)
		s.notifier <- syscall.SIGINT
	}

	s.sysLogger.Println("Socket opened")
	s.sysLogger.Println("Server started")

	if err := s.srv.Serve(s.sock); err != nil {
		s.sysLogger.Println(err)
		//s.notifier <- syscall.SIGINT
	}
	return nil
}

func (s *Service) parseConfig(_url string) error {
	var configRaw []byte

	_, err := url.Parse(_url)

	if err == nil {
		res, err := http.Get(_url)
		if err != nil {
			s.sysLogger.Println(err)
			return err
		}

		configRaw, err = ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			s.sysLogger.Println(err)
			return err
		}

	} else {
		var err error
		configRaw, err = ioutil.ReadFile(_url)
		if err != nil {
			s.sysLogger.Println(err)
			return err
		}
	}

	if err := json.Unmarshal(configRaw, &s.config); err != nil {
		s.sysLogger.Println(err)
		return err
	}

	return nil

}

func (s *Service) signalProcessing() {
	signal.Notify(s.notifier, syscall.SIGINT)
	go s.handler(s.notifier)
}

func (s *Service) handler(c chan os.Signal) {
	for {
		<-c
		s.sysLogger.Println("Gracefully stopping...")
		close(s.poolsDone)
		s.poolsWG.Wait()
		if err := s.srv.Shutdown(nil); err != nil {
			s.sysLogger.Println(err)
			return
		}
		//if err := s.sock.Close(); err != nil {
		//	s.sysLogger.Println(err)
		//}
		//if err := os.Remove(s.config["server"]["socket"].(string)); err != nil {
		//	s.sysLogger.Println(err)
		//	return
		//}
		os.Exit(0)
	}
}
<|endoftext|>


<commit_before>package service

import (
	"net/http"
	"time"
	"github.com/rwl/go-endpoints/endpoints"
)

func init() {
	greetService := &GreetingService{}
	api, err := endpoints.RegisterService(greetService,
	"greeting", "v1", "Greetings API", true)
	if err != nil {
		panic(err.Error())
	}

	info := api.MethodByName("List").Info()
	info.Name, info.HttpMethod, info.Path, info.Desc =
	"greets.list", "GET", "greetings", "List most recent greetings."

	endpoints.HandleHttp()
}

// Greeting is a datastore entity that represents a single greeting.
// It also serves as (a part of) a response of GreetingService.
type Greeting struct {
	Key     string		`json:"id"`
	Author  string		`json:"author"`
	Content string		`json:"content"`
	Date    time.Time	`json:"date"`
}

// GreetingsList is a response type of GreetingService.List method
type GreetingsList struct {
	Items []*Greeting `json:"items"`
}

// Request type for GreetingService.List
type GreetingsListReq struct {
	Limit int `json:"limit" endpoints:"d=10"`
}

// GreetingService can sign the guesbook, list all greetings and delete
// a greeting from the guestbook.
type GreetingService struct {
}

// List responds with a list of all greetings ordered by Date field.
// Most recent greets come first.
func (gs *GreetingService) List(
	r *http.Request, req *GreetingsListReq, resp *GreetingsList) error {

		if req.Limit <= 0 {
			req.Limit = 10
		}

		greets := make([]*Greeting, 0, req.Limit)

		resp.Items = greets
		return nil
	}
<commit_msg>fix(service): initialize the greetings<commit_after>package service

import (
	"fmt"
	"net/http"
	"time"
	"github.com/rwl/go-endpoints/endpoints"
)

func init() {
	greetService := &GreetingService{}
	api, err := endpoints.RegisterService(greetService,
	"greeting", "v1", "Greetings API", true)
	if err != nil {
		panic(err.Error())
	}

	info := api.MethodByName("List").Info()
	info.Name, info.HttpMethod, info.Path, info.Desc =
	"greets.list", "GET", "greetings", "List most recent greetings."

	endpoints.HandleHttp()
}

// Greeting is a datastore entity that represents a single greeting.
// It also serves as (a part of) a response of GreetingService.
type Greeting struct {
	Key     string		`json:"id"`
	Author  string		`json:"author"`
	Content string		`json:"content"`
	Date    time.Time	`json:"date"`
}

// GreetingsList is a response type of GreetingService.List method
type GreetingsList struct {
	Items []*Greeting `json:"items"`
}

// Request type for GreetingService.List
type GreetingsListReq struct {
	Limit int `json:"limit" endpoints:"d=10"`
}

// GreetingService can sign the guesbook, list all greetings and delete
// a greeting from the guestbook.
type GreetingService struct {
}

// List responds with a list of all greetings ordered by Date field.
// Most recent greets come first.
func (gs *GreetingService) List(
	r *http.Request, req *GreetingsListReq, resp *GreetingsList) error {

		if req.Limit <= 0 {
			req.Limit = 10
		}

		greets := make([]*Greeting, 10)

		for i, _ := range greets {
			greets[i] = &Greeting{}
			greets[i].Author = fmt.Sprintf("Name %v", i)
			greets[i].Content = fmt.Sprintf("Message %v", i)
			greets[i].Date = time.Now()
		}

		resp.Items = greets
		return nil
	}
<|endoftext|>


<commit_before>package services

import (
	"errors"
	"fmt"
	"net/http"
)

// Github service name checker
// TODO: use Github API token
type Github struct{}

// Check implements the NameChecker interface
func (g *Github) Check(name string) (bool, error) {
	resp, err := http.Get(fmt.Sprintf("https://api.github.com/users/%s", name))
	if err != nil {
		// TODO: wrap error
		return false, errors.New("Cannot determine name availability")
	}
	resp.Body.Close()

	if resp.StatusCode != http.StatusNotFound {
		return false, nil
	}

	resp, err = http.Get(fmt.Sprintf("https://api.github.com/orgs/%s", name))
	if err != nil {
		// TODO: wrap error
		return false, errors.New("Cannot determine name availability")
	}
	resp.Body.Close()

	if resp.StatusCode != http.StatusNotFound {
		return false, nil
	}

	return true, nil
}
<commit_msg>Simplify github implementation<commit_after>package services

import (
	"errors"
	"fmt"
	"net/http"
)

// Github service name checker
// TODO: use Github API token
type Github struct{}

// Check implements the NameChecker interface
func (g *Github) Check(name string) (bool, error) {
	result, err := g.check("users", name)

	if !result {
		return result, err
	}

	result, err = g.check("orgs", name)

	return result, err
}

func (g *Github) check(entity, name string) (bool, error) {
	resp, err := http.Get(fmt.Sprintf("https://api.github.com/%s/%s", entity, name))
	if err != nil {
		// TODO: wrap error
		return false, errors.New("Cannot determine name availability")
	}
	resp.Body.Close()

	if resp.StatusCode == http.StatusNotFound {
		return true, nil
	} else if resp.StatusCode == http.StatusOK {
		return false, nil
	}

	return false, errors.New("Cannot determine name availability")
}
<|endoftext|>