<commit_before>package vbox

import (
    "errors"
    "fmt"
    "time"

    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors"
    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/ssh"

    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/vbox"

    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
)

type VboxExecutor struct {
    executors.AbstractExecutor
	// cmd             *exec.Cmd
    vmName          string
    sshCommand      ssh.Command
    sshPort         string
    provisioned     bool
    machineVerified bool
}

func (s *VboxExecutor) verifyMachine(vmName string, sshPort string) error {
	if s.machineVerified {
		return nil
	}

	// Create SSH command
	sshCommand := ssh.Command{
		Config:         *s.Config.SSH,
		Command:        "exit 0",
		Stdout:         s.BuildLog,
		Stderr:         s.BuildLog,
		ConnectRetries: 30,
	}
    sshCommand.Port = &sshPort

	s.Debugln("Connecting to SSH...")
    err := sshCommand.Connect()
	if err != nil {
		return err
	}
	defer sshCommand.Cleanup()
	err = sshCommand.Run()
	if err != nil {
		return err
	}
	s.machineVerified = true
	return nil
}

func (s *VboxExecutor) restoreFromSnapshot() error {
    s.Debugln("Reverting VM to current snapshot...")
    err := vbox.RevertToSnapshot(s.vmName)
    if err != nil {
        return err
    }

    return nil
}

// Vbox doesn't support templates
func (s *VboxExecutor) CreateVM(vmName string) error {
	baseImage := s.Config.Vbox.BaseName
	if baseImage == "" {
		return errors.New("Missing Image setting from Vbox config")
	}

    vmStatus, _ := vbox.Status(vmName)
    if vmStatus == vbox.Invalid {
        vbox.Unregister(vmName)
    }

    if !vbox.Exist(vmName) {
        s.Debugln("Creating testing VM from VM", baseImage, "...")
        err := vbox.CreateOsVM(baseImage, vmName)
        if err != nil {
            return err
        }
    }

    s.Debugln("Creating localhost ssh forwarding...")
    err := vbox.ConfigureSSH(vmName)
    if err != nil {
        return err
    }

    s.Debugln("Bootstraping VM...")
    err = vbox.Start(s.vmName)
    if err != nil {
        return err
    }

    s.Debugln("Identify SSH Port...")
    sshPort, err := vbox.FindSshPort(s.vmName)
    s.sshPort = sshPort
    if err != nil {
        return err
    }

    s.Debugln("Waiting for VM to become responsive...")
    time.Sleep(10)
    err = s.verifyMachine(s.vmName, s.sshPort)
    if err != nil {
        return err
    }

    return nil
}

func (s *VboxExecutor) Prepare(globalConfig *common.Config, config *common.RunnerConfig, build *common.Build) error {
    err:= s.AbstractExecutor.Prepare(globalConfig, config, build)
    if err != nil {
        return err
    }

    if s.ShellScript.PassFile {
        return errors.New("Vbox doesn't support shells that require script file")
    }

    if s.Config.SSH == nil {
        return errors.New("Missing SSH config")
    }

    if s.Config.Vbox == nil {
        return errors.New("Missing Vbox configuration")
    }

    if s.Config.Vbox.BaseName == "" {
        return errors.New("Missing BaseName setting from Vbox config")
    }

    version, err := vbox.Version()
    if err != nil {
        panic(err)
    }

    s.Println("Using VirtualBox version", version, "executor...")

    vmStatus, _ := vbox.Status(s.vmName)
    if vmStatus == vbox.Invalid {
        vbox.Unregister(s.vmName)
    }

    if helpers.BoolOrDefault(s.Config.Vbox.DisableSnapshots, false) {
        s.vmName =s.Config.Vbox.BaseName + "-" + s.Build.ProjectUniqueName()
        if vbox.Exist(s.vmName) {
            s.Debugln("Deleting old VM...")
            vbox.Stop(s.vmName)
            vbox.Delete(s.vmName)
            vbox.Unregister(s.vmName)
        }
    } else {
        s.vmName = fmt.Sprintf("%s-runner-%s-concurrent-%d",
            s.Config.Vbox.BaseName,
            s.Build.Runner.ShortDescription(),
            s.Build.RunnerID)
    }

    if vbox.Exist(s.vmName) {
        s.Println("Restoring VM from snapshot...")
        err := s.restoreFromSnapshot()
        if err != nil {
            s.Println("Previous VM failed. Deleting, because", err)
            vbox.Stop(s.vmName)
            vbox.Delete(s.vmName)
            vbox.Unregister(s.vmName)
        }
    }

    if !vbox.Exist(s.vmName) {
        s.Println("Creating new VM...")
        err := s.CreateVM(s.vmName)
        if err != nil {
            return err
        }

        if !helpers.BoolOrDefault(s.Config.Vbox.DisableSnapshots, false) {
            s.Println("Creating default snapshot...")
            err = vbox.CreateSnapshot(s.vmName, "Started")
            if err != nil {
                return err
            }
        }
    }

    s.Debugln("Checking VM status...")
    status, err := vbox.Status(s.vmName)
    if err != nil {
        return err
    }

    if status == vbox.Stopped || status == vbox.Suspended || status == vbox.Saved {
        s.Println("Starting VM...")
        err := vbox.Start(s.vmName)
        if err != nil {
            return err
        }
    }

    if status != vbox.Running {
        s.Debugln("Waiting for VM to run...")
        err = vbox.WaitForStatus(s.vmName, vbox.Running, 60)
        if err != nil {
            return err
        }
    }

    s.Debugln("Identify SSH Port...")
    sshPort, err := vbox.FindSshPort(s.vmName)
    s.sshPort = sshPort
    if err != nil {
        return err
    }

    s.Println("Waiting VM to become responsive...")
    err = s.verifyMachine(s.vmName, s.sshPort)
    if err != nil {
        return err
    }

    s.provisioned = true

    return nil
}

func (s *VboxExecutor) Start() error {
    s.Println("Starting SSH command...")
    s.sshCommand = ssh.Command{
		Config:      *s.Config.SSH,
		Environment: s.ShellScript.Environment,
		Command:     s.ShellScript.GetFullCommand(),
		Stdin:       s.ShellScript.GetScriptBytes(),
		Stdout:      s.BuildLog,
		Stderr:      s.BuildLog,
	}
	s.sshCommand.Port = &s.sshPort

    s.Debugln("Connecting to SSH server...")
    err := s.sshCommand.Connect()
	if err != nil {
		return err
	}

	// Wait for process to exit
	go func() {
		s.Debugln("Will run SSH command...")
		err := s.sshCommand.Run()
		s.Debugln("SSH command finished with", err)
		s.BuildFinish <- err
	}()
	return nil
}

func (s *VboxExecutor) Cleanup() {
    s.sshCommand.Cleanup()

    if s.vmName != "" {
        vbox.Kill(s.vmName)

        if helpers.BoolOrDefault(s.Config.Vbox.DisableSnapshots, false) || !s.provisioned {
            vbox.Delete(s.vmName)
        }
    }
}

func init() {
    options := executors.ExecutorOptions{
        DefaultBuildsDir: "builds",
        SharedBuildsDir: false,
        Shell: common.ShellScriptInfo{
            Shell: "bash",
            Type:  common.LoginShell,
        },
        ShowHostname: true,
    }

    create := func() common.Executor {
		return &VboxExecutor{
			AbstractExecutor: executors.AbstractExecutor{
				ExecutorOptions: options,
			},
		}
    }

    common.RegisterExecutor("vbox", common.ExecutorFactory{
        Create: create,
        Features: common.FeaturesInfo{
                Variables: true,
        },
    })
}
<commit_msg>override host value in ssh config for vbox executors<commit_after>package vbox

import (
    "errors"
    "fmt"
    "time"

    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors"
    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers/ssh"

    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/vbox"

    "gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
)

type VboxExecutor struct {
    executors.AbstractExecutor
	// cmd             *exec.Cmd
    vmName          string
    sshCommand      ssh.Command
    sshPort         string
    provisioned     bool
    machineVerified bool
}

func (s *VboxExecutor) verifyMachine(vmName string, sshPort string) error {
	if s.machineVerified {
		return nil
	}

	// Create SSH command
	sshCommand := ssh.Command{
		Config:         *s.Config.SSH,
		Command:        "exit 0",
		Stdout:         s.BuildLog,
		Stderr:         s.BuildLog,
		ConnectRetries: 30,
	}
    host := `localhost`
    sshCommand.Port = &sshPort
    sshCommand.Host = &host

	s.Debugln("Connecting to SSH...")
    err := sshCommand.Connect()
	if err != nil {
		return err
	}
	defer sshCommand.Cleanup()
	err = sshCommand.Run()
	if err != nil {
		return err
	}
	s.machineVerified = true
	return nil
}

func (s *VboxExecutor) restoreFromSnapshot() error {
    s.Debugln("Reverting VM to current snapshot...")
    err := vbox.RevertToSnapshot(s.vmName)
    if err != nil {
        return err
    }

    return nil
}

// Vbox doesn't support templates
func (s *VboxExecutor) CreateVM(vmName string) error {
	baseImage := s.Config.Vbox.BaseName
	if baseImage == "" {
		return errors.New("Missing Image setting from Vbox config")
	}

    vmStatus, _ := vbox.Status(vmName)
    if vmStatus == vbox.Invalid {
        vbox.Unregister(vmName)
    }

    if !vbox.Exist(vmName) {
        s.Debugln("Creating testing VM from VM", baseImage, "...")
        err := vbox.CreateOsVM(baseImage, vmName)
        if err != nil {
            return err
        }
    }

    s.Debugln("Creating localhost ssh forwarding...")
    err := vbox.ConfigureSSH(vmName)
    if err != nil {
        return err
    }

    s.Debugln("Bootstraping VM...")
    err = vbox.Start(s.vmName)
    if err != nil {
        return err
    }

    s.Debugln("Identify SSH Port...")
    sshPort, err := vbox.FindSshPort(s.vmName)
    s.sshPort = sshPort
    if err != nil {
        return err
    }

    s.Debugln("Waiting for VM to become responsive...")
    time.Sleep(10)
    err = s.verifyMachine(s.vmName, s.sshPort)
    if err != nil {
        return err
    }

    return nil
}

func (s *VboxExecutor) Prepare(globalConfig *common.Config, config *common.RunnerConfig, build *common.Build) error {
    err:= s.AbstractExecutor.Prepare(globalConfig, config, build)
    if err != nil {
        return err
    }

    if s.ShellScript.PassFile {
        return errors.New("Vbox doesn't support shells that require script file")
    }

    if s.Config.SSH == nil {
        return errors.New("Missing SSH config")
    }

    if s.Config.Vbox == nil {
        return errors.New("Missing Vbox configuration")
    }

    if s.Config.Vbox.BaseName == "" {
        return errors.New("Missing BaseName setting from Vbox config")
    }

    version, err := vbox.Version()
    if err != nil {
        panic(err)
    }

    s.Println("Using VirtualBox version", version, "executor...")

    vmStatus, _ := vbox.Status(s.vmName)
    if vmStatus == vbox.Invalid {
        vbox.Unregister(s.vmName)
    }

    if helpers.BoolOrDefault(s.Config.Vbox.DisableSnapshots, false) {
        s.vmName =s.Config.Vbox.BaseName + "-" + s.Build.ProjectUniqueName()
        if vbox.Exist(s.vmName) {
            s.Debugln("Deleting old VM...")
            vbox.Stop(s.vmName)
            vbox.Delete(s.vmName)
            vbox.Unregister(s.vmName)
        }
    } else {
        s.vmName = fmt.Sprintf("%s-runner-%s-concurrent-%d",
            s.Config.Vbox.BaseName,
            s.Build.Runner.ShortDescription(),
            s.Build.RunnerID)
    }

    if vbox.Exist(s.vmName) {
        s.Println("Restoring VM from snapshot...")
        err := s.restoreFromSnapshot()
        if err != nil {
            s.Println("Previous VM failed. Deleting, because", err)
            vbox.Stop(s.vmName)
            vbox.Delete(s.vmName)
            vbox.Unregister(s.vmName)
        }
    }

    if !vbox.Exist(s.vmName) {
        s.Println("Creating new VM...")
        err := s.CreateVM(s.vmName)
        if err != nil {
            return err
        }

        if !helpers.BoolOrDefault(s.Config.Vbox.DisableSnapshots, false) {
            s.Println("Creating default snapshot...")
            err = vbox.CreateSnapshot(s.vmName, "Started")
            if err != nil {
                return err
            }
        }
    }

    s.Debugln("Checking VM status...")
    status, err := vbox.Status(s.vmName)
    if err != nil {
        return err
    }

    if status == vbox.Stopped || status == vbox.Suspended || status == vbox.Saved {
        s.Println("Starting VM...")
        err := vbox.Start(s.vmName)
        if err != nil {
            return err
        }
    }

    if status != vbox.Running {
        s.Debugln("Waiting for VM to run...")
        err = vbox.WaitForStatus(s.vmName, vbox.Running, 60)
        if err != nil {
            return err
        }
    }

    s.Debugln("Identify SSH Port...")
    sshPort, err := vbox.FindSshPort(s.vmName)
    s.sshPort = sshPort
    if err != nil {
        return err
    }

    s.Println("Waiting VM to become responsive...")
    err = s.verifyMachine(s.vmName, s.sshPort)
    if err != nil {
        return err
    }

    s.provisioned = true

    return nil
}

func (s *VboxExecutor) Start() error {
    s.Println("Starting SSH command...")
    s.sshCommand = ssh.Command{
		Config:      *s.Config.SSH,
		Environment: s.ShellScript.Environment,
		Command:     s.ShellScript.GetFullCommand(),
		Stdin:       s.ShellScript.GetScriptBytes(),
		Stdout:      s.BuildLog,
		Stderr:      s.BuildLog,
	}
    host := `localhost`
	s.sshCommand.Port = &s.sshPort
    s.sshCommand.Host = &host

    s.Debugln("Connecting to SSH server...")
    err := s.sshCommand.Connect()
	if err != nil {
		return err
	}

	// Wait for process to exit
	go func() {
		s.Debugln("Will run SSH command...")
		err := s.sshCommand.Run()
		s.Debugln("SSH command finished with", err)
		s.BuildFinish <- err
	}()
	return nil
}

func (s *VboxExecutor) Cleanup() {
    s.sshCommand.Cleanup()

    if s.vmName != "" {
        vbox.Kill(s.vmName)

        if helpers.BoolOrDefault(s.Config.Vbox.DisableSnapshots, false) || !s.provisioned {
            vbox.Delete(s.vmName)
        }
    }
}

func init() {
    options := executors.ExecutorOptions{
        DefaultBuildsDir: "builds",
        SharedBuildsDir: false,
        Shell: common.ShellScriptInfo{
            Shell: "bash",
            Type:  common.LoginShell,
        },
        ShowHostname: true,
    }

    create := func() common.Executor {
		return &VboxExecutor{
			AbstractExecutor: executors.AbstractExecutor{
				ExecutorOptions: options,
			},
		}
    }

    common.RegisterExecutor("vbox", common.ExecutorFactory{
        Create: create,
        Features: common.FeaturesInfo{
                Variables: true,
        },
    })
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/hectane/hectane/email"
	"github.com/hectane/hectane/queue"

	"encoding/json"
	"net/http"
)

type rawParams struct {
	From string   `json:"from"`
	To   []string `json:"to"`
	Body string   `json:"body"`
}

// Send a raw MIME message.
func (a *API) raw(r *http.Request) interface{} {
	var p rawParams
	if err := json.NewDecoder(r.Body).Decode(&p); err == nil {
		if w, body, err := a.queue.Storage.NewBody(); err == nil {
			if _, err := w.Write([]byte(p.Body)); err != nil {
				return err
			}
			if err := w.Close(); err != nil {
				return err
			}
			if hostMap, err := email.GroupAddressesByHost(p.To); err == nil {
				for h, to := range hostMap {
					m := &queue.Message{
						Host: h,
						From: p.From,
						To:   to,
					}
					if err := a.queue.Storage.SaveMessage(m, body); err != nil {
						return err
					}
					a.queue.Deliver(m)
				}
			} else {
				return err
			}
		} else {
			return err
		}
		return struct{}{}
	} else {
		return err
	}
}

// Send an email with the specified parameters.
func (a *API) send(r *http.Request) interface{} {
	var e email.Email
	if err := json.NewDecoder(r.Body).Decode(&e); err == nil {
		if messages, err := e.Messages(a.queue.Storage); err == nil {
			for _, m := range messages {
				a.queue.Deliver(m)
			}
			return struct{}{}
		} else {
			return map[string]string{
				"error": err.Error(),
			}
		}
	} else {
		return err
	}
}

// Retrieve status information.
func (a *API) status(r *http.Request) interface{} {
	return a.queue.Status()
}

// Retrieve version information, including the current version of the
// application.
func (a *API) version(r *http.Request) interface{} {
	return map[string]string{
		"version": "0.3.0",
	}
}
<commit_msg>Reformatted some code in the api package.<commit_after>package api

import (
	"github.com/hectane/hectane/email"
	"github.com/hectane/hectane/queue"

	"encoding/json"
	"net/http"
)

type rawParams struct {
	From string   `json:"from"`
	To   []string `json:"to"`
	Body string   `json:"body"`
}

// Send a raw MIME message.
func (a *API) raw(r *http.Request) interface{} {
	var p rawParams
	if err := json.NewDecoder(r.Body).Decode(&p); err != nil {
		return err
	}
	w, body, err := a.queue.Storage.NewBody()
	if err != nil {
		return err
	}
	if _, err := w.Write([]byte(p.Body)); err != nil {
		return err
	}
	if err := w.Close(); err != nil {
		return err
	}
	hostMap, err := email.GroupAddressesByHost(p.To)
	if err == nil {
		return err
	}
	for h, to := range hostMap {
		m := &queue.Message{
			Host: h,
			From: p.From,
			To:   to,
		}
		if err := a.queue.Storage.SaveMessage(m, body); err != nil {
			return err
		}
		a.queue.Deliver(m)
	}
	return struct{}{}
}

// Send an email with the specified parameters.
func (a *API) send(r *http.Request) interface{} {
	var e email.Email
	if err := json.NewDecoder(r.Body).Decode(&e); err != nil {
		return err
	}
	messages, err := e.Messages(a.queue.Storage)
	if err != nil {
		return map[string]string{
			"error": err.Error(),
		}
	}
	for _, m := range messages {
		a.queue.Deliver(m)
	}
	return struct{}{}
}

// Retrieve status information.
func (a *API) status(r *http.Request) interface{} {
	return a.queue.Status()
}

// Retrieve version information, including the current version of the
// application.
func (a *API) version(r *http.Request) interface{} {
	return map[string]string{
		"version": "0.3.0",
	}
}
<|endoftext|>


<commit_before>// This file is licensed under the MIT license.
// See the LICENSE file.

package app

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/coopernurse/gorp"

	// Blank import because we are using postgresql
	_ "github.com/lib/pq"
)

// maxConnectionTries contains the number of connection attempts that this
// application is going to make before panic'ing.
//const maxConnectionTries = 10
const maxConnectionTries = 5

// Global instance that holds a connection to the DB. It gets initialized after
// calling the InitDB function. You have to call CloseDB in order to close the
// connection.
var Db gorp.DbMap

// EnvOrElse returns the value of the given environment variable. If this
// environment variable is not set, then it returns the provided alternative
// value.
func EnvOrElse(name, value string) string {
	if env := os.Getenv(name); env != "" {
		return env
	}
	return value
}

// configURL returns the string being used to connect with our PostgreSQL
// database.
func configURL() string {
	user := EnvOrElse("TODO_DB_USER", "postgres")
	dbname := EnvOrElse("TODO_DB_NAME", "todo-dev")
	password := EnvOrElse("TODO_DB_PASSWORD", "")
	host := EnvOrElse("TODO_DB_HOST", "localhost")
	sslmode := EnvOrElse("TODO_DB_SSLMODE", "disable")

	str := "user=%s host=%s port=5432 dbname=%s sslmode=%s"
	if password != "" {
		str += " password=%s"
		return fmt.Sprintf(str, user, host, dbname, sslmode, password)
	}
	return fmt.Sprintf(str, user, host, dbname, sslmode)
}

// establishConnection tries to establish a connection to the DB. It tries to
// do so until maxConnectionTries is reached, at which point it panics.
func establishConnection() *sql.DB {
	var err error

	str := configURL()
	log.Printf("Trying with: '%s'", str)
	d, err := sql.Open("postgres", str)

	for i := 0; i < maxConnectionTries; i++ {
		if err = d.Ping(); err == nil {
			log.Printf("postgres: connection established.")
			return d
		}
		if i < maxConnectionTries-1 {
			log.Printf("postgres: ping failed: %v", err)
			log.Printf("posgres: retrying in 5 seconds...")
			time.Sleep(5 * time.Second)
		}
	}
	log.Fatalf("postgres: could not establish connection with '%s'.", str)
	return nil
}

// InitDB initializes the global DB connection.
func InitDB() {
	d := establishConnection()

	Db = gorp.DbMap{Db: d, Dialect: gorp.PostgresDialect{}}
	Db.AddTableWithName(User{}, "users")
	Db.AddTableWithName(Topic{}, "topics")
}

// CloseDB close the global DB connection.
func CloseDB() {
	if err := Db.Db.Close(); err != nil {
		log.Printf("Could not close database: %v", err)
	}
}

// Exists returns true if there is a row in the given table that matches the
// given id. It returns false otherwise.
func Exists(name, id string) bool {
	q := fmt.Sprintf("select count(*) from %v where id=$1", name)
	c, err := Db.SelectInt(q, id)
	return err == nil && c == 1
}

// Count the number of rows for the given table. Returns a 0 on error. I know
// that this is not idiomatic, but it comes in handy in this case.
func Count(name string) int64 {
	count, err := Db.SelectInt("select count(*) from " + name)
	if err != nil {
		return 0
	}
	return count
}
<commit_msg>app: fixed golint complain<commit_after>// This file is licensed under the MIT license.
// See the LICENSE file.

package app

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/coopernurse/gorp"

	// Blank import because we are using postgresql
	_ "github.com/lib/pq"
)

// maxConnectionTries contains the number of connection attempts that this
// application is going to make before panic'ing.
//const maxConnectionTries = 10
const maxConnectionTries = 5

// Db is a global instance that holds a connection to the DB. It gets
// initialized after calling the InitDB function. You have to call CloseDB in
// order to close the connection.
var Db gorp.DbMap

// EnvOrElse returns the value of the given environment variable. If this
// environment variable is not set, then it returns the provided alternative
// value.
func EnvOrElse(name, value string) string {
	if env := os.Getenv(name); env != "" {
		return env
	}
	return value
}

// configURL returns the string being used to connect with our PostgreSQL
// database.
func configURL() string {
	user := EnvOrElse("TODO_DB_USER", "postgres")
	dbname := EnvOrElse("TODO_DB_NAME", "todo-dev")
	password := EnvOrElse("TODO_DB_PASSWORD", "")
	host := EnvOrElse("TODO_DB_HOST", "localhost")
	sslmode := EnvOrElse("TODO_DB_SSLMODE", "disable")

	str := "user=%s host=%s port=5432 dbname=%s sslmode=%s"
	if password != "" {
		str += " password=%s"
		return fmt.Sprintf(str, user, host, dbname, sslmode, password)
	}
	return fmt.Sprintf(str, user, host, dbname, sslmode)
}

// establishConnection tries to establish a connection to the DB. It tries to
// do so until maxConnectionTries is reached, at which point it panics.
func establishConnection() *sql.DB {
	var err error

	str := configURL()
	log.Printf("Trying with: '%s'", str)
	d, err := sql.Open("postgres", str)

	for i := 0; i < maxConnectionTries; i++ {
		if err = d.Ping(); err == nil {
			log.Printf("postgres: connection established.")
			return d
		}
		if i < maxConnectionTries-1 {
			log.Printf("postgres: ping failed: %v", err)
			log.Printf("posgres: retrying in 5 seconds...")
			time.Sleep(5 * time.Second)
		}
	}
	log.Fatalf("postgres: could not establish connection with '%s'.", str)
	return nil
}

// InitDB initializes the global DB connection.
func InitDB() {
	d := establishConnection()

	Db = gorp.DbMap{Db: d, Dialect: gorp.PostgresDialect{}}
	Db.AddTableWithName(User{}, "users")
	Db.AddTableWithName(Topic{}, "topics")
}

// CloseDB close the global DB connection.
func CloseDB() {
	if err := Db.Db.Close(); err != nil {
		log.Printf("Could not close database: %v", err)
	}
}

// Exists returns true if there is a row in the given table that matches the
// given id. It returns false otherwise.
func Exists(name, id string) bool {
	q := fmt.Sprintf("select count(*) from %v where id=$1", name)
	c, err := Db.SelectInt(q, id)
	return err == nil && c == 1
}

// Count the number of rows for the given table. Returns a 0 on error. I know
// that this is not idiomatic, but it comes in handy in this case.
func Count(name string) int64 {
	count, err := Db.SelectInt("select count(*) from " + name)
	if err != nil {
		return 0
	}
	return count
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
)

const tmplt = "newsaggtemplate.gohtml"

var wg sync.WaitGroup
var HomeDir, execname, workingdir, ans, ExecTimeStamp, fulltmplt string

type NewsMap struct {
	Keyword  string
	Location string
}

type NewsAggPage struct {
	Title string
	News  map[string]NewsMap
}

type Sitemapindex struct {
	Locations []string `xml:"sitemap>loc"`
}

type News struct {
	Titles    []string `xml:"url>news>title"`
	Keywords  []string `xml:"url>news>keywords"`
	Locations []string `xml:"url>loc"`
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "
 Washington Post Aggregator
")
}

func newsRoutine(c chan News, Location string) {
	defer wg.Done()
	var n News
	resp, _ := http.Get(Location)
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &n)
	resp.Body.Close()
	c <- n
}

func newsAggHandler(w http.ResponseWriter, r *http.Request) {

	var s Sitemapindex
	resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml")
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &s)
	news_map := make(map[string]NewsMap)
	resp.Body.Close()
	queue := make(chan News, 300)

	for _, Location := range s.Locations {
		wg.Add(1)
		go newsRoutine(queue, Location)
	}
	wg.Wait()
	close(queue)

	for elem := range queue {
		for idx, _ := range elem.Keywords {
			news_map[elem.Titles[idx]] = NewsMap{elem.Keywords[idx], elem.Locations[idx]}
		}
	}

	p := NewsAggPage{Title: "A News Aggregator based on the Washington Post", News: news_map}
	t, err := template.ParseFiles(tmplt)
	//	t, err := template.ParseFiles("newsaggtemplate.gohtml")
	check(err)
	//	t, _ := template.ParseFiles("aggregatorfinish.html")
	t.Execute(w, p)
}

func main() {
	if runtime.GOOS == "linux" {
		HomeDir = os.Getenv("HOME")
	} else if runtime.GOOS == "windows" {
		HomeDir = os.Getenv("userprofile")
	} else { // then HomeDir will be empty.
		fmt.Println(" runtime.GOOS does not say linux or windows.  Is this a Mac?")
	}
	workingdir, _ := os.Getwd()

	execname, _ := os.Executable()
	ExecFI, _ := os.Stat(execname)
	ExecTimeStamp = ExecFI.ModTime().Format("Mon Jan 2 2006 15:04:05 MST")
	pathsep := string(os.PathSeparator)
	tmplt1 := workingdir + pathsep + tmplt
	tmplt2 := HomeDir + pathsep + tmplt

	fmt.Println(ExecFI.Name(), "timestamp is", ExecTimeStamp, ".  Full exec is", execname)
	fulltmplt = tmplt1
	_, err := os.Stat(tmplt1)
	if err != nil {
		fulltmplt = tmplt2
		_, err = os.Stat(tmplt2)
		if err != nil {
			fmt.Println(" Template file not found in ", workingdir, " or ", HomeDir, ".  Exiting.")
			os.Exit(1)
		}
	}
	fmt.Println(" Using", fulltmplt, " as template file.")
	fmt.Println()

	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/agg/", newsAggHandler)
	fmt.Print("Hit <enter> to continue   ")
	fmt.Scanln(&ans)
	if strings.ToUpper(ans) == "QUIT" {
		os.Exit(0)
	}

	http.ListenAndServe(":8000", nil)
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}
<commit_msg>modified:   newsagg/newsagg.go -- can be bkgrnd process now<commit_after>package main

import (
	"encoding/xml"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"sync"
)

const tmplt = "newsaggtemplate.gohtml"

var wg sync.WaitGroup
var HomeDir, execname, workingdir, ans, ExecTimeStamp, fulltmplt string

type NewsMap struct {
	Keyword  string
	Location string
}

type NewsAggPage struct {
	Title string
	News  map[string]NewsMap
}

type Sitemapindex struct {
	Locations []string `xml:"sitemap>loc"`
}

type News struct {
	Titles    []string `xml:"url>news>title"`
	Keywords  []string `xml:"url>news>keywords"`
	Locations []string `xml:"url>loc"`
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "
 Washington Post Aggregator
")
}

func newsRoutine(c chan News, Location string) {
	defer wg.Done()
	var n News
	resp, _ := http.Get(Location)
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &n)
	resp.Body.Close()
	c <- n
}

func newsAggHandler(w http.ResponseWriter, r *http.Request) {

	var s Sitemapindex
	resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml")
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &s)
	news_map := make(map[string]NewsMap)
	resp.Body.Close()
	queue := make(chan News, 300)

	for _, Location := range s.Locations {
		wg.Add(1)
		go newsRoutine(queue, Location)
	}
	wg.Wait()
	close(queue)

	for elem := range queue {
		for idx, _ := range elem.Keywords {
			news_map[elem.Titles[idx]] = NewsMap{elem.Keywords[idx], elem.Locations[idx]}
		}
	}

	p := NewsAggPage{Title: "A News Aggregator based on the Washington Post", News: news_map}
	t, err := template.ParseFiles(tmplt)
	//	t, err := template.ParseFiles("newsaggtemplate.gohtml")
	check(err)
	//	t, _ := template.ParseFiles("aggregatorfinish.html")
	t.Execute(w, p)
}

func main() {
	if runtime.GOOS == "linux" {
		HomeDir = os.Getenv("HOME")
	} else if runtime.GOOS == "windows" {
		HomeDir = os.Getenv("userprofile")
	} else { // then HomeDir will be empty.
		fmt.Println(" runtime.GOOS does not say linux or windows.  Is this a Mac?")
	}
	workingdir, _ := os.Getwd()

	execname, _ := os.Executable()
	ExecFI, _ := os.Stat(execname)
	ExecTimeStamp = ExecFI.ModTime().Format("Mon Jan 2 2006 15:04:05 MST")
	pathsep := string(os.PathSeparator)
	tmplt1 := workingdir + pathsep + tmplt
	tmplt2 := HomeDir + pathsep + tmplt

	fmt.Println(ExecFI.Name(), "timestamp is", ExecTimeStamp, ".  Full exec is", execname)
	fulltmplt = tmplt1
	_, err := os.Stat(tmplt1)
	if err != nil {
		fulltmplt = tmplt2
		_, err = os.Stat(tmplt2)
		if err != nil {
			fmt.Println(" Template file not found in ", workingdir, " or ", HomeDir, ".  Exiting.")
			os.Exit(1)
		}
	}
	fmt.Println(" Using", fulltmplt, " as template file.")
	fmt.Println()

	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/agg/", newsAggHandler)
	//	fmt.Print("Hit <enter> to continue   ")
	//	fmt.Scanln(&ans)
	//	if strings.ToUpper(ans) == "QUIT" {
	//		os.Exit(0)
	//	}

	http.ListenAndServe(":8000", nil)
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}
<|endoftext|>


<commit_before>package baudio

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	//"os"
	"os/exec"
	"runtime"
	"strconv"
	//"time"
)

const (
	FuncValueTypeFloat    = 0
	FuncValueTypeNotFloat = 1
)

type BChannel struct {
	funcValueType int
	funcs         []func(float64, int) float64
}

func newBChannel(fvt int) *BChannel {
	bc := &BChannel{
		funcValueType: fvt,
		funcs:         make([]func(float64, int) float64, 0),
	}
	return bc
}

func (bc *BChannel) push(fn func(float64, int) float64) {
	bc.funcs = append(bc.funcs, fn)
}

type bOptions struct {
	size int
	rate int
}

func NewBOptions() *bOptions {
	return &bOptions{
		size: 2048,
		rate: 44000,
	}
}

type B struct {
	readable   bool
	size       int
	rate       int
	t          float64
	i          int
	paused     bool
	ended      bool
	destroyed  bool
	channels   []*BChannel
	chEnd      chan bool
	chEndSox   chan bool
	chResume   chan func()
	chNextTick chan bool
	pipeReader *io.PipeReader
	pipeWriter *io.PipeWriter
}

func New(opts *bOptions, fn func(float64, int) float64) *B {
	b := &B{
		readable:   true,
		size:       2048,
		rate:       44000,
		t:          0,
		i:          0,
		paused:     false,
		ended:      false,
		destroyed:  false,
		chEnd:      make(chan bool),
		chEndSox:   make(chan bool),
		chResume:   make(chan func()),
		chNextTick: make(chan bool),
	}
	b.pipeReader, b.pipeWriter = io.Pipe()
	if opts != nil {
		b.size = opts.size
		b.rate = opts.rate
	}
	if fn != nil {
		b.Push(fn)
	}
	go func() {
		if b.paused {
			b.chResume <- func() {
				go b.loop()
				b.main()
			}
		} else {
			go b.loop()
			b.main()
		}
	}()
	//go b.loop()
	return b
}

func (b *B) main() {
	for {
		// 2013-02-28 koyachi ここで何かしないとループまわらないのなぜ
		// => fmt.PrinfすることでnodeのnextTick的なものがつまれててそのうちPlay()のread待ちまで進めるのでは。
		//L1:
		//fmt.Println("main loop header")
		//fmt.Printf(".")
		//time.Sleep(1 * time.Millisecond)
		runtime.Gosched()
		select {
		case <-b.chEnd:
			fmt.Println("main chEnd")
			b.terminateMain()
			break
		case fn := <-b.chResume:
			//fmt.Println("main chResume")
			fn()
		case <-b.chNextTick:
			//fmt.Println("main chNextTick")
			go b.loop()
			//b.loop()
		default:
			//fmt.Println("main default")
			//go b.loop()
			//goto L1
		}
	}
}

func (b *B) terminateMain() {
	b.pipeWriter.Close()
	b.ended = true
	b.chEndSox <- true
}

func (b *B) end() {
	b.ended = true
}

func (b *B) destroy() {
	b.destroyed = true
	b.chEnd <- true
}

func (b *B) pause() {
	b.paused = true
}

func (b *B) resume() {
	if !b.paused {
		return
	}
	b.paused = false
	b.chResume <- func() {}
}

func (b *B) AddChannel(funcValueType int, fn func(float64, int) float64) {
	bc := newBChannel(funcValueType)
	bc.push(fn)
	b.channels = append(b.channels, bc)
}

func (b *B) PushTo(index int, fn func(float64, int) float64) {
	if len(b.channels) <= index {
		bc := newBChannel(FuncValueTypeFloat)
		b.channels = append(b.channels, bc)
	}
	b.channels[index].funcs = append(b.channels[index].funcs, fn)
}

func (b *B) Push(fn func(float64, int) float64) {
	b.PushTo(len(b.channels), fn)
}

func (b *B) loop() {
	buf := b.tick()
	if b.destroyed {
		// no more events
		//fmt.Println("loop destroyed")
	} else if b.paused {
		//fmt.Println("loop paused")
		b.chResume <- func() {
			b.pipeWriter.Write(buf.Bytes())
			b.chNextTick <- true
		}
	} else {
		//fmt.Println("loop !(destroyed || paused)")
		b.pipeWriter.Write(buf.Bytes())
		if b.ended {
			//fmt.Println("loop ended")
			b.chEnd <- true
		} else {
			//fmt.Println("loop !ended")
			b.chNextTick <- true
		}
	}
}

func (b *B) tick() *bytes.Buffer {
	bufSize := b.size * len(b.channels)
	byteBuffer := make([]byte, 0)
	buf := bytes.NewBuffer(byteBuffer)
	for i := 0; i < bufSize; i += 2 {
		lrIndex := int(i / 2)
		lenCh := len(b.channels)
		ch := b.channels[lrIndex%lenCh]
		t := float64(b.t) + math.Floor(float64(lrIndex))/float64(b.rate)/float64(lenCh)
		counter := b.i + int(math.Floor(float64(lrIndex)/float64(lenCh)))

		value := float64(0)
		n := float64(0)
		for j := 0; j < len(ch.funcs); j++ {
			x := ch.funcs[j](float64(t), counter)
			n += x
		}
		n /= float64(len(ch.funcs))

		if ch.funcValueType == FuncValueTypeFloat {
			value = signed(n)
		} else {
			b_ := math.Pow(2, float64(ch.funcValueType))
			x := math.Mod(math.Floor(n), b_) / b_ * math.Pow(2, 15)
			value = x
		}
		if err := binary.Write(buf, binary.LittleEndian, int16(clamp(value))); err != nil {
			panic(err)
		}
	}
	b.i += b.size / 2
	b.t += float64(b.size) / float64(2) / float64(b.rate)
	return buf
}

func clamp(x float64) float64 {
	return math.Max(math.Min(x, math.Pow(2, 15)-1), -math.Pow(2, 15))
}

func signed(n float64) float64 {
	b := math.Pow(2, 15)
	if n > 0 {
		return math.Min(b-1, math.Floor(b*n-1))
	}
	return math.Max(-b, math.Ceil(b*n-1))
}

func mergeArgs(opts, args map[string]string) []string {
	for k, _ := range opts {
		args[k] = opts[k]
	}
	var resultsLast []string
	var results []string
	for k, _ := range args {
		switch k {
		case "-":
			resultsLast = append(resultsLast, k)
		case "-o":
			resultsLast = append(resultsLast, k, args[k])
		default:
			var dash string
			if len(k) == 1 {
				dash = "-"
			} else {
				dash = "--"
			}
			results = append(results, dash+k, args[k])
		}
	}
	results = append(results, resultsLast...)
	fmt.Printf("results = %v\n", results)
	return results
}

func (b *B) runCommand(command string, mergedArgs []string) {
	cmd := exec.Command(command, mergedArgs...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	defer func() {
		fmt.Println("runCommand: before stdin.Close()")
		stdin.Close()
	}()
	var out bytes.Buffer
	cmd.Stdout = &out
	// TODO: option
	//cmd.Stdout = os.Stdout
	//cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		panic(err)
	}
	defer func() {
		if p := cmd.Process; p != nil {
			fmt.Println("runCommand: before p.Kill()")
			p.Kill()
		}
	}()

	readBuf := make([]byte, b.size*len(b.channels))
	for {
		//fmt.Println("play loop header")
		if _, err := b.pipeReader.Read(readBuf); err != nil {
			panic(err)
		}
		if _, err = stdin.Write(readBuf); err != nil {
			// TODO: more better error handling
			if err.Error() == "write |1: broken pipe" {
				fmt.Printf("ERR: stdin.Write(readBuf): err = %v\n", err)
				runtime.Gosched()
				break
			}
			panic(err)
		}
	}
}

func (b *B) Play(opts map[string]string) {
	go b.runCommand("play", mergeArgs(opts, map[string]string{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) Record(file string, opts map[string]string) {
	go b.runCommand("sox", mergeArgs(opts, map[string]string{
		"c":  strconv.Itoa(len(b.channels)),
		"r":  strconv.Itoa(b.rate),
		"t":  "s16",
		"-":  "DUMMY",
		"-o": file,
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}
<commit_msg>to public few methods.<commit_after>package baudio

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	//"os"
	"os/exec"
	"runtime"
	"strconv"
	//"time"
)

const (
	FuncValueTypeFloat    = 0
	FuncValueTypeNotFloat = 1
)

type BChannel struct {
	funcValueType int
	funcs         []func(float64, int) float64
}

func newBChannel(fvt int) *BChannel {
	bc := &BChannel{
		funcValueType: fvt,
		funcs:         make([]func(float64, int) float64, 0),
	}
	return bc
}

func (bc *BChannel) push(fn func(float64, int) float64) {
	bc.funcs = append(bc.funcs, fn)
}

type bOptions struct {
	size int
	rate int
}

func NewBOptions() *bOptions {
	return &bOptions{
		size: 2048,
		rate: 44000,
	}
}

type B struct {
	readable   bool
	size       int
	rate       int
	t          float64
	i          int
	paused     bool
	ended      bool
	destroyed  bool
	channels   []*BChannel
	chEnd      chan bool
	chEndSox   chan bool
	chResume   chan func()
	chNextTick chan bool
	pipeReader *io.PipeReader
	pipeWriter *io.PipeWriter
}

func New(opts *bOptions, fn func(float64, int) float64) *B {
	b := &B{
		readable:   true,
		size:       2048,
		rate:       44000,
		t:          0,
		i:          0,
		paused:     false,
		ended:      false,
		destroyed:  false,
		chEnd:      make(chan bool),
		chEndSox:   make(chan bool),
		chResume:   make(chan func()),
		chNextTick: make(chan bool),
	}
	b.pipeReader, b.pipeWriter = io.Pipe()
	if opts != nil {
		b.size = opts.size
		b.rate = opts.rate
	}
	if fn != nil {
		b.Push(fn)
	}
	go func() {
		if b.paused {
			b.chResume <- func() {
				go b.loop()
				b.main()
			}
		} else {
			go b.loop()
			b.main()
		}
	}()
	//go b.loop()
	return b
}

func (b *B) main() {
	for {
		// 2013-02-28 koyachi ここで何かしないとループまわらないのなぜ
		// => fmt.PrinfすることでnodeのnextTick的なものがつまれててそのうちPlay()のread待ちまで進めるのでは。
		//L1:
		//fmt.Println("main loop header")
		//fmt.Printf(".")
		//time.Sleep(1 * time.Millisecond)
		runtime.Gosched()
		select {
		case <-b.chEnd:
			fmt.Println("main chEnd")
			b.terminateMain()
			break
		case fn := <-b.chResume:
			//fmt.Println("main chResume")
			fn()
		case <-b.chNextTick:
			//fmt.Println("main chNextTick")
			go b.loop()
			//b.loop()
		default:
			//fmt.Println("main default")
			//go b.loop()
			//goto L1
		}
	}
}

func (b *B) terminateMain() {
	b.pipeWriter.Close()
	b.ended = true
	b.chEndSox <- true
}

func (b *B) End() {
	b.ended = true
}

func (b *B) Destroy() {
	b.destroyed = true
	b.chEnd <- true
}

func (b *B) Pause() {
	b.paused = true
}

func (b *B) Resume() {
	if !b.paused {
		return
	}
	b.paused = false
	b.chResume <- func() {}
}

func (b *B) AddChannel(funcValueType int, fn func(float64, int) float64) {
	bc := newBChannel(funcValueType)
	bc.push(fn)
	b.channels = append(b.channels, bc)
}

func (b *B) PushTo(index int, fn func(float64, int) float64) {
	if len(b.channels) <= index {
		bc := newBChannel(FuncValueTypeFloat)
		b.channels = append(b.channels, bc)
	}
	b.channels[index].funcs = append(b.channels[index].funcs, fn)
}

func (b *B) Push(fn func(float64, int) float64) {
	b.PushTo(len(b.channels), fn)
}

func (b *B) loop() {
	buf := b.tick()
	if b.destroyed {
		// no more events
		//fmt.Println("loop destroyed")
	} else if b.paused {
		//fmt.Println("loop paused")
		b.chResume <- func() {
			b.pipeWriter.Write(buf.Bytes())
			b.chNextTick <- true
		}
	} else {
		//fmt.Println("loop !(destroyed || paused)")
		b.pipeWriter.Write(buf.Bytes())
		if b.ended {
			//fmt.Println("loop ended")
			b.chEnd <- true
		} else {
			//fmt.Println("loop !ended")
			b.chNextTick <- true
		}
	}
}

func (b *B) tick() *bytes.Buffer {
	bufSize := b.size * len(b.channels)
	byteBuffer := make([]byte, 0)
	buf := bytes.NewBuffer(byteBuffer)
	for i := 0; i < bufSize; i += 2 {
		lrIndex := int(i / 2)
		lenCh := len(b.channels)
		ch := b.channels[lrIndex%lenCh]
		t := float64(b.t) + math.Floor(float64(lrIndex))/float64(b.rate)/float64(lenCh)
		counter := b.i + int(math.Floor(float64(lrIndex)/float64(lenCh)))

		value := float64(0)
		n := float64(0)
		for j := 0; j < len(ch.funcs); j++ {
			x := ch.funcs[j](float64(t), counter)
			n += x
		}
		n /= float64(len(ch.funcs))

		if ch.funcValueType == FuncValueTypeFloat {
			value = signed(n)
		} else {
			b_ := math.Pow(2, float64(ch.funcValueType))
			x := math.Mod(math.Floor(n), b_) / b_ * math.Pow(2, 15)
			value = x
		}
		if err := binary.Write(buf, binary.LittleEndian, int16(clamp(value))); err != nil {
			panic(err)
		}
	}
	b.i += b.size / 2
	b.t += float64(b.size) / float64(2) / float64(b.rate)
	return buf
}

func clamp(x float64) float64 {
	return math.Max(math.Min(x, math.Pow(2, 15)-1), -math.Pow(2, 15))
}

func signed(n float64) float64 {
	b := math.Pow(2, 15)
	if n > 0 {
		return math.Min(b-1, math.Floor(b*n-1))
	}
	return math.Max(-b, math.Ceil(b*n-1))
}

func mergeArgs(opts, args map[string]string) []string {
	for k, _ := range opts {
		args[k] = opts[k]
	}
	var resultsLast []string
	var results []string
	for k, _ := range args {
		switch k {
		case "-":
			resultsLast = append(resultsLast, k)
		case "-o":
			resultsLast = append(resultsLast, k, args[k])
		default:
			var dash string
			if len(k) == 1 {
				dash = "-"
			} else {
				dash = "--"
			}
			results = append(results, dash+k, args[k])
		}
	}
	results = append(results, resultsLast...)
	fmt.Printf("results = %v\n", results)
	return results
}

func (b *B) runCommand(command string, mergedArgs []string) {
	cmd := exec.Command(command, mergedArgs...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	defer func() {
		fmt.Println("runCommand: before stdin.Close()")
		stdin.Close()
	}()
	var out bytes.Buffer
	cmd.Stdout = &out
	// TODO: option
	//cmd.Stdout = os.Stdout
	//cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		panic(err)
	}
	defer func() {
		if p := cmd.Process; p != nil {
			fmt.Println("runCommand: before p.Kill()")
			p.Kill()
		}
	}()

	readBuf := make([]byte, b.size*len(b.channels))
	for {
		//fmt.Println("play loop header")
		if _, err := b.pipeReader.Read(readBuf); err != nil {
			panic(err)
		}
		if _, err = stdin.Write(readBuf); err != nil {
			// TODO: more better error handling
			if err.Error() == "write |1: broken pipe" {
				fmt.Printf("ERR: stdin.Write(readBuf): err = %v\n", err)
				runtime.Gosched()
				break
			}
			panic(err)
		}
	}
}

func (b *B) Play(opts map[string]string) {
	go b.runCommand("play", mergeArgs(opts, map[string]string{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) Record(file string, opts map[string]string) {
	go b.runCommand("sox", mergeArgs(opts, map[string]string{
		"c":  strconv.Itoa(len(b.channels)),
		"r":  strconv.Itoa(b.rate),
		"t":  "s16",
		"-":  "DUMMY",
		"-o": file,
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}
<|endoftext|>


<commit_before>package sparse

import (
	"bytes"
	"fmt"
	"unsafe"

	"gonum.org/v1/gonum/mat"
)

const (
	// maxLen is the biggest slice/array len one can create on a 32/64b platform.
	//bitLen = ^uint(0) >> 1
	maxDataLen = ^uint(0)

	// the wordSize of a binary vector.  Bits are stored in slices of words.
	wordSize = uint(64)

	// log2WordSize is the binary logarithm (log base 2) of (wordSize)
	log2WordSize = uint(6)
)

// BinaryVec is a Binary Vector or Bit Vector type.  This is useful
// for representing vectors of features with a binary state (1 or 0).
// Although part of the sparse package, this type is not sparse itself
// and stores all bits even 0s.  However, as it makes use of 64 bit
// integers to store each set of 64 bits and then bitwise operators to
// manipulate the elements it will be more efficient in terms of both
// storage requirements and performance than a slice of bool values
// (8 bits per element) or even a typical Dense matrix of float64
// elements.  A compressed bitmap scheme could be used to take advantage
// of sparseness but may have an associated overhead.
type BinaryVec struct {
	length int
	data   []uint64
}

// NewBinaryVec creates a new BitSet with a hint that length bits will be required
func NewBinaryVec(length int) *BinaryVec {
	maxSize := (maxDataLen - wordSize + uint(1))
	if uint(length) > maxSize {
		panic(fmt.Errorf("sparse: Requested bit length of Binary vector (%d) too large.  %d is the maximum allowed", length, maxSize))
	}
	elements := int((uint(length) + (wordSize - 1)) >> log2WordSize)

	vec := &BinaryVec{
		length: length,
		data:   make([]uint64, elements),
	}

	return vec
}

// DistanceFrom is the number of bits that are different between the
// receiver and rhs i.e.
// 	recevier 	= 1001001
//	rhs 		= 1010101
// 	Distance	= 3
// because there are three bits that are different between the 2
// binary vectors.  This is sometimes referred to as the `Hamming
// distance` or `Matching distance`.  In this case, the distance
// is not normalised and is simply the raw count of differences.  To
// normalise the value simply divide this value by the vector's length.
func (b *BinaryVec) DistanceFrom(rhs *BinaryVec) int {
	differences := uint64(0)
	for i, word := range b.data {
		differences += popcount(word ^ rhs.data[i])
	}
	return int(differences)
}

// Dims returns the dimensions of the matrix as the number of rows, columns.
// As this is a vector, the second value representing the number of columns
// will be 1. This method is part of the Gonum mat.Matrix interface
func (b *BinaryVec) Dims() (int, int) {
	return b.Len(), 1
}

// At returns the value of the element at row i and column j.
// As this is a vector (only one column), j must be 0 otherwise the
// method panics.  This method is part of the Gonum mat.Matrix interface.
func (b *BinaryVec) At(i, j int) float64 {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	if j != 0 {
		panic(mat.ErrColAccess)
	}

	if b.bitIsSet(i) {
		return 1.0
	}
	return 0.0
}

// AtVec returns the value of the element at row i.  This method will panic if
// i > Len().  This method is part of the Gonum mat.Vector interface.
func (b *BinaryVec) AtVec(i int) float64 {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}

	if b.bitIsSet(i) {
		return 1.0
	}
	return 0.0
}

// T performs an implicit transpose by returning the receiver inside a Transpose.
// This method is part of the Gonum mat.Matrix interface
func (b *BinaryVec) T() mat.Matrix {
	return mat.Transpose{Matrix: b}
}

// NNZ returns the Number of Non-Zero elements (bits).  This is the number of set
// bits (represented by 1s rather than 0s) in the vector.  This is also known as the
// `Hamming weight` or `population count` (popcount).
func (b *BinaryVec) NNZ() int {
	nnz := uint64(0)
	for _, word := range b.data {
		nnz += popcount(word)
	}
	return int(nnz)
}

// Len returns the length of the vector or the total number of elements.  This method
// is part of the Gonum mat.Vector interface
func (b *BinaryVec) Len() int {
	return b.length
}

// BitIsSet tests whether the element (bit) at position i is set (equals 1) and
// returns true if so.  If the element (bit) is not set or has been unset (equal
// to 0) the the method will return false.  The method will panic if i is greater
// than Len().
func (b *BinaryVec) BitIsSet(i int) bool {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	return b.bitIsSet(i)
}

// bitIsSet tests whether the element (bit) at position i is set (equals 1) and
// returns true if so.  If the element (bit) is not set or has been unset (equal
// to 0) the the method will return false.
func (b *BinaryVec) bitIsSet(i int) bool {
	return b.data[i>>log2WordSize]&(1<<(uint(i)&(wordSize-1))) != 0
}

// SetBit sets the bit at the specified index (i) to 1.  If the bit is already set
// there are no adverse effects.  The method will panic if index is larger
// than Len()
func (b *BinaryVec) SetBit(i int) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	b.setBit(i)
}

// setBit sets the bit at the specified index (i) to 1.  If the bit is already set
// there are no adverse effects.
func (b *BinaryVec) setBit(i int) {
	b.data[i>>log2WordSize] |= 1 << (uint(i) & (wordSize - 1))
}

// UnsetBit unsets the bit at the specified index (i) (sets it to 0).  If the bit
// is already unset or has simply never been set (default bit values are 0)
// there are no adverse effects.  The method will panic if index is larger
// than Len()
func (b *BinaryVec) UnsetBit(i int) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	b.unsetBit(i)
}

// unsetBit unsets the bit at the specified index (i) (sets it to 0).  If the bit
// is already unset or has simply never been set (default bit values are 0)
// there are no adverse effects.
func (b *BinaryVec) unsetBit(i int) {
	b.data[i>>log2WordSize] &^= 1 << (uint(i) & (wordSize - 1))
}

// Set sets the element of the matrix located at row i and column j to 1 if v != 0
// or 0 otherwise. Set will panic if specified values for i or j fall outside the
// dimensions of the matrix.
func (b *BinaryVec) Set(i int, j int, v float64) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	if j != 0 {
		panic(mat.ErrColAccess)
	}

	if v != 0 {
		b.setBit(i)
		return
	}
	b.unsetBit(i)
}

// SetVec sets the element of the vector located at row i to 1 if v != 0
// or 0 otherwise. The method will panic if i is greater than Len().
func (b *BinaryVec) SetVec(i int, v float64) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}

	if v != 0 {
		b.setBit(i)
		return
	}
	b.unsetBit(i)
}

// SliceToUint64 returns a new uint64.
// The returned matrix starts at element from of the receiver and extends
// to - from rows. The final row in the resulting matrix is to-1.
// Slice panics with ErrIndexOutOfRange if the slice is outside the capacity
// of the receiver.
func (b *BinaryVec) SliceToUint64(from, to int) uint64 {
	if from < 0 || to <= from || to >= b.length || to-from > 64 {
		panic(mat.ErrIndexOutOfRange)
	}

	var result uint64
	var k uint64
	for i := from; i < to; i++ {
		if b.bitIsSet(i) {
			result |= 1 << k
		}
		k++
	}

	return result
}

// String will output the vector as a string representation of its bits
// This method implements the fmt.Stringer interface.
func (b BinaryVec) String() string {
	buf := bytes.NewBuffer(make([]byte, 0, b.Len()))

	width := b.length % int(wordSize)
	if width == 0 {
		width = 64
	}

	fmt.Fprintf(buf, fmt.Sprintf("%%0%db", width), b.data[len(b.data)-1])
	for i := len(b.data) - 2; i >= 0; i-- {
		fmt.Fprintf(buf, "%064b", b.data[i])
	}

	s := buf.Bytes()
	return *(*string)(unsafe.Pointer(&s))
}

// Format outputs the vector to f and allows the output format
// to be specified.  Supported values of c are `x`, `X`, `b`` and `s`
// to format the bits of the vector as a hex digit or binary digit string.
// `s` (the default format) will output as binary digits.
// Please refer to the fmt package documentation for more information.
// This method implements the fmt.Formatter interface.
func (b BinaryVec) Format(f fmt.State, c rune) {
	var buf bytes.Buffer
	var format string
	var leadFormat string
	switch c {
	case 'x':
		format = ".%x"
		leadFormat = "%x"
	case 'X':
		format = ".%X"
		leadFormat = "%X"
	case 'b':
		f.Write([]byte(b.String()))
		return
	case 's':
		f.Write([]byte(b.String()))
		return
	default:
		panic(fmt.Errorf("sparse: unsupported format verb '%c' for Binary vector", c))
	}
	fmt.Fprintf(&buf, leadFormat, b.data[len(b.data)-1])
	for i := len(b.data) - 2; i >= 0; i-- {
		fmt.Fprintf(&buf, format, b.data[i])
	}
	f.Write(buf.Bytes())
}

// popcount calculates the population count of the vector (also known
// as `Hamming weight`).  This uses fewer arithmetic operations than
// any other known implementation on machines with fast multiplication.
// Thanks to Wikipedia and Hacker's Delight.
func popcount(x uint64) (n uint64) {
	x -= (x >> 1) & 0x5555555555555555
	x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
	x += x >> 4
	x &= 0x0f0f0f0f0f0f0f0f
	x *= 0x0101010101010101
	return x >> 56
}

// Binary is a Binary Matrix or Bit Matrix type.
// Although part of the sparse package, this type is not sparse itself
// and stores all bits even 0s.  However, as it makes use of 64 bit
// integers to store each set of 64 bits and then bitwise operators to
// manipulate the elements it will be more efficient in terms of both
// storage requirements and performance than a slice of bool values
// (8 bits per element) or even a typical Dense matrix of float64
// elements.  A compressed bitmap scheme could be used to take advantage
// of sparseness but may have an associated overhead.
type Binary struct {
	r, c int
	cols []BinaryVec
}

// NewBinary constructs a new Binary matrix or r rows and c columns.
// If vecs is not nil, it will be used as the underlying binary column vectors.
// If vecs is nil, new storage will be allocated.
func NewBinary(r, c int, vecs []BinaryVec) *Binary {
	if vecs == nil {
		vecs = make([]BinaryVec, c)
		for i := 0; i < c; i++ {
			vecs[i] = *NewBinaryVec(r)
		}
	}

	return &Binary{r: r, c: c, cols: vecs}
}

// Dims returns the dimensions of the matrix as the number of rows, columns.
// This method is part of the Gonum mat.Matrix interface
func (b *Binary) Dims() (int, int) {
	return b.r, b.c
}

// At returns the value of the element at row i and column k.
// i (row) and j (col) must be within the dimensions of the matrix otherwise the
// method panics.  This method is part of the Gonum mat.Matrix interface.
func (b *Binary) At(i int, j int) float64 {
	if j < 0 || j >= b.c {
		panic(mat.ErrColAccess)
	}
	return b.cols[j].AtVec(i)
}

// T performs an implicit transpose by returning the receiver inside a Transpose.
// This method is part of the Gonum mat.Matrix interface
func (b *Binary) T() mat.Matrix {
	return mat.Transpose{Matrix: b}
}

// ColView returns the mat.Vector representing the column j.  This vector will
// be a BinaryVec and will share the same storage as the matrix so any changes
// to the vector will be reflected in the matrix and vice versa.
// if j is outside the dimensions of the matrix the method will panic.
func (b *Binary) ColView(j int) mat.Vector {
	if j < 0 || j >= b.c {
		panic(mat.ErrColAccess)
	}
	return &b.cols[j]
}
<commit_msg>added type assertions for static interface implementation checks<commit_after>package sparse

import (
	"bytes"
	"fmt"
	"unsafe"

	"gonum.org/v1/gonum/mat"
)

var (
	_ mat.Matrix = (*BinaryVec)(nil)
	_ mat.Vector = (*BinaryVec)(nil)

	_ mat.Matrix    = (*Binary)(nil)
	_ mat.ColViewer = (*Binary)(nil)
)

const (
	// maxLen is the biggest slice/array len one can create on a 32/64b platform.
	//bitLen = ^uint(0) >> 1
	maxDataLen = ^uint(0)

	// the wordSize of a binary vector.  Bits are stored in slices of words.
	wordSize = uint(64)

	// log2WordSize is the binary logarithm (log base 2) of (wordSize)
	log2WordSize = uint(6)
)

// BinaryVec is a Binary Vector or Bit Vector type.  This is useful
// for representing vectors of features with a binary state (1 or 0).
// Although part of the sparse package, this type is not sparse itself
// and stores all bits even 0s.  However, as it makes use of 64 bit
// integers to store each set of 64 bits and then bitwise operators to
// manipulate the elements it will be more efficient in terms of both
// storage requirements and performance than a slice of bool values
// (8 bits per element) or even a typical Dense matrix of float64
// elements.  A compressed bitmap scheme could be used to take advantage
// of sparseness but may have an associated overhead.
type BinaryVec struct {
	length int
	data   []uint64
}

// NewBinaryVec creates a new BitSet with a hint that length bits will be required
func NewBinaryVec(length int) *BinaryVec {
	maxSize := (maxDataLen - wordSize + uint(1))
	if uint(length) > maxSize {
		panic(fmt.Errorf("sparse: Requested bit length of Binary vector (%d) too large.  %d is the maximum allowed", length, maxSize))
	}
	elements := int((uint(length) + (wordSize - 1)) >> log2WordSize)

	vec := &BinaryVec{
		length: length,
		data:   make([]uint64, elements),
	}

	return vec
}

// DistanceFrom is the number of bits that are different between the
// receiver and rhs i.e.
// 	recevier 	= 1001001
//	rhs 		= 1010101
// 	Distance	= 3
// because there are three bits that are different between the 2
// binary vectors.  This is sometimes referred to as the `Hamming
// distance` or `Matching distance`.  In this case, the distance
// is not normalised and is simply the raw count of differences.  To
// normalise the value simply divide this value by the vector's length.
func (b *BinaryVec) DistanceFrom(rhs *BinaryVec) int {
	differences := uint64(0)
	for i, word := range b.data {
		differences += popcount(word ^ rhs.data[i])
	}
	return int(differences)
}

// Dims returns the dimensions of the matrix as the number of rows, columns.
// As this is a vector, the second value representing the number of columns
// will be 1. This method is part of the Gonum mat.Matrix interface
func (b *BinaryVec) Dims() (int, int) {
	return b.Len(), 1
}

// At returns the value of the element at row i and column j.
// As this is a vector (only one column), j must be 0 otherwise the
// method panics.  This method is part of the Gonum mat.Matrix interface.
func (b *BinaryVec) At(i, j int) float64 {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	if j != 0 {
		panic(mat.ErrColAccess)
	}

	if b.bitIsSet(i) {
		return 1.0
	}
	return 0.0
}

// AtVec returns the value of the element at row i.  This method will panic if
// i > Len().  This method is part of the Gonum mat.Vector interface.
func (b *BinaryVec) AtVec(i int) float64 {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}

	if b.bitIsSet(i) {
		return 1.0
	}
	return 0.0
}

// T performs an implicit transpose by returning the receiver inside a Transpose.
// This method is part of the Gonum mat.Matrix interface
func (b *BinaryVec) T() mat.Matrix {
	return mat.Transpose{Matrix: b}
}

// NNZ returns the Number of Non-Zero elements (bits).  This is the number of set
// bits (represented by 1s rather than 0s) in the vector.  This is also known as the
// `Hamming weight` or `population count` (popcount).
func (b *BinaryVec) NNZ() int {
	nnz := uint64(0)
	for _, word := range b.data {
		nnz += popcount(word)
	}
	return int(nnz)
}

// Len returns the length of the vector or the total number of elements.  This method
// is part of the Gonum mat.Vector interface
func (b *BinaryVec) Len() int {
	return b.length
}

// BitIsSet tests whether the element (bit) at position i is set (equals 1) and
// returns true if so.  If the element (bit) is not set or has been unset (equal
// to 0) the the method will return false.  The method will panic if i is greater
// than Len().
func (b *BinaryVec) BitIsSet(i int) bool {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	return b.bitIsSet(i)
}

// bitIsSet tests whether the element (bit) at position i is set (equals 1) and
// returns true if so.  If the element (bit) is not set or has been unset (equal
// to 0) the the method will return false.
func (b *BinaryVec) bitIsSet(i int) bool {
	return b.data[i>>log2WordSize]&(1<<(uint(i)&(wordSize-1))) != 0
}

// SetBit sets the bit at the specified index (i) to 1.  If the bit is already set
// there are no adverse effects.  The method will panic if index is larger
// than Len()
func (b *BinaryVec) SetBit(i int) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	b.setBit(i)
}

// setBit sets the bit at the specified index (i) to 1.  If the bit is already set
// there are no adverse effects.
func (b *BinaryVec) setBit(i int) {
	b.data[i>>log2WordSize] |= 1 << (uint(i) & (wordSize - 1))
}

// UnsetBit unsets the bit at the specified index (i) (sets it to 0).  If the bit
// is already unset or has simply never been set (default bit values are 0)
// there are no adverse effects.  The method will panic if index is larger
// than Len()
func (b *BinaryVec) UnsetBit(i int) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	b.unsetBit(i)
}

// unsetBit unsets the bit at the specified index (i) (sets it to 0).  If the bit
// is already unset or has simply never been set (default bit values are 0)
// there are no adverse effects.
func (b *BinaryVec) unsetBit(i int) {
	b.data[i>>log2WordSize] &^= 1 << (uint(i) & (wordSize - 1))
}

// Set sets the element of the matrix located at row i and column j to 1 if v != 0
// or 0 otherwise. Set will panic if specified values for i or j fall outside the
// dimensions of the matrix.
func (b *BinaryVec) Set(i int, j int, v float64) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}
	if j != 0 {
		panic(mat.ErrColAccess)
	}

	if v != 0 {
		b.setBit(i)
		return
	}
	b.unsetBit(i)
}

// SetVec sets the element of the vector located at row i to 1 if v != 0
// or 0 otherwise. The method will panic if i is greater than Len().
func (b *BinaryVec) SetVec(i int, v float64) {
	if i < 0 || i >= b.length {
		panic(mat.ErrRowAccess)
	}

	if v != 0 {
		b.setBit(i)
		return
	}
	b.unsetBit(i)
}

// SliceToUint64 returns a new uint64.
// The returned matrix starts at element from of the receiver and extends
// to - from rows. The final row in the resulting matrix is to-1.
// Slice panics with ErrIndexOutOfRange if the slice is outside the capacity
// of the receiver.
func (b *BinaryVec) SliceToUint64(from, to int) uint64 {
	if from < 0 || to <= from || to >= b.length || to-from > 64 {
		panic(mat.ErrIndexOutOfRange)
	}

	var result uint64
	var k uint64
	for i := from; i < to; i++ {
		if b.bitIsSet(i) {
			result |= 1 << k
		}
		k++
	}

	return result
}

// String will output the vector as a string representation of its bits
// This method implements the fmt.Stringer interface.
func (b BinaryVec) String() string {
	buf := bytes.NewBuffer(make([]byte, 0, b.Len()))

	width := b.length % int(wordSize)
	if width == 0 {
		width = 64
	}

	fmt.Fprintf(buf, fmt.Sprintf("%%0%db", width), b.data[len(b.data)-1])
	for i := len(b.data) - 2; i >= 0; i-- {
		fmt.Fprintf(buf, "%064b", b.data[i])
	}

	s := buf.Bytes()
	return *(*string)(unsafe.Pointer(&s))
}

// Format outputs the vector to f and allows the output format
// to be specified.  Supported values of c are `x`, `X`, `b`` and `s`
// to format the bits of the vector as a hex digit or binary digit string.
// `s` (the default format) will output as binary digits.
// Please refer to the fmt package documentation for more information.
// This method implements the fmt.Formatter interface.
func (b BinaryVec) Format(f fmt.State, c rune) {
	var buf bytes.Buffer
	var format string
	var leadFormat string
	switch c {
	case 'x':
		format = ".%x"
		leadFormat = "%x"
	case 'X':
		format = ".%X"
		leadFormat = "%X"
	case 'b':
		f.Write([]byte(b.String()))
		return
	case 's':
		f.Write([]byte(b.String()))
		return
	default:
		panic(fmt.Errorf("sparse: unsupported format verb '%c' for Binary vector", c))
	}
	fmt.Fprintf(&buf, leadFormat, b.data[len(b.data)-1])
	for i := len(b.data) - 2; i >= 0; i-- {
		fmt.Fprintf(&buf, format, b.data[i])
	}
	f.Write(buf.Bytes())
}

// popcount calculates the population count of the vector (also known
// as `Hamming weight`).  This uses fewer arithmetic operations than
// any other known implementation on machines with fast multiplication.
// Thanks to Wikipedia and Hacker's Delight.
func popcount(x uint64) (n uint64) {
	x -= (x >> 1) & 0x5555555555555555
	x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
	x += x >> 4
	x &= 0x0f0f0f0f0f0f0f0f
	x *= 0x0101010101010101
	return x >> 56
}

// Binary is a Binary Matrix or Bit Matrix type.
// Although part of the sparse package, this type is not sparse itself
// and stores all bits even 0s.  However, as it makes use of 64 bit
// integers to store each set of 64 bits and then bitwise operators to
// manipulate the elements it will be more efficient in terms of both
// storage requirements and performance than a slice of bool values
// (8 bits per element) or even a typical Dense matrix of float64
// elements.  A compressed bitmap scheme could be used to take advantage
// of sparseness but may have an associated overhead.
type Binary struct {
	r, c int
	cols []BinaryVec
}

// NewBinary constructs a new Binary matrix or r rows and c columns.
// If vecs is not nil, it will be used as the underlying binary column vectors.
// If vecs is nil, new storage will be allocated.
func NewBinary(r, c int, vecs []BinaryVec) *Binary {
	if vecs == nil {
		vecs = make([]BinaryVec, c)
		for i := 0; i < c; i++ {
			vecs[i] = *NewBinaryVec(r)
		}
	}

	return &Binary{r: r, c: c, cols: vecs}
}

// Dims returns the dimensions of the matrix as the number of rows, columns.
// This method is part of the Gonum mat.Matrix interface
func (b *Binary) Dims() (int, int) {
	return b.r, b.c
}

// At returns the value of the element at row i and column k.
// i (row) and j (col) must be within the dimensions of the matrix otherwise the
// method panics.  This method is part of the Gonum mat.Matrix interface.
func (b *Binary) At(i int, j int) float64 {
	if j < 0 || j >= b.c {
		panic(mat.ErrColAccess)
	}
	return b.cols[j].AtVec(i)
}

// T performs an implicit transpose by returning the receiver inside a Transpose.
// This method is part of the Gonum mat.Matrix interface
func (b *Binary) T() mat.Matrix {
	return mat.Transpose{Matrix: b}
}

// ColView returns the mat.Vector representing the column j.  This vector will
// be a BinaryVec and will share the same storage as the matrix so any changes
// to the vector will be reflected in the matrix and vice versa.
// if j is outside the dimensions of the matrix the method will panic.
func (b *Binary) ColView(j int) mat.Vector {
	if j < 0 || j >= b.c {
		panic(mat.ErrColAccess)
	}
	return &b.cols[j]
}
<|endoftext|>


<commit_before>package gokeepasslib

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"

	w "github.com/tobischo/gokeepasslib/v3/wrappers"
)

// Binaries Stores a slice of binaries in the metadata header of a database
// This will be used only on KDBX 3.1
// Since KDBX 4, binaries are stored into the InnerHeader
type Binaries []Binary

// Binary stores a binary found in the metadata header of a database
type Binary struct {
	ID               int           `xml:"ID,attr"`         // Index of binary (Manually counted on KDBX v4)
	MemoryProtection byte          `xml:"-"`               // Memory protection flag (Only KDBX v4)
	Content          []byte        `xml:",innerxml"`       // Binary content
	Compressed       w.BoolWrapper `xml:"Compressed,attr"` // Compressed flag (Only KDBX v3.1)
	isKDBX4          bool          `xml:"-"`
}

// BinaryReference stores a reference to a binary which appears in the xml of an entry
type BinaryReference struct {
	Name  string `xml:"Key"`
	Value struct {
		ID int `xml:"Ref,attr"`
	} `xml:"Value"`
}

// Find returns a reference to a binary with the same ID as id, or nil if none if found
func (bs Binaries) Find(id int) *Binary {
	for i := range bs {
		if bs[i].ID == id {
			return &bs[i]
		}
	}
	return nil
}

// Deprecated: Find returns a reference to a binary in the database db with the same id as br, or nil if none is found
// Note: this function should not be used directly, use `Database.FindBinary(id int) *Binary` instead
func (br *BinaryReference) Find(db *Database) *Binary {
	if db.Header.IsKdbx4() {
		return db.Content.InnerHeader.Binaries.Find(br.Value.ID)
	}
	return db.Content.Meta.Binaries.Find(br.Value.ID)
}

// BinaryOption is the option function type for use with Binary structs
type BinaryOption func(binary *Binary)

// WithKDBXv4Binary can be passed to the Binaries.Add function as an option to ensure
// that the Binary will follow the KDBXv4 format
func WithKDBXv4Binary(binary *Binary) {
	binary.Compressed = w.NewBoolWrapper(false)
	binary.isKDBX4 = true
}

// WithKDBXv31Binary can be passed to the Binaries.Add function as an option to ensure
// that the Binary will follow the KDBXv31 format
func WithKDBXv31Binary(binary *Binary) {
	binary.Compressed = w.NewBoolWrapper(true)
	binary.isKDBX4 = false
}

// Deprecated: Add appends binary data to the slice
// Note: this function should not be used directly, use `Database.AddBinary(c []byte) *Binary` instead
func (bs *Binaries) Add(c []byte, options ...BinaryOption) *Binary {
	for _, binary := range *bs {
		if bytes.Equal(binary.Content, c) {
			return &binary
		}
	}

	binary := Binary{
		Compressed: w.NewBoolWrapper(true),
	}

	for _, option := range options {
		option(&binary)
	}

	if len(*bs) == 0 {
		binary.ID = 0
	} else {
		binary.ID = (*bs)[len(*bs)-1].ID + 1
	}
	binary.SetContent(c)
	*bs = append(*bs, binary)
	return &(*bs)[len(*bs)-1]
}

// GetContentBytes returns a bytes slice containing content of a binary
func (b Binary) GetContentBytes() ([]byte, error) {
	// Check for base64 content (KDBX 3.1), if it fail try with KDBX 4
	decoded := make([]byte, base64.StdEncoding.DecodedLen(len(b.Content)))
	_, err := base64.StdEncoding.Decode(decoded, b.Content)
	if err != nil {
		// KDBX 4 doesn't encode it
		decoded = b.Content[:]
	}

	if b.Compressed.Bool {
		reader, err := gzip.NewReader(bytes.NewReader(decoded))
		if err != nil {
			return nil, err
		}
		defer reader.Close()
		bts, err := ioutil.ReadAll(reader)
		if err != nil && err != io.ErrUnexpectedEOF {
			return nil, err
		}

		return bts, nil
	}
	return decoded, nil
}

// GetContentString returns the content of a binary as a string
func (b Binary) GetContentString() (string, error) {
	data, err := b.GetContentBytes()

	if err != nil {
		return "", err
	}

	return string(data), nil
}

// GetContent returns a string which is the plaintext content of a binary
//
// Deprecated: use GetContentString() instead
func (b Binary) GetContent() (string, error) {
	return b.GetContentString()
}

type writeCloser struct {
	io.Writer
}

func (wc writeCloser) Close() error {
	return nil
}

// SetContent encodes and (if Compressed=true) compresses c and sets b's content
func (b *Binary) SetContent(c []byte) error {
	buff := &bytes.Buffer{}

	var writer io.WriteCloser

	if b.isKDBX4 {
		writer = writeCloser{Writer: buff}
	} else {
		writer = base64.NewEncoder(base64.StdEncoding, buff)
	}

	if b.Compressed.Bool {
		writer = gzip.NewWriter(writer)
	}
	_, err := writer.Write(c)
	if err != nil {
		return err
	}
	if err := writer.Close(); err != nil {
		return err
	}
	b.Content = buff.Bytes()

	return nil
}

// CreateReference creates a reference with the same id as b with filename f
func (b Binary) CreateReference(f string) BinaryReference {
	return NewBinaryReference(f, b.ID)
}

// NewBinaryReference creates a new BinaryReference with the given name and id
func NewBinaryReference(name string, id int) BinaryReference {
	ref := BinaryReference{}
	ref.Name = name
	ref.Value.ID = id
	return ref
}

func (b Binary) String() string {
	return fmt.Sprintf(
		"ID: %d, MemoryProtection: %x, Compressed:%#v, Content:%x",
		b.ID,
		b.MemoryProtection,
		b.Compressed,
		b.Content,
	)
}
func (br BinaryReference) String() string {
	return fmt.Sprintf("ID: %d, File Name: %s", br.Value.ID, br.Name)
}
<commit_msg>Use getBinaries() in `BinaryReference.Find`<commit_after>package gokeepasslib

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"

	w "github.com/tobischo/gokeepasslib/v3/wrappers"
)

// Binaries Stores a slice of binaries in the metadata header of a database
// This will be used only on KDBX 3.1
// Since KDBX 4, binaries are stored into the InnerHeader
type Binaries []Binary

// Binary stores a binary found in the metadata header of a database
type Binary struct {
	ID               int           `xml:"ID,attr"`         // Index of binary (Manually counted on KDBX v4)
	MemoryProtection byte          `xml:"-"`               // Memory protection flag (Only KDBX v4)
	Content          []byte        `xml:",innerxml"`       // Binary content
	Compressed       w.BoolWrapper `xml:"Compressed,attr"` // Compressed flag (Only KDBX v3.1)
	isKDBX4          bool          `xml:"-"`
}

// BinaryReference stores a reference to a binary which appears in the xml of an entry
type BinaryReference struct {
	Name  string `xml:"Key"`
	Value struct {
		ID int `xml:"Ref,attr"`
	} `xml:"Value"`
}

// Find returns a reference to a binary with the same ID as id, or nil if none if found
func (bs Binaries) Find(id int) *Binary {
	for i := range bs {
		if bs[i].ID == id {
			return &bs[i]
		}
	}
	return nil
}

// Deprecated: Find returns a reference to a binary in the database db with the same id as br, or nil if none is found
// Note: this function should not be used directly, use `Database.FindBinary(id int) *Binary` instead
func (br *BinaryReference) Find(db *Database) *Binary {
	return db.getBinaries().Find(br.Value.ID)
}

// BinaryOption is the option function type for use with Binary structs
type BinaryOption func(binary *Binary)

// WithKDBXv4Binary can be passed to the Binaries.Add function as an option to ensure
// that the Binary will follow the KDBXv4 format
func WithKDBXv4Binary(binary *Binary) {
	binary.Compressed = w.NewBoolWrapper(false)
	binary.isKDBX4 = true
}

// WithKDBXv31Binary can be passed to the Binaries.Add function as an option to ensure
// that the Binary will follow the KDBXv31 format
func WithKDBXv31Binary(binary *Binary) {
	binary.Compressed = w.NewBoolWrapper(true)
	binary.isKDBX4 = false
}

// Deprecated: Add appends binary data to the slice
// Note: this function should not be used directly, use `Database.AddBinary(c []byte) *Binary` instead
func (bs *Binaries) Add(c []byte, options ...BinaryOption) *Binary {
	for _, binary := range *bs {
		if bytes.Equal(binary.Content, c) {
			return &binary
		}
	}

	binary := Binary{
		Compressed: w.NewBoolWrapper(true),
	}

	for _, option := range options {
		option(&binary)
	}

	if len(*bs) == 0 {
		binary.ID = 0
	} else {
		binary.ID = (*bs)[len(*bs)-1].ID + 1
	}
	binary.SetContent(c)
	*bs = append(*bs, binary)
	return &(*bs)[len(*bs)-1]
}

// GetContentBytes returns a bytes slice containing content of a binary
func (b Binary) GetContentBytes() ([]byte, error) {
	// Check for base64 content (KDBX 3.1), if it fail try with KDBX 4
	decoded := make([]byte, base64.StdEncoding.DecodedLen(len(b.Content)))
	_, err := base64.StdEncoding.Decode(decoded, b.Content)
	if err != nil {
		// KDBX 4 doesn't encode it
		decoded = b.Content[:]
	}

	if b.Compressed.Bool {
		reader, err := gzip.NewReader(bytes.NewReader(decoded))
		if err != nil {
			return nil, err
		}
		defer reader.Close()
		bts, err := ioutil.ReadAll(reader)
		if err != nil && err != io.ErrUnexpectedEOF {
			return nil, err
		}

		return bts, nil
	}
	return decoded, nil
}

// GetContentString returns the content of a binary as a string
func (b Binary) GetContentString() (string, error) {
	data, err := b.GetContentBytes()

	if err != nil {
		return "", err
	}

	return string(data), nil
}

// GetContent returns a string which is the plaintext content of a binary
//
// Deprecated: use GetContentString() instead
func (b Binary) GetContent() (string, error) {
	return b.GetContentString()
}

type writeCloser struct {
	io.Writer
}

func (wc writeCloser) Close() error {
	return nil
}

// SetContent encodes and (if Compressed=true) compresses c and sets b's content
func (b *Binary) SetContent(c []byte) error {
	buff := &bytes.Buffer{}

	var writer io.WriteCloser

	if b.isKDBX4 {
		writer = writeCloser{Writer: buff}
	} else {
		writer = base64.NewEncoder(base64.StdEncoding, buff)
	}

	if b.Compressed.Bool {
		writer = gzip.NewWriter(writer)
	}
	_, err := writer.Write(c)
	if err != nil {
		return err
	}
	if err := writer.Close(); err != nil {
		return err
	}
	b.Content = buff.Bytes()

	return nil
}

// CreateReference creates a reference with the same id as b with filename f
func (b Binary) CreateReference(f string) BinaryReference {
	return NewBinaryReference(f, b.ID)
}

// NewBinaryReference creates a new BinaryReference with the given name and id
func NewBinaryReference(name string, id int) BinaryReference {
	ref := BinaryReference{}
	ref.Name = name
	ref.Value.ID = id
	return ref
}

func (b Binary) String() string {
	return fmt.Sprintf(
		"ID: %d, MemoryProtection: %x, Compressed:%#v, Content:%x",
		b.ID,
		b.MemoryProtection,
		b.Compressed,
		b.Content,
	)
}
func (br BinaryReference) String() string {
	return fmt.Sprintf("ID: %d, File Name: %s", br.Value.ID, br.Name)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"time"

	"github.com/dfordsoft/golib/ebook"
	"github.com/dfordsoft/golib/httputil"
)

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
	isAbsoluteURL   bool
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

func init() {
	dl := func(u string, tocPatterns []tocPattern, pageContentMarkers []pageContentMarker) {
		dlPage := func(u string) (c []byte) {
			var err error
			theURL, _ := url.Parse(u)
			headers := map[string]string{
				"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
				"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
				"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
				"Accept-Language":           `en-US,en;q=0.8`,
				"Upgrade-Insecure-Requests": "1",
			}
			c, err = httputil.GetBytes(u, headers, 60*time.Second, 3)
			if err != nil {
				return
			}
			c = bytes.Replace(c, []byte("\r\n"), []byte(""), -1)
			c = bytes.Replace(c, []byte("\r"), []byte(""), -1)
			c = bytes.Replace(c, []byte("\n"), []byte(""), -1)
			for _, m := range pageContentMarkers {
				if theURL.Host == m.host {
					idx := bytes.Index(c, m.start)
					if idx > 1 {
						fmt.Println("found start")
						c = c[idx+len(m.start):]
					}
					idx = bytes.Index(c, m.end)
					if idx > 1 {
						fmt.Println("found end")
						c = c[:idx]
					}
					break
				}
			}

			c = bytes.Replace(c, []byte("

    "), []byte("

"), -1)
			c = bytes.Replace(c, []byte("

"), []byte("

"), -1)
			c = bytes.Replace(c, []byte(`　　`), []byte(""), -1)
			return
		}

		theURL, _ := url.Parse(u)
		headers := map[string]string{
			"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
			"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
			"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
			"Accept-Language":           `en-US,en;q=0.8`,
			"Upgrade-Insecure-Requests": "1",
		}
		b, err := httputil.GetBytes(u, headers, 60*time.Second, 3)
		if err != nil {
			return
		}

		b = bytes.Replace(b, []byte("
"), []byte("
\n"), -1)

		mobi := &ebook.Mobi{}
		mobi.Begin()

		var title string
		var lines []string

		var p tocPattern
		for _, patt := range tocPatterns {
			if theURL.Host == patt.host {
				p = patt
				break
			}
		}
		r, _ := regexp.Compile(p.item)
		re, _ := regexp.Compile(p.bookTitle)
		scanner := bufio.NewScanner(bytes.NewReader(b))
		scanner.Split(bufio.ScanLines)
		for scanner.Scan() {
			line := scanner.Text()
			if title == "" {
				ss := re.FindAllStringSubmatch(line, -1)
				if len(ss) > 0 && len(ss[0]) > 0 {
					s := ss[0]
					title = s[p.bookTitlePos]
					mobi.SetTitle(title)
					continue
				}
			}
			if r.MatchString(line) {
				lines = append(lines, line)
			}
		}
		for i := len(lines) - 1; i >= 0 && i < len(lines) && lines[0] == lines[i]; i -= 2 {
			lines = lines[1:]
		}

		for _, line := range lines {
			ss := r.FindAllStringSubmatch(line, -1)
			s := ss[0]
			finalURL := fmt.Sprintf("%s://%s%s", theURL.Scheme, theURL.Host, s[p.articleURLPos])
			if p.isAbsoluteURL {
				finalURL = s[p.articleURLPos]
			}
			c := dlPage(finalURL)
			mobi.AppendContent(s[p.articleTitlePos], finalURL, string(c))
			fmt.Println(s[p.articleTitlePos], finalURL, len(c), "bytes")
		}
		mobi.End()
	}

	registerNovelSiteHandler(&novelSiteHandler{
		Title: `笔趣阁系列`,
		MatchPatterns: []string{
			`http://www\.biqudu\.com/[0-9]+_[0-9]+/`,
			`http://www\.biquge\.cm/[0-9]+/[0-9]+/`,
			`http://www\.qu\.la/book/[0-9]+/`,
			`http://www\.biqugezw\.com/[0-9]+_[0-9]+/`,
			`http://www\.630zw\.com/[0-9]+_[0-9]+/`,
			`http://www\.biquge\.lu/book/[0-9]+/`,
			`http://www\.biquge5200\.com/[0-9]+_[0-9]+/`,
		},
		Download: func(u string) {
			tocPatterns := []tocPattern{
				{
					host:            "www.biqudu.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.biquge.cm",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.qu.la",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*(style=""\s*)?href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   2,
					articleTitlePos: 3,
				},
				{
					host:            "www.biqugezw.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.630zw.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.biquge.lu",
					bookTitle:       `<h2>([^<]+)</h2>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.biquge5200.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
					isAbsoluteURL:   true,
				},
			}
			pageContentMarkers := []pageContentMarker{
				{
					host:  "www.biqudu.com",
					start: []byte(`<div id="content"><script>readx();</script>`),
					end:   []byte(`<script>chaptererror();</script>`),
				},
				{
					host:  "www.biquge.cm",
					start: []byte(`<div id="content">    `),
					end:   []byte(`找本站搜索"笔趣阁CM" 或输入网址:www.biquge.cm</div>`),
				},
				{
					host:  "www.qu.la",
					start: []byte(`<div id="content">`),
					end:   []byte(`<script>chaptererror();</script>`),
				},
				{
					host:  "www.biqugezw.com",
					start: []byte(`<div id="content">    一秒记住【笔趣阁中文网<a href="http://www.biqugezw.com" target="_blank">www.biqugezw.com</a>】，为您提供精彩小说阅读。`),
					end:   []byte(`手机用户请浏览m.biqugezw.com阅读，更优质的阅读体验。</div>`),
				},
				{
					host:  "www.630zw.com",
					start: []byte(`<div id="content">    `),
					end:   []byte(`(新笔趣阁：biqugee.cc，手机笔趣阁 m.biqugee.cc )</div>`),
				},
				{
					host:  "www.biquge.lu",
					start: []byte(`<div id="content" class="showtxt">        `),
					end:   []byte(`请记住本书首发域名：www.biquge.lu。笔趣阁手机版阅读网址：m.biquge.lu</div>`),
				},
				{
					host:  "www.biquge5200.com",
					start: []byte(`<div id="content">`),
					end:   []byte(`</div>`),
				},
			}
			dl(u, tocPatterns, pageContentMarkers)
		},
	})
}
<commit_msg>(+)support ranwena<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"time"

	"github.com/dfordsoft/golib/ebook"
	"github.com/dfordsoft/golib/httputil"
	"github.com/dfordsoft/golib/ic"
)

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
	isAbsoluteURL   bool
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

func init() {
	dl := func(u string, tocPatterns []tocPattern, pageContentMarkers []pageContentMarker) {
		dlPage := func(u string) (c []byte) {
			var err error
			theURL, _ := url.Parse(u)
			headers := map[string]string{
				"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
				"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
				"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
				"Accept-Language":           `en-US,en;q=0.8`,
				"Upgrade-Insecure-Requests": "1",
			}
			c, err = httputil.GetBytes(u, headers, 60*time.Second, 3)
			if err != nil {
				return
			}

			c = ic.Convert("gbk", "utf-8", c)
			c = bytes.Replace(c, []byte("\r\n"), []byte(""), -1)
			c = bytes.Replace(c, []byte("\r"), []byte(""), -1)
			c = bytes.Replace(c, []byte("\n"), []byte(""), -1)
			for _, m := range pageContentMarkers {
				if theURL.Host == m.host {
					idx := bytes.Index(c, m.start)
					if idx > 1 {
						//fmt.Println("found start")
						c = c[idx+len(m.start):]
					}
					idx = bytes.Index(c, m.end)
					if idx > 1 {
						//fmt.Println("found end")
						c = c[:idx]
					}
					break
				}
			}

			c = bytes.Replace(c, []byte("

    "), []byte("

"), -1)
			c = bytes.Replace(c, []byte("

"), []byte("

"), -1)
			c = bytes.Replace(c, []byte(`　　`), []byte(""), -1)
			return
		}

		theURL, _ := url.Parse(u)
		headers := map[string]string{
			"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
			"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
			"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
			"Accept-Language":           `en-US,en;q=0.8`,
			"Upgrade-Insecure-Requests": "1",
		}
		b, err := httputil.GetBytes(u, headers, 60*time.Second, 3)
		if err != nil {
			return
		}

		b = bytes.Replace(b, []byte("
"), []byte("
\n"), -1)
		b = ic.Convert("gbk", "utf-8", b)

		mobi := &ebook.Mobi{}
		mobi.Begin()

		var title string
		var lines []string

		var p tocPattern
		for _, patt := range tocPatterns {
			if theURL.Host == patt.host {
				p = patt
				break
			}
		}
		r, _ := regexp.Compile(p.item)
		re, _ := regexp.Compile(p.bookTitle)
		scanner := bufio.NewScanner(bytes.NewReader(b))
		scanner.Split(bufio.ScanLines)
		for scanner.Scan() {
			line := scanner.Text()
			if title == "" {
				ss := re.FindAllStringSubmatch(line, -1)
				if len(ss) > 0 && len(ss[0]) > 0 {
					s := ss[0]
					title = s[p.bookTitlePos]
					mobi.SetTitle(title)
					continue
				}
			}
			if r.MatchString(line) {
				lines = append(lines, line)
			}
		}
		for i := len(lines) - 1; i >= 0 && i < len(lines) && lines[0] == lines[i]; i -= 2 {
			lines = lines[1:]
		}

		for _, line := range lines {
			ss := r.FindAllStringSubmatch(line, -1)
			s := ss[0]
			finalURL := fmt.Sprintf("%s://%s%s", theURL.Scheme, theURL.Host, s[p.articleURLPos])
			if p.isAbsoluteURL {
				finalURL = s[p.articleURLPos]
			}
			c := dlPage(finalURL)
			mobi.AppendContent(s[p.articleTitlePos], finalURL, string(c))
			fmt.Println(s[p.articleTitlePos], finalURL, len(c), "bytes")
		}
		mobi.End()
	}

	registerNovelSiteHandler(&novelSiteHandler{
		Title:         `燃文小说`,
		MatchPatterns: []string{`http://www\.ranwena\.com/files/article/[0-9]+/[0-9]+/`},
		Download: func(u string) {
			tocPatterns := []tocPattern{
				{
					host:            "www.ranwena.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
					isAbsoluteURL:   true,
				},
			}
			pageContentMarkers := []pageContentMarker{
				{
					host:  "www.ranwena.com",
					start: []byte(`<div id="content">`),
					end:   []byte(`</div>`),
				},
			}
			dl(u, tocPatterns, pageContentMarkers)
		},
	})
	registerNovelSiteHandler(&novelSiteHandler{
		Title: `笔趣阁系列`,
		MatchPatterns: []string{
			`http://www\.biqudu\.com/[0-9]+_[0-9]+/`,
			`http://www\.biquge\.cm/[0-9]+/[0-9]+/`,
			`http://www\.qu\.la/book/[0-9]+/`,
			`http://www\.biqugezw\.com/[0-9]+_[0-9]+/`,
			`http://www\.630zw\.com/[0-9]+_[0-9]+/`,
			`http://www\.biquge\.lu/book/[0-9]+/`,
			`http://www\.biquge5200\.com/[0-9]+_[0-9]+/`,
		},
		Download: func(u string) {
			tocPatterns := []tocPattern{
				{
					host:            "www.biqudu.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.biquge.cm",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.qu.la",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*(style=""\s*)?href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   2,
					articleTitlePos: 3,
				},
				{
					host:            "www.biqugezw.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.630zw.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.biquge.lu",
					bookTitle:       `<h2>([^<]+)</h2>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
				},
				{
					host:            "www.biquge5200.com",
					bookTitle:       `<h1>([^<]+)</h1>$`,
					bookTitlePos:    1,
					item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
					articleURLPos:   1,
					articleTitlePos: 2,
					isAbsoluteURL:   true,
				},
			}
			pageContentMarkers := []pageContentMarker{
				{
					host:  "www.biqudu.com",
					start: []byte(`<div id="content"><script>readx();</script>`),
					end:   []byte(`<script>chaptererror();</script>`),
				},
				{
					host:  "www.biquge.cm",
					start: []byte(`<div id="content">    `),
					end:   []byte(`找本站搜索"笔趣阁CM" 或输入网址:www.biquge.cm</div>`),
				},
				{
					host:  "www.qu.la",
					start: []byte(`<div id="content">`),
					end:   []byte(`<script>chaptererror();</script>`),
				},
				{
					host:  "www.biqugezw.com",
					start: []byte(`<div id="content">    一秒记住【笔趣阁中文网<a href="http://www.biqugezw.com" target="_blank">www.biqugezw.com</a>】，为您提供精彩小说阅读。`),
					end:   []byte(`手机用户请浏览m.biqugezw.com阅读，更优质的阅读体验。</div>`),
				},
				{
					host:  "www.630zw.com",
					start: []byte(`<div id="content">    `),
					end:   []byte(`(新笔趣阁：biqugee.cc，手机笔趣阁 m.biqugee.cc )</div>`),
				},
				{
					host:  "www.biquge.lu",
					start: []byte(`<div id="content" class="showtxt">        `),
					end:   []byte(`请记住本书首发域名：www.biquge.lu。笔趣阁手机版阅读网址：m.biquge.lu</div>`),
				},
				{
					host:  "www.biquge5200.com",
					start: []byte(`<div id="content">`),
					end:   []byte(`</div>`),
				},
			}
			dl(u, tocPatterns, pageContentMarkers)
		},
	})
}
<|endoftext|>