<commit_before>// launchpad.net/juju/state
//
// Copyright (c) 2011-2012 Canonical Ltd.

// The state package enables reading, observing, and changing
// the state stored in ZooKeeper of a whole environment
// managed by juju.
package state

import (
	"fmt"
	"launchpad.net/goyaml"
	"launchpad.net/gozk/zookeeper"
	"launchpad.net/juju/go/charm"
	"net/url"
	"strings"
	"time"
)

// State represents the state of an environment
// managed by juju.
type State struct {
	zk *zookeeper.Conn
}

// AddMachine creates a new machine state.
func (s *State) AddMachine() (*Machine, error) {
	path, err := s.zk.Create("/machines/machine-", "", zookeeper.SEQUENCE, zkPermAll)
	if err != nil {
		return nil, err
	}
	key := strings.Split(path, "/")[2]
	addMachine := func(t *topology) error {
		return t.AddMachine(key)
	}
	if err = retryTopologyChange(s.zk, addMachine); err != nil {
		return nil, err
	}
	return &Machine{s, key}, nil
}

// RemoveMachine removes the machine with the given id.
func (s *State) RemoveMachine(id int) error {
	key := machineKey(id)
	removeMachine := func(t *topology) error {
		if !t.HasMachine(key) {
			return fmt.Errorf("machine not found")
		}
		hasUnits, err := t.MachineHasUnits(key)
		if err != nil {
			return err
		}
		if hasUnits {
			return fmt.Errorf("machine has units")
		}
		return t.RemoveMachine(key)
	}
	if err := retryTopologyChange(s.zk, removeMachine); err != nil {
		return fmt.Errorf("can't remove machine %d: %v", id, err)
	}
	return zkRemoveTree(s.zk, fmt.Sprintf("/machines/%s", key))
}

// Machine returns the machine with the given id.
func (s *State) Machine(id int) (*Machine, error) {
	key := machineKey(id)
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	if !topology.HasMachine(key) {
		return nil, fmt.Errorf("machine %d not found", id)
	}
	return &Machine{s, key}, nil
}

// AllMachines returns all machines in the environment.
func (s *State) AllMachines() ([]*Machine, error) {
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	machines := []*Machine{}
	for _, key := range topology.MachineKeys() {
		machines = append(machines, &Machine{s, key})
	}
	return machines, nil
}

// AddCharm adds the ch charm with curl to the state.
// bundleUrl must be set to a URL where the bundle for ch
// may be downloaded from.
// On success the newly added charm state is returned.
func (s *State) AddCharm(ch charm.Charm, curl *charm.URL, bundleURL *url.URL) (*Charm, error) {
	data := &charmData{
		Meta:      ch.Meta(),
		Config:    ch.Config(),
		BundleURL: bundleURL.String(),
	}
	yaml, err := goyaml.Marshal(data)
	if err != nil {
		return nil, err
	}
	path, err := charmPath(curl)
	if err != nil {
		return nil, err
	}
	_, err = s.zk.Create(path, string(yaml), 0, zkPermAll)
	if err != nil {
		return nil, err
	}
	return newCharm(s, curl, data)
}

// Charm returns a charm by the given id.
func (s *State) Charm(curl *charm.URL) (*Charm, error) {
	path, err := charmPath(curl)
	if err != nil {
		return nil, err
	}
	yaml, _, err := s.zk.Get(path)
	if zookeeper.IsError(err, zookeeper.ZNONODE) {
		return nil, fmt.Errorf("charm not found: %q", curl)
	}
	if err != nil {
		return nil, err
	}
	data := &charmData{}
	if err := goyaml.Unmarshal([]byte(yaml), data); err != nil {
		return nil, err
	}
	return newCharm(s, curl, data)
}

// AddService creates a new service state with the given unique name
// and the charm state.
func (s *State) AddService(name string, ch *Charm) (*Service, error) {
	details := map[string]interface{}{"charm": ch.URL().String()}
	yaml, err := goyaml.Marshal(details)
	if err != nil {
		return nil, err
	}
	path, err := s.zk.Create("/services/service-", string(yaml), zookeeper.SEQUENCE, zkPermAll)
	if err != nil {
		return nil, err
	}
	key := strings.Split(path, "/")[2]
	service := &Service{s, key, name}
	// Create an empty configuration node.
	_, err = createConfigNode(s.zk, service.zkConfigPath(), map[string]interface{}{})
	if err != nil {
		return nil, err
	}
	addService := func(t *topology) error {
		if _, err := t.ServiceKey(name); err == nil {
			// No error, so service name already in use.
			return fmt.Errorf("service name %q is already in use", name)
		}
		return t.AddService(key, name)
	}
	if err = retryTopologyChange(s.zk, addService); err != nil {
		return nil, err
	}
	return service, nil
}

// RemoveService removes a service from the state. It will
// also remove all its units and break any of its existing
// relations.
func (s *State) RemoveService(svc *Service) error {
	// TODO Remove relations first, to prevent spurious hook execution.

	// Remove the units.
	units, err := svc.AllUnits()
	if err != nil {
		return err
	}
	for _, unit := range units {
		if err = svc.RemoveUnit(unit); err != nil {
			return err
		}
	}
	// Remove the service from the topology.
	removeService := func(t *topology) error {
		if !t.HasService(svc.key) {
			return stateChanged
		}
		t.RemoveService(svc.key)
		return nil
	}
	if err = retryTopologyChange(s.zk, removeService); err != nil {
		return err
	}
	return zkRemoveTree(s.zk, svc.zkPath())
}

// Service returns a service state by name.
func (s *State) Service(name string) (*Service, error) {
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	key, err := topology.ServiceKey(name)
	if err != nil {
		return nil, err
	}
	return &Service{s, key, name}, nil
}

// AllServices returns all deployed services in the environment.
func (s *State) AllServices() ([]*Service, error) {
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	services := []*Service{}
	for _, key := range topology.ServiceKeys() {
		name, err := topology.ServiceName(key)
		if err != nil {
			return nil, err
		}
		services = append(services, &Service{s, key, name})
	}
	return services, nil
}

// Unit returns a unit by name.
func (s *State) Unit(name string) (*Unit, error) {
	serviceName, _, err := parseUnitName(name)
	if err != nil {
		return nil, err
	}
	service, err := s.Service(serviceName)
	if err != nil {
		return nil, err
	}
	return service.Unit(name)
}

func (s *State) waitForInitialization() error {
	stat, watch, err := s.zk.ExistsW("/initialized")
	if err != nil {
		return err
	}
	if stat != nil {
		return nil
	}
	select {
	case e := <-watch:
		if !e.Ok() {
			return fmt.Errorf("session error: %v", e)
		}
	case <-time.After(3 * time.Minute):
		return fmt.Errorf("timed out waiting for initialization")
	}
	return nil
}

func (s *State) initialize() error {
	stat, err := s.zk.Exists("/initialized")
	if err != nil {
		return err
	}
	if stat != nil {
		return nil
	}
	// Create new nodes.
	if _, err := s.zk.Create("/charms", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/services", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/machines", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/units", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/relations", "", 0, zkPermAll); err != nil {
		return err
	}
	// TODO Create node for bootstrap machine.

	// TODO Setup default global settings information.

	// Finally creation of /initialized as marker.
	if _, err := s.zk.Create("/initialized", "", 0, zkPermAll); err != nil {
		return err
	}
	return nil
}
<commit_msg>state: remove timeout on waitForInitialization<commit_after>// launchpad.net/juju/state
//
// Copyright (c) 2011-2012 Canonical Ltd.

// The state package enables reading, observing, and changing
// the state stored in ZooKeeper of a whole environment
// managed by juju.
package state

import (
	"fmt"
	"launchpad.net/goyaml"
	"launchpad.net/gozk/zookeeper"
	"launchpad.net/juju/go/charm"
	"net/url"
	"strings"
)

// State represents the state of an environment
// managed by juju.
type State struct {
	zk *zookeeper.Conn
}

// AddMachine creates a new machine state.
func (s *State) AddMachine() (*Machine, error) {
	path, err := s.zk.Create("/machines/machine-", "", zookeeper.SEQUENCE, zkPermAll)
	if err != nil {
		return nil, err
	}
	key := strings.Split(path, "/")[2]
	addMachine := func(t *topology) error {
		return t.AddMachine(key)
	}
	if err = retryTopologyChange(s.zk, addMachine); err != nil {
		return nil, err
	}
	return &Machine{s, key}, nil
}

// RemoveMachine removes the machine with the given id.
func (s *State) RemoveMachine(id int) error {
	key := machineKey(id)
	removeMachine := func(t *topology) error {
		if !t.HasMachine(key) {
			return fmt.Errorf("machine not found")
		}
		hasUnits, err := t.MachineHasUnits(key)
		if err != nil {
			return err
		}
		if hasUnits {
			return fmt.Errorf("machine has units")
		}
		return t.RemoveMachine(key)
	}
	if err := retryTopologyChange(s.zk, removeMachine); err != nil {
		return fmt.Errorf("can't remove machine %d: %v", id, err)
	}
	return zkRemoveTree(s.zk, fmt.Sprintf("/machines/%s", key))
}

// Machine returns the machine with the given id.
func (s *State) Machine(id int) (*Machine, error) {
	key := machineKey(id)
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	if !topology.HasMachine(key) {
		return nil, fmt.Errorf("machine %d not found", id)
	}
	return &Machine{s, key}, nil
}

// AllMachines returns all machines in the environment.
func (s *State) AllMachines() ([]*Machine, error) {
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	machines := []*Machine{}
	for _, key := range topology.MachineKeys() {
		machines = append(machines, &Machine{s, key})
	}
	return machines, nil
}

// AddCharm adds the ch charm with curl to the state.
// bundleUrl must be set to a URL where the bundle for ch
// may be downloaded from.
// On success the newly added charm state is returned.
func (s *State) AddCharm(ch charm.Charm, curl *charm.URL, bundleURL *url.URL) (*Charm, error) {
	data := &charmData{
		Meta:      ch.Meta(),
		Config:    ch.Config(),
		BundleURL: bundleURL.String(),
	}
	yaml, err := goyaml.Marshal(data)
	if err != nil {
		return nil, err
	}
	path, err := charmPath(curl)
	if err != nil {
		return nil, err
	}
	_, err = s.zk.Create(path, string(yaml), 0, zkPermAll)
	if err != nil {
		return nil, err
	}
	return newCharm(s, curl, data)
}

// Charm returns a charm by the given id.
func (s *State) Charm(curl *charm.URL) (*Charm, error) {
	path, err := charmPath(curl)
	if err != nil {
		return nil, err
	}
	yaml, _, err := s.zk.Get(path)
	if zookeeper.IsError(err, zookeeper.ZNONODE) {
		return nil, fmt.Errorf("charm not found: %q", curl)
	}
	if err != nil {
		return nil, err
	}
	data := &charmData{}
	if err := goyaml.Unmarshal([]byte(yaml), data); err != nil {
		return nil, err
	}
	return newCharm(s, curl, data)
}

// AddService creates a new service state with the given unique name
// and the charm state.
func (s *State) AddService(name string, ch *Charm) (*Service, error) {
	details := map[string]interface{}{"charm": ch.URL().String()}
	yaml, err := goyaml.Marshal(details)
	if err != nil {
		return nil, err
	}
	path, err := s.zk.Create("/services/service-", string(yaml), zookeeper.SEQUENCE, zkPermAll)
	if err != nil {
		return nil, err
	}
	key := strings.Split(path, "/")[2]
	service := &Service{s, key, name}
	// Create an empty configuration node.
	_, err = createConfigNode(s.zk, service.zkConfigPath(), map[string]interface{}{})
	if err != nil {
		return nil, err
	}
	addService := func(t *topology) error {
		if _, err := t.ServiceKey(name); err == nil {
			// No error, so service name already in use.
			return fmt.Errorf("service name %q is already in use", name)
		}
		return t.AddService(key, name)
	}
	if err = retryTopologyChange(s.zk, addService); err != nil {
		return nil, err
	}
	return service, nil
}

// RemoveService removes a service from the state. It will
// also remove all its units and break any of its existing
// relations.
func (s *State) RemoveService(svc *Service) error {
	// TODO Remove relations first, to prevent spurious hook execution.

	// Remove the units.
	units, err := svc.AllUnits()
	if err != nil {
		return err
	}
	for _, unit := range units {
		if err = svc.RemoveUnit(unit); err != nil {
			return err
		}
	}
	// Remove the service from the topology.
	removeService := func(t *topology) error {
		if !t.HasService(svc.key) {
			return stateChanged
		}
		t.RemoveService(svc.key)
		return nil
	}
	if err = retryTopologyChange(s.zk, removeService); err != nil {
		return err
	}
	return zkRemoveTree(s.zk, svc.zkPath())
}

// Service returns a service state by name.
func (s *State) Service(name string) (*Service, error) {
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	key, err := topology.ServiceKey(name)
	if err != nil {
		return nil, err
	}
	return &Service{s, key, name}, nil
}

// AllServices returns all deployed services in the environment.
func (s *State) AllServices() ([]*Service, error) {
	topology, err := readTopology(s.zk)
	if err != nil {
		return nil, err
	}
	services := []*Service{}
	for _, key := range topology.ServiceKeys() {
		name, err := topology.ServiceName(key)
		if err != nil {
			return nil, err
		}
		services = append(services, &Service{s, key, name})
	}
	return services, nil
}

// Unit returns a unit by name.
func (s *State) Unit(name string) (*Unit, error) {
	serviceName, _, err := parseUnitName(name)
	if err != nil {
		return nil, err
	}
	service, err := s.Service(serviceName)
	if err != nil {
		return nil, err
	}
	return service.Unit(name)
}

func (s *State) waitForInitialization() error {
	stat, watch, err := s.zk.ExistsW("/initialized")
	if err != nil {
		return err
	}
	if stat != nil {
		return nil
	}
	e := <-watch
	if !e.Ok() {
		return fmt.Errorf("session error: %v", e)
	}
	return nil
}

func (s *State) initialize() error {
	stat, err := s.zk.Exists("/initialized")
	if err != nil {
		return err
	}
	if stat != nil {
		return nil
	}
	// Create new nodes.
	if _, err := s.zk.Create("/charms", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/services", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/machines", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/units", "", 0, zkPermAll); err != nil {
		return err
	}
	if _, err := s.zk.Create("/relations", "", 0, zkPermAll); err != nil {
		return err
	}
	// TODO Create node for bootstrap machine.

	// TODO Setup default global settings information.

	// Finally creation of /initialized as marker.
	if _, err := s.zk.Create("/initialized", "", 0, zkPermAll); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package imdb

import (
	"strings"

	"github.com/BurntSushi/ty/fun"

	"github.com/BurntSushi/csql"

	"github.com/BurntSushi/migration"
)

var migrations = map[string][]migration.Migrator{
	"sqlite3": {
		func(tx migration.LimitedTx) error {
			var err error
			_, err = tx.Exec(`
				CREATE TABLE atom (
					id integer,
					hash BLOB NOT NULL,
					PRIMARY KEY (id)
				);
				CREATE TABLE name (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL
				);
				CREATE TABLE movie (
					atom_id INTEGER NOT NULL,
					year INTEGER NOT NULL,
					sequence TEXT,
					tv INTEGER NOT NULL,
					video INTEGER NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE tvshow (
					atom_id INTEGER NOT NULL,
					year INTEGER NOT NULL,
					sequence TEXT,
					year_start INTEGER,
					year_end INTEGER,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE episode (
					atom_id INTEGER NOT NULL,
					tvshow_atom_id INTEGER NOT NULL,
					year INTEGER NOT NULL,
					season INTEGER NOT NULL,
					episode_num INTEGER NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE release_date (
					atom_id INTEGER,
					country TEXT,
					released DATE,
					attrs TEXT
				);
				CREATE TABLE running_time (
					atom_id INTEGER,
					country TEXT,
					minutes INTEGER,
					attrs TEXT
				);
				CREATE TABLE aka_title (
					atom_id INTEGER,
					title TEXT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE alternate_version (
					atom_id INTEGER,
					about TEXT
				);
				CREATE TABLE color_info (
					atom_id INTEGER,
					color INTEGER NOT NULL,
					attrs TEXT
				);
				CREATE TABLE mpaa_rating (
					atom_id INTEGER,
					rating TEXT
						CHECK (rating = "G"
						       OR rating = "PG"
							   OR rating = "PG-13"
							   OR rating = "R"
							   OR rating = "NC-17"
							  ),
					reason TEXT
				);
				CREATE TABLE sound_mix (
					atom_id INTEGER,
					mix TEXT,
					attrs TEXT
				);
				`)
			return err
		},
	},
	"postgres": {
		func(tx migration.LimitedTx) error {
			var err error
			_, err = tx.Exec(`
				CREATE TYPE mpaa AS ENUM ('G', 'PG', 'PG-13', 'R', 'NC-17');

				CREATE TABLE atom (
					id INTEGER,
					hash BYTEA NOT NULL,
					PRIMARY KEY (id)
				);
				CREATE TABLE name (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL
				);
				CREATE TABLE actor (
					atom_id INTEGER,
					sequence TEXT
				);
				CREATE TABLE credit (
					actor_atom_id INTEGER NOT NULL,
					media_atom_id INTEGER NOT NULL,
					character TEXT,
					position INTEGER,
					attrs TEXT
				);
				CREATE TABLE movie (
					atom_id INTEGER,
					year SMALLINT NOT NULL,
					sequence TEXT,
					tv BOOLEAN NOT NULL,
					video BOOLEAN NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE tvshow (
					atom_id INTEGER,
					year SMALLINT NOT NULL,
					sequence TEXT,
					year_start SMALLINT,
					year_end SMALLINT,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE episode (
					atom_id INTEGER,
					tvshow_atom_id INTEGER NOT NULL,
					year SMALLINT NOT NULL,
					season SMALLINT NOT NULL,
					episode_num INTEGER NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE release_date (
					atom_id INTEGER NOT NULL,
					country TEXT,
					released DATE NOT NULL,
					attrs TEXT
				);
				CREATE TABLE running_time (
					atom_id INTEGER NOT NULL,
					country TEXT,
					minutes SMALLINT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE aka_title (
					atom_id INTEGER NOT NULL,
					title TEXT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE alternate_version (
					atom_id INTEGER NOT NULL,
					about TEXT NOT NULL
				);
				CREATE TABLE color_info (
					atom_id INTEGER NOT NULL,
					color BOOLEAN NOT NULL,
					attrs TEXT
				);
				CREATE TABLE mpaa_rating (
					atom_id INTEGER NOT NULL,
					rating mpaa NOT NULL,
					reason TEXT NOT NULL
				);
				CREATE TABLE sound_mix (
					atom_id INTEGER NOT NULL,
					mix TEXT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE tagline (
					atom_id INTEGER NOT NULL,
					tag TEXT NOT NULL
				);
				CREATE TABLE trivia (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE genre (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL
				);
				CREATE TABLE goof (
					atom_id INTEGER NOT NULL,
					goof_type TEXT NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE language (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE literature (
					atom_id INTEGER NOT NULL,
					lit_type TEXT NOT NULL,
					ref TEXT NOT NULL
				);
				CREATE TABLE location (
					atom_id INTEGER NOT NULL,
					place TEXT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE link (
					atom_id INTEGER NOT NULL,
					link_type TEXT NOT NULL,
					link_atom_id INTEGER NOT NULL,
					entity TEXT NOT NULL
						CHECK (entity = 'movie'
						       OR entity = 'tvshow'
							   OR entity = 'episode')
				);
				CREATE TABLE plot (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL,
					by TEXT NOT NULL
				);
				CREATE TABLE quote (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE rating (
					atom_id INTEGER NOT NULL,
					votes INTEGER NOT NULL,
					rank INTEGER NOT NULL
				);
				`)
			return err
		},
	},
}

type index struct {
	unique   bool
	table    string
	name     string
	fulltext string // empty, "gin" or "gist"
	columns  []string
}

var indices = []index{
	{true, "atom", "", "", []string{"hash"}},
	{false, "episode", "tv", "", []string{"tvshow_atom_id"}},
	{false, "episode", "tvseason", "", []string{"tvshow_atom_id", "season"}},

	{false, "release_date", "", "", []string{"atom_id"}},
	{false, "running_time", "", "", []string{"atom_id"}},
	{false, "aka_title", "", "", []string{"atom_id"}},
	{false, "alternate_version", "", "", []string{"atom_id"}},
	{false, "color_info", "", "", []string{"atom_id"}},
	{false, "mpaa_rating", "", "", []string{"atom_id"}},
	{false, "sound_mix", "", "", []string{"atom_id"}},
	{false, "genre", "", "", []string{"atom_id"}},
	{false, "tagline", "", "", []string{"atom_id"}},
	{false, "trivia", "", "", []string{"atom_id"}},
	{false, "goof", "", "", []string{"atom_id"}},
	{false, "language", "", "", []string{"atom_id"}},
	{false, "literature", "", "", []string{"atom_id"}},
	{false, "location", "", "", []string{"atom_id"}},
	{false, "link", "", "", []string{"atom_id"}},
	{false, "plot", "", "", []string{"atom_id"}},
	{false, "quote", "", "", []string{"atom_id"}},
	{false, "rating", "", "", []string{"atom_id"}},
	{false, "name", "", "", []string{"atom_id"}},
	{false, "actor", "", "", []string{"atom_id"}},
	{false, "credit", "", "", []string{"actor_atom_id"}},
	{false, "credit", "", "", []string{"media_atom_id"}},

	{false, "name", "trgm_name", "gist", []string{"name"}},
	{false, "aka_title", "trgm_title", "gist", []string{"title"}},
}

func (in index) sqlName() string {
	name := in.name
	if len(in.columns) == 0 {
		panic("indices must have at least one column")
	}
	if len(name) == 0 {
		if len(in.columns) > 1 {
			panic("must specify index name for multi-column indices")
		}
		name = in.columns[0]
	}
	return sf("idx_%s_%s", in.table, name)
}

func (in index) sqlCreate(db *DB) string {
	uni := ""
	if in.unique {
		uni = " UNIQUE "
	}
	using, class := "", ""
	if in.isFulltext() {
		using = sf(" USING %s ", in.fulltext)
		switch in.fulltext {
		case "gin":
			class = " gin_trgm_ops"
		case "gist":
			class = " gist_trgm_ops"
		default:
			panic(sf("unrecognized fulltext index type: %s", in.fulltext))
		}
	}
	return sf("CREATE %s INDEX %s ON %s %s (%s%s)",
		uni, in.sqlName(), in.table, using,
		strings.Join(in.columns, ", "), class)
}

func (in index) isFulltext() bool {
	return len(in.fulltext) > 0
}

func (in index) sqlDrop(db *DB) string {
	return sf("DROP INDEX IF EXISTS %s", in.sqlName())
}

func doIndices(
	db *DB,
	getSql func(index, *DB) string,
	tables ...string,
) (err error) {
	defer csql.Safe(&err)

	trgmEnabled := db.IsFuzzyEnabled()
	var q string
	var ok bool
	for _, idx := range indices {
		if idx.isFulltext() && !trgmEnabled {
			// Only show the error message if we're on PostgreSQL.
			if db.Driver == "postgres" {
				logf("Skipping fulltext index '%s' since "+
					"the pg_trgm extension is not enabled.", idx.sqlName())
			}
			continue
		}
		if len(tables) == 0 || fun.In(idx.table, tables) {
			q += getSql(idx, db) + "; "
			ok = true
		}
	}
	if ok {
		csql.Exec(db, q)
	}
	return
}

func (db *DB) CreateIndices(tables ...string) error {
	return doIndices(db, index.sqlCreate, tables...)
}

func (db *DB) DropIndices(tables ...string) error {
	return doIndices(db, index.sqlDrop, tables...)
}
<commit_msg>Update SQLite schema.<commit_after>package imdb

import (
	"strings"

	"github.com/BurntSushi/ty/fun"

	"github.com/BurntSushi/csql"

	"github.com/BurntSushi/migration"
)

var migrations = map[string][]migration.Migrator{
	"sqlite3": {
		func(tx migration.LimitedTx) error {
			var err error
			_, err = tx.Exec(`
				CREATE TABLE atom (
					id INTEGER NOT NULL,
					hash BLOB NOT NULL,
					PRIMARY KEY (id)
				);
				CREATE TABLE name (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL
				);
				CREATE TABLE actor (
					atom_id INTEGER NOT NULL,
					sequence TEXT NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE credit (
					actor_atom_id INTEGER NOT NULL,
					media_atom_id INTEGER NOT NULL,
					character TEXT NOT NULL,
					position INTEGER NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE movie (
					atom_id INTEGER NOT NULL,
					year SMALLINT NOT NULL,
					sequence TEXT NOT NULL,
					tv BOOLEAN NOT NULL,
					video BOOLEAN NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE tvshow (
					atom_id INTEGER NOT NULL,
					year SMALLINT NOT NULL,
					sequence TEXT NOT NULL,
					year_start SMALLINT NOT NULL,
					year_end SMALLINT NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE episode (
					atom_id INTEGER NOT NULL,
					tvshow_atom_id INTEGER NOT NULL,
					year SMALLINT NOT NULL,
					season SMALLINT NOT NULL,
					episode_num INTEGER NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE release_date (
					atom_id INTEGER NOT NULL,
					country TEXT NOT NULL,
					released DATE NOT NULL,
					attrs TEXT
				);
				CREATE TABLE running_time (
					atom_id INTEGER NOT NULL,
					country TEXT NOT NULL,
					minutes SMALLINT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE aka_title (
					atom_id INTEGER NOT NULL,
					title TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE alternate_version (
					atom_id INTEGER NOT NULL,
					about TEXT NOT NULL
				);
				CREATE TABLE color_info (
					atom_id INTEGER NOT NULL,
					color BOOLEAN NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE sound_mix (
					atom_id INTEGER NOT NULL,
					mix TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE tagline (
					atom_id INTEGER NOT NULL,
					tag TEXT NOT NULL
				);
				CREATE TABLE trivia (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE genre (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL
				);
				CREATE TABLE goof (
					atom_id INTEGER NOT NULL,
					goof_type TEXT NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE language (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE literature (
					atom_id INTEGER NOT NULL,
					lit_type TEXT NOT NULL,
					ref TEXT NOT NULL
				);
				CREATE TABLE location (
					atom_id INTEGER NOT NULL,
					place TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE link (
					atom_id INTEGER NOT NULL,
					link_type TEXT NOT NULL,
					link_atom_id INTEGER NOT NULL,
					entity TEXT NOT NULL
						CHECK (entity = 'movie'
						       OR entity = 'tvshow'
							   OR entity = 'episode')
				);
				CREATE TABLE plot (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL,
					by TEXT NOT NULL
				);
				CREATE TABLE quote (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE rating (
					atom_id INTEGER NOT NULL,
					votes INTEGER NOT NULL,
					rank INTEGER NOT NULL
				);
				CREATE TABLE mpaa_rating (
					atom_id INTEGER NOT NULL,
					rating TEXT NOT NULL
						CHECK (rating = "G"
						       OR rating = "PG"
							   OR rating = "PG-13"
							   OR rating = "R"
							   OR rating = "NC-17"
							  ),
					reason TEXT NOT NULL
				);
				`)
			return err
		},
	},
	"postgres": {
		func(tx migration.LimitedTx) error {
			var err error
			_, err = tx.Exec(`
				CREATE TYPE mpaa AS ENUM ('G', 'PG', 'PG-13', 'R', 'NC-17');

				CREATE TABLE atom (
					id INTEGER NOT NULL,
					hash BYTEA NOT NULL,
					PRIMARY KEY (id)
				);
				CREATE TABLE name (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL
				);
				CREATE TABLE actor (
					atom_id INTEGER NOT NULL,
					sequence TEXT NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE credit (
					actor_atom_id INTEGER NOT NULL,
					media_atom_id INTEGER NOT NULL,
					character TEXT NOT NULL,
					position INTEGER NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE movie (
					atom_id INTEGER NOT NULL,
					year SMALLINT NOT NULL,
					sequence TEXT NOT NULL,
					tv BOOLEAN NOT NULL,
					video BOOLEAN NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE tvshow (
					atom_id INTEGER NOT NULL,
					year SMALLINT NOT NULL,
					sequence TEXT NOT NULL,
					year_start SMALLINT NOT NULL,
					year_end SMALLINT NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE episode (
					atom_id INTEGER NOT NULL,
					tvshow_atom_id INTEGER NOT NULL,
					year SMALLINT NOT NULL,
					season SMALLINT NOT NULL,
					episode_num INTEGER NOT NULL,
					PRIMARY KEY (atom_id)
				);
				CREATE TABLE release_date (
					atom_id INTEGER NOT NULL,
					country TEXT NOT NULL,
					released DATE NOT NULL,
					attrs TEXT
				);
				CREATE TABLE running_time (
					atom_id INTEGER NOT NULL,
					country TEXT NOT NULL,
					minutes SMALLINT NOT NULL,
					attrs TEXT
				);
				CREATE TABLE aka_title (
					atom_id INTEGER NOT NULL,
					title TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE alternate_version (
					atom_id INTEGER NOT NULL,
					about TEXT NOT NULL
				);
				CREATE TABLE color_info (
					atom_id INTEGER NOT NULL,
					color BOOLEAN NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE sound_mix (
					atom_id INTEGER NOT NULL,
					mix TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE tagline (
					atom_id INTEGER NOT NULL,
					tag TEXT NOT NULL
				);
				CREATE TABLE trivia (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE genre (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL
				);
				CREATE TABLE goof (
					atom_id INTEGER NOT NULL,
					goof_type TEXT NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE language (
					atom_id INTEGER NOT NULL,
					name TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE literature (
					atom_id INTEGER NOT NULL,
					lit_type TEXT NOT NULL,
					ref TEXT NOT NULL
				);
				CREATE TABLE location (
					atom_id INTEGER NOT NULL,
					place TEXT NOT NULL,
					attrs TEXT NOT NULL
				);
				CREATE TABLE link (
					atom_id INTEGER NOT NULL,
					link_type TEXT NOT NULL,
					link_atom_id INTEGER NOT NULL,
					entity TEXT NOT NULL
						CHECK (entity = 'movie'
						       OR entity = 'tvshow'
							   OR entity = 'episode')
				);
				CREATE TABLE plot (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL,
					by TEXT NOT NULL
				);
				CREATE TABLE quote (
					atom_id INTEGER NOT NULL,
					entry TEXT NOT NULL
				);
				CREATE TABLE rating (
					atom_id INTEGER NOT NULL,
					votes INTEGER NOT NULL,
					rank INTEGER NOT NULL
				);
				CREATE TABLE mpaa_rating (
					atom_id INTEGER NOT NULL,
					rating mpaa NOT NULL,
					reason TEXT NOT NULL
				);
				`)
			return err
		},
	},
}

type index struct {
	unique   bool
	table    string
	name     string
	fulltext string // empty, "gin" or "gist"
	columns  []string
}

var indices = []index{
	{true, "atom", "", "", []string{"hash"}},
	{false, "episode", "tv", "", []string{"tvshow_atom_id"}},
	{false, "episode", "tvseason", "", []string{"tvshow_atom_id", "season"}},

	{false, "release_date", "", "", []string{"atom_id"}},
	{false, "running_time", "", "", []string{"atom_id"}},
	{false, "aka_title", "", "", []string{"atom_id"}},
	{false, "alternate_version", "", "", []string{"atom_id"}},
	{false, "color_info", "", "", []string{"atom_id"}},
	{false, "mpaa_rating", "", "", []string{"atom_id"}},
	{false, "sound_mix", "", "", []string{"atom_id"}},
	{false, "genre", "", "", []string{"atom_id"}},
	{false, "tagline", "", "", []string{"atom_id"}},
	{false, "trivia", "", "", []string{"atom_id"}},
	{false, "goof", "", "", []string{"atom_id"}},
	{false, "language", "", "", []string{"atom_id"}},
	{false, "literature", "", "", []string{"atom_id"}},
	{false, "location", "", "", []string{"atom_id"}},
	{false, "link", "", "", []string{"atom_id"}},
	{false, "plot", "", "", []string{"atom_id"}},
	{false, "quote", "", "", []string{"atom_id"}},
	{false, "rating", "", "", []string{"atom_id"}},
	{false, "name", "", "", []string{"atom_id"}},
	{false, "credit", "", "", []string{"actor_atom_id"}},
	{false, "credit", "", "", []string{"media_atom_id"}},

	{false, "name", "trgm_name", "gist", []string{"name"}},
	{false, "aka_title", "trgm_title", "gist", []string{"title"}},
}

func (in index) sqlName() string {
	name := in.name
	if len(in.columns) == 0 {
		panic("indices must have at least one column")
	}
	if len(name) == 0 {
		if len(in.columns) > 1 {
			panic("must specify index name for multi-column indices")
		}
		name = in.columns[0]
	}
	return sf("idx_%s_%s", in.table, name)
}

func (in index) sqlCreate(db *DB) string {
	uni := ""
	if in.unique {
		uni = " UNIQUE "
	}
	using, class := "", ""
	if in.isFulltext() {
		using = sf(" USING %s ", in.fulltext)
		switch in.fulltext {
		case "gin":
			class = " gin_trgm_ops"
		case "gist":
			class = " gist_trgm_ops"
		default:
			panic(sf("unrecognized fulltext index type: %s", in.fulltext))
		}
	}
	return sf("CREATE %s INDEX %s ON %s %s (%s%s)",
		uni, in.sqlName(), in.table, using,
		strings.Join(in.columns, ", "), class)
}

func (in index) isFulltext() bool {
	return len(in.fulltext) > 0
}

func (in index) sqlDrop(db *DB) string {
	return sf("DROP INDEX IF EXISTS %s", in.sqlName())
}

func doIndices(
	db *DB,
	getSql func(index, *DB) string,
	tables ...string,
) (err error) {
	defer csql.Safe(&err)

	trgmEnabled := db.IsFuzzyEnabled()
	var q string
	var ok bool
	for _, idx := range indices {
		if idx.isFulltext() && !trgmEnabled {
			// Only show the error message if we're on PostgreSQL.
			if db.Driver == "postgres" {
				logf("Skipping fulltext index '%s' since "+
					"the pg_trgm extension is not enabled.", idx.sqlName())
			}
			continue
		}
		if len(tables) == 0 || fun.In(idx.table, tables) {
			q += getSql(idx, db) + "; "
			ok = true
		}
	}
	if ok {
		csql.Exec(db, q)
	}
	return
}

func (db *DB) CreateIndices(tables ...string) error {
	return doIndices(db, index.sqlCreate, tables...)
}

func (db *DB) DropIndices(tables ...string) error {
	return doIndices(db, index.sqlDrop, tables...)
}
<|endoftext|>


<commit_before><commit_msg>Added note about query parameter binding.<commit_after><|endoftext|>


<commit_before>package slices

import (
	"fmt"
	"reflect"
	"testing"
)

// https://golang.org/cmd/go/#hdr-Description_of_testing_flags
// go test -v -bench string_test.go -benchtime 2s

var ()

func TestIsEmpty(t *testing.T) {

	if err := expect(IsEmpty([]string{}), true); err != nil {
		t.Error(err)
	}

	if err := expect(IsEmpty([]string{""}), true); err != nil {
		t.Error(err)
	}

	if err := expect(IsEmpty([]string{"a", "b", "c"}), false); err != nil {
		t.Error(err)
	}
}

func TestConcat(t *testing.T) {

	if err := expect(Concat([]string{}), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Concat([]string{""}), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Concat([]string{"a", "b", "c"}), "abc"); err != nil {
		t.Error(err)
	}
}

func TestJoin(t *testing.T) {

	if err := expect(Join([]string{}, "-"), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Join([]string{""}, "-"), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Join([]string{"a", "b", "c"}, "-"), "a-b-c"); err != nil {
		t.Error(err)
	}
}

func TestCut(t *testing.T) {

	if err := expect(Cut([]string{}, 0, -1), []string{}); err != nil {
		t.Error(err)
	}

	if err := expectNot(Cut([]string{}, 0, -1), []string{""}); err != nil {
		t.Error(err)
	}

	if err := expect(Cut([]string{""}, 0, -1), []string{""}); err != nil {
		t.Error(err)
	}
}

func TestIsSameArray(t *testing.T) {
	slc1 := []string{"same elements,", "different arrays"}
	if slc2 := Cut(slc1, 0, -1); IsSameArray(slc2, slc1) {
		t.Error(errExpected(false, slc2, slc1))
	}
}

// func BenchmarkStringsReplaceAll(b *testing.B) {
// 	for i := 0; i < b.N; i++ {
// 		strings.Replace(TestFileContent, ToFind, ToReplace, -1)
// 	}
// }

func expect(args ...interface{}) error {
	switch len(args) {
	case 0, 1:
		return fmt.Errorf("Not enough arguments to expect. Args passed: %v", args)
	default:
		return _expect(true, args...)
	}
}

func expectNot(args ...interface{}) error {
	switch len(args) {
	case 0, 1:
		return fmt.Errorf("Not enough arguments to expect. Args passed: %v", args)
	default:
		return _expect(false, args...)
	}
}

func _expect(boolWanted bool, args ...interface{}) error {

	result := args[0]
	expected := args[1]

	if boolReceived := _areEqual(result, expected); boolReceived != boolWanted {
		return errExpected(boolWanted, args...)
	}
	return nil
}

func _areEqual(result interface{}, expected interface{}) bool {
	return reflect.DeepEqual(result, expected)
}

func errExpected(boolWanted bool, args ...interface{}) error {

	condition := "to equal"
	if boolWanted == false {
		condition = "not to equal"
	}

	result := args[0]
	expected := args[1]

	switch len(args) {
	case 2:
		return fmt.Errorf("Expected '%v' %v '%v'", result, condition, expected)
	default:
		return fmt.Errorf("%v: Expected '%v' %v '%v'", args[2], result, condition, expected)
	}
}
<commit_msg>TestDuplicate()<commit_after>package slices

import (
	"fmt"
	"reflect"
	"testing"
)

// https://golang.org/cmd/go/#hdr-Description_of_testing_flags
// go test -v -bench string_test.go -benchtime 2s

var ()

func TestIsEmpty(t *testing.T) {

	if err := expect(IsEmpty([]string{}), true); err != nil {
		t.Error(err)
	}

	if err := expect(IsEmpty([]string{""}), true); err != nil {
		t.Error(err)
	}

	if err := expect(IsEmpty([]string{"a", "b", "c"}), false); err != nil {
		t.Error(err)
	}
}

func TestConcat(t *testing.T) {

	if err := expect(Concat([]string{}), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Concat([]string{""}), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Concat([]string{"a", "b", "c"}), "abc"); err != nil {
		t.Error(err)
	}
}

func TestJoin(t *testing.T) {

	if err := expect(Join([]string{}, "-"), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Join([]string{""}, "-"), ""); err != nil {
		t.Error(err)
	}

	if err := expect(Join([]string{"a", "b", "c"}, "-"), "a-b-c"); err != nil {
		t.Error(err)
	}
}

func TestCut(t *testing.T) {

	if err := expect(Cut([]string{}, 0, -1), []string{}); err != nil {
		t.Error(err)
	}

	if err := expectNot(Cut([]string{}, 0, -1), []string{""}); err != nil {
		t.Error(err)
	}

	if err := expect(Cut([]string{""}, 0, -1), []string{""}); err != nil {
		t.Error(err)
	}
}

func TestIsSameArray(t *testing.T) {
	slc1 := []string{"same elements,", "different arrays"}
	if slc2 := Cut(slc1, 0, -1); IsSameArray(slc2, slc1) {
		t.Error(errExpected(false, slc2, slc1))
	}
}

func TestDuplicate(t *testing.T) {
	slc1 := []string{"same elements,", "different arrays"}
	if slc2 := Duplicate(slc1); IsSameArray(slc2, slc1) {
		t.Error(errExpected(false, slc2, slc1))
	}
}

// func BenchmarkStringsReplaceAll(b *testing.B) {
// 	for i := 0; i < b.N; i++ {
// 		strings.Replace(TestFileContent, ToFind, ToReplace, -1)
// 	}
// }

func expect(args ...interface{}) error {
	switch len(args) {
	case 0, 1:
		return fmt.Errorf("Not enough arguments to expect. Args passed: %v", args)
	default:
		return _expect(true, args...)
	}
}

func expectNot(args ...interface{}) error {
	switch len(args) {
	case 0, 1:
		return fmt.Errorf("Not enough arguments to expect. Args passed: %v", args)
	default:
		return _expect(false, args...)
	}
}

func _expect(boolWanted bool, args ...interface{}) error {

	result := args[0]
	expected := args[1]

	if boolReceived := _areEqual(result, expected); boolReceived != boolWanted {
		return errExpected(boolWanted, args...)
	}
	return nil
}

func _areEqual(result interface{}, expected interface{}) bool {
	return reflect.DeepEqual(result, expected)
}

func errExpected(boolWanted bool, args ...interface{}) error {

	condition := "to equal"
	if boolWanted == false {
		condition = "not to equal"
	}

	result := args[0]
	expected := args[1]

	switch len(args) {
	case 2:
		return fmt.Errorf("Expected '%v' %v '%v'", result, condition, expected)
	default:
		return fmt.Errorf("%v: Expected '%v' %v '%v'", args[2], result, condition, expected)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The os package provides a platform-independent interface to operating
// system functionality.  The design is Unix-like.
package os

import (
	"os";
	"syscall";
)

// Auxiliary information if the File describes a directory
type dirInfo struct {
	buf	[]byte;	// buffer for directory I/O
	nbuf	int64;	// length of buf; return value from Getdirentries
	bufp	int64;	// location of next record in buf.
}

// File represents an open file descriptor.
type File struct {
	fd int64;
	name	string;
	dirinfo	*dirInfo;	// nil unless directory being read
}

// Fd returns the integer Unix file descriptor referencing the open file.
func (file *File) Fd() int64 {
	return file.fd
}

// Name returns the name of the file as presented to Open.
func (file *File) Name() string {
	return file.name
}

// NewFile returns a new File with the given file descriptor and name.
func NewFile(file int64, name string) *File {
	if file < 0 {
		return nil
	}
	return &File{file, name, nil}
}

// Stdin, Stdout, and Stderr are open Files pointing to the standard input,
// standard output, and standard error file descriptors.
var (
	Stdin  = NewFile(0, "/dev/stdin");
	Stdout = NewFile(1, "/dev/stdout");
	Stderr = NewFile(2, "/dev/stderr");
)

// Flags to Open wrapping those of the underlying system. Not all flags
// may be implemented on a given system.
const (
	O_RDONLY = syscall.O_RDONLY;	// open the file read-only.
	O_WRONLY = syscall.O_WRONLY;	// open the file write-only.
	O_RDWR = syscall.O_RDWR;	// open the file read-write.
	O_APPEND = syscall.O_APPEND;	// open the file append-only.
	O_ASYNC = syscall.O_ASYNC;	// generate a signal when I/O is available.
	O_CREAT = syscall.O_CREAT;	// create a new file if none exists.
	O_NOCTTY = syscall.O_NOCTTY;	// do not make file the controlling tty.
	O_NONBLOCK = syscall.O_NONBLOCK;	// open in non-blocking mode.
	O_NDELAY = O_NONBLOCK;		// synonym for O_NONBLOCK
	O_SYNC = syscall.O_SYNC;	// open for synchronous I/O.
	O_TRUNC = syscall.O_TRUNC;	// if possible, truncate file when opened.
)

// Open opens the named file with specified flag (O_RDONLY etc.) and perm, (0666 etc.)
// if applicable.  If successful, methods on the returned File can be used for I/O.
// It returns the File and an Error, if any.
func Open(name string, flag int, perm int) (file *File, err Error) {
	r, e := syscall.Open(name, int64(flag | syscall.O_CLOEXEC), int64(perm));
	if e != 0 {
		return nil, ErrnoToError(e);
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec.go
	if syscall.O_CLOEXEC == 0 {	// O_CLOEXEC not supported
		syscall.CloseOnExec(r);
	}

	return NewFile(r, name), ErrnoToError(e)
}

// Close closes the File, rendering it unusable for I/O.
// It returns an Error, if any.
func (file *File) Close() Error {
	if file == nil {
		return EINVAL
	}
	r, e := syscall.Close(file.fd);
	file.fd = -1;  // so it can't be closed again
	return ErrnoToError(e)
}

// Read reads up to len(b) bytes from the File.
// It returns the number of bytes read and an Error, if any.
// EOF is signaled by a zero count with a nil Error.
// TODO(r): Add Pread, Pwrite (maybe ReadAt, WriteAt).
func (file *File) Read(b []byte) (ret int, err Error) {
	if file == nil {
		return 0, EINVAL
	}
	var r, e int64;
	if len(b) > 0 {  // because we access b[0]
		r, e = syscall.Read(file.fd, &b[0], int64(len(b)));
		if r < 0 {
			r = 0
		}
	}
	return int(r), ErrnoToError(e)
}

// Write writes len(b) bytes to the File.
// It returns the number of bytes written and an Error, if any.
// If the byte count differs from len(b), it usually implies an error occurred.
func (file *File) Write(b []byte) (ret int, err Error) {
	if file == nil {
		return 0, EINVAL
	}
	var r, e int64;
	if len(b) > 0 {  // because we access b[0]
		r, e = syscall.Write(file.fd, &b[0], int64(len(b)));
		if r < 0 {
			r = 0
		}
	}
	return int(r), ErrnoToError(e)
}

// Seek sets the offset for the next Read or Write on file to offset, interpreted
// according to whence: 0 means relative to the origin of the file, 1 means
// relative to the current offset, and 2 means relative to the end.
// It returns the new offset and an Error, if any.
func (file *File) Seek(offset int64, whence int) (ret int64, err Error) {
	r, e := syscall.Seek(file.fd, offset, int64(whence));
	if e != 0 {
		return -1, ErrnoToError(e)
	}
	if file.dirinfo != nil && r != 0 {
		return -1, ErrnoToError(syscall.EISDIR)
	}
	return r, nil
}

// WriteString is like Write, but writes the contents of string s rather than
// an array of bytes.
func (file *File) WriteString(s string) (ret int, err Error) {
	if file == nil {
		return 0, EINVAL
	}
	r, e := syscall.Write(file.fd, syscall.StringBytePtr(s), int64(len(s)));
	if r < 0 {
		r = 0
	}
	return int(r), ErrnoToError(e)
}

// Pipe returns a connected pair of Files; reads from r return bytes written to w.
// It returns the files and an Error, if any.
func Pipe() (r *File, w *File, err Error) {
	var p [2]int64;

	// See ../syscall/exec.go for description of lock.
	syscall.ForkLock.RLock();
	ret, e := syscall.Pipe(&p);
	if e != 0 {
		syscall.ForkLock.RUnlock();
		return nil, nil, ErrnoToError(e)
	}
	syscall.CloseOnExec(p[0]);
	syscall.CloseOnExec(p[1]);
	syscall.ForkLock.RUnlock();

	return NewFile(p[0], "|0"), NewFile(p[1], "|1"), nil
}

// Mkdir creates a new directory with the specified name and permission bits.
// It returns an error, if any.
func Mkdir(name string, perm int) Error {
	r, e := syscall.Mkdir(name, int64(perm));
	return ErrnoToError(e)
}

// Stat returns the Dir structure describing the named file. If the file
// is a symbolic link, it returns information about the file the link
// references.
// It returns the Dir and an error, if any.
func Stat(name string) (dir *Dir, err Error) {
	stat := new(syscall.Stat_t);
	r, e := syscall.Stat(name, stat);
	if e != 0 {
		return nil, ErrnoToError(e)
	}
	return dirFromStat(name, new(Dir), stat), nil
}

// Stat returns the Dir structure describing file.
// It returns the Dir and an error, if any.
func (file *File) Stat() (dir *Dir, err Error) {
	stat := new(syscall.Stat_t);
	r, e := syscall.Fstat(file.fd, stat);
	if e != 0 {
		return nil, ErrnoToError(e)
	}
	return dirFromStat(file.name, new(Dir), stat), nil
}

// Lstat returns the Dir structure describing the named file. If the file
// is a symbolic link, it returns information about the link itself.
// It returns the Dir and an error, if any.
func Lstat(name string) (dir *Dir, err Error) {
	stat := new(syscall.Stat_t);
	r, e := syscall.Lstat(name, stat);
	if e != 0 {
		return nil, ErrnoToError(e)
	}
	return dirFromStat(name, new(Dir), stat), nil
}

// Readdirnames has a non-portable implemenation so its code is separated into an
// operating-system-dependent file.
func readdirnames(file *File, count int) (names []string, err Error)

// Readdirnames reads the contents of the directory associated with file and
// returns an array of up to count names, in directory order.  Subsequent
// calls on the same file will yield further names.
// A negative count means to read until EOF.
// It returns the array and an Error, if any.
func (file *File) Readdirnames(count int) (names []string, err Error) {
	return readdirnames(file, count);
}

// Readdir reads the contents of the directory associated with file and
// returns an array of up to count Dir structures, in directory order.  Subsequent
// calls on the same file will yield further Dirs.
// A negative count means to read until EOF.
// It returns the array and an Error, if any.
func (file *File) Readdir(count int) (dirs []Dir, err Error) {
	dirname := file.name;
	if dirname == "" {
		dirname = ".";
	}
	dirname += "/";
	names, err1 := file.Readdirnames(count);
	if err1 != nil {
		return nil, err1
	}
	dirs = make([]Dir, len(names));
	for i, filename := range names {
		dirp, err := Stat(dirname + filename);
		if dirp ==  nil || err != nil {
			dirs[i].Name = filename	// rest is already zeroed out
		} else {
			dirs[i] = *dirp
		}
	}
	return
}

// Chdir changes the current working directory to the named directory.
func Chdir(dir string) Error {
	r, e := syscall.Chdir(dir);
	return ErrnoToError(e);
}

// Remove removes the named file or directory.
func Remove(name string) Error {
	// System call interface forces us to know
	// whether name is a file or directory.
	// Try both: it is cheaper on average than
	// doing a Stat plus the right one.
	r, e := syscall.Unlink(name);
	if e == 0 {
		return nil;
	}
	r1, e1 := syscall.Rmdir(name);
	if e1 == 0 {
		return nil;
	}

	// Both failed: figure out which error to return.
	// OS X and Linux differ on whether unlink(dir)
	// returns EISDIR, so can't use that.  However,
	// both agree that rmdir(file) returns ENOTDIR,
	// so we can use that to decide which error is real.
	// Rmdir might also return ENOTDIR if given a bad
	// file path, like /etc/passwd/foo, but in that case,
	// both errors will be ENOTDIR, so it's okay to
	// use the error from unlink.
	if e1 != syscall.ENOTDIR {
		e = e1;
	}
	return ErrnoToError(e);
}

<commit_msg>exit with error status EPIPE if one fd gets too many EPIPEs in a row during write.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The os package provides a platform-independent interface to operating
// system functionality.  The design is Unix-like.
package os

import (
	"os";
	"syscall";
)

// Auxiliary information if the File describes a directory
type dirInfo struct {
	buf	[]byte;	// buffer for directory I/O
	nbuf	int64;	// length of buf; return value from Getdirentries
	bufp	int64;	// location of next record in buf.
}

// File represents an open file descriptor.
type File struct {
	fd int64;
	name	string;
	dirinfo	*dirInfo;	// nil unless directory being read
	nepipe	int;	// number of consecutive EPIPE in Write
}

// Fd returns the integer Unix file descriptor referencing the open file.
func (file *File) Fd() int64 {
	return file.fd
}

// Name returns the name of the file as presented to Open.
func (file *File) Name() string {
	return file.name
}

// NewFile returns a new File with the given file descriptor and name.
func NewFile(file int64, name string) *File {
	if file < 0 {
		return nil
	}
	return &File{file, name, nil, 0}
}

// Stdin, Stdout, and Stderr are open Files pointing to the standard input,
// standard output, and standard error file descriptors.
var (
	Stdin  = NewFile(0, "/dev/stdin");
	Stdout = NewFile(1, "/dev/stdout");
	Stderr = NewFile(2, "/dev/stderr");
)

// Flags to Open wrapping those of the underlying system. Not all flags
// may be implemented on a given system.
const (
	O_RDONLY = syscall.O_RDONLY;	// open the file read-only.
	O_WRONLY = syscall.O_WRONLY;	// open the file write-only.
	O_RDWR = syscall.O_RDWR;	// open the file read-write.
	O_APPEND = syscall.O_APPEND;	// open the file append-only.
	O_ASYNC = syscall.O_ASYNC;	// generate a signal when I/O is available.
	O_CREAT = syscall.O_CREAT;	// create a new file if none exists.
	O_NOCTTY = syscall.O_NOCTTY;	// do not make file the controlling tty.
	O_NONBLOCK = syscall.O_NONBLOCK;	// open in non-blocking mode.
	O_NDELAY = O_NONBLOCK;		// synonym for O_NONBLOCK
	O_SYNC = syscall.O_SYNC;	// open for synchronous I/O.
	O_TRUNC = syscall.O_TRUNC;	// if possible, truncate file when opened.
)

// Open opens the named file with specified flag (O_RDONLY etc.) and perm, (0666 etc.)
// if applicable.  If successful, methods on the returned File can be used for I/O.
// It returns the File and an Error, if any.
func Open(name string, flag int, perm int) (file *File, err Error) {
	r, e := syscall.Open(name, int64(flag | syscall.O_CLOEXEC), int64(perm));
	if e != 0 {
		return nil, ErrnoToError(e);
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec.go
	if syscall.O_CLOEXEC == 0 {	// O_CLOEXEC not supported
		syscall.CloseOnExec(r);
	}

	return NewFile(r, name), ErrnoToError(e)
}

// Close closes the File, rendering it unusable for I/O.
// It returns an Error, if any.
func (file *File) Close() Error {
	if file == nil {
		return EINVAL
	}
	r, e := syscall.Close(file.fd);
	file.fd = -1;  // so it can't be closed again
	return ErrnoToError(e)
}

// Read reads up to len(b) bytes from the File.
// It returns the number of bytes read and an Error, if any.
// EOF is signaled by a zero count with a nil Error.
// TODO(r): Add Pread, Pwrite (maybe ReadAt, WriteAt).
func (file *File) Read(b []byte) (ret int, err Error) {
	if file == nil {
		return 0, EINVAL
	}
	var r, e int64;
	if len(b) > 0 {  // because we access b[0]
		r, e = syscall.Read(file.fd, &b[0], int64(len(b)));
		if r < 0 {
			r = 0
		}
	}
	return int(r), ErrnoToError(e)
}

// Write writes len(b) bytes to the File.
// It returns the number of bytes written and an Error, if any.
// If the byte count differs from len(b), it usually implies an error occurred.
func (file *File) Write(b []byte) (ret int, err Error) {
	if file == nil {
		return 0, EINVAL
	}
	var r, e int64;
	if len(b) > 0 {  // because we access b[0]
		r, e = syscall.Write(file.fd, &b[0], int64(len(b)));
		if r < 0 {
			r = 0
		}
	}
	if e == syscall.EPIPE {
		file.nepipe++;
		if file.nepipe >= 10 {
			sys.Exit(syscall.EPIPE);
		}
	} else {
		file.nepipe = 0;
	}
	return int(r), ErrnoToError(e)
}

// Seek sets the offset for the next Read or Write on file to offset, interpreted
// according to whence: 0 means relative to the origin of the file, 1 means
// relative to the current offset, and 2 means relative to the end.
// It returns the new offset and an Error, if any.
func (file *File) Seek(offset int64, whence int) (ret int64, err Error) {
	r, e := syscall.Seek(file.fd, offset, int64(whence));
	if e != 0 {
		return -1, ErrnoToError(e)
	}
	if file.dirinfo != nil && r != 0 {
		return -1, ErrnoToError(syscall.EISDIR)
	}
	return r, nil
}

// WriteString is like Write, but writes the contents of string s rather than
// an array of bytes.
func (file *File) WriteString(s string) (ret int, err Error) {
	if file == nil {
		return 0, EINVAL
	}
	r, e := syscall.Write(file.fd, syscall.StringBytePtr(s), int64(len(s)));
	if r < 0 {
		r = 0
	}
	return int(r), ErrnoToError(e)
}

// Pipe returns a connected pair of Files; reads from r return bytes written to w.
// It returns the files and an Error, if any.
func Pipe() (r *File, w *File, err Error) {
	var p [2]int64;

	// See ../syscall/exec.go for description of lock.
	syscall.ForkLock.RLock();
	ret, e := syscall.Pipe(&p);
	if e != 0 {
		syscall.ForkLock.RUnlock();
		return nil, nil, ErrnoToError(e)
	}
	syscall.CloseOnExec(p[0]);
	syscall.CloseOnExec(p[1]);
	syscall.ForkLock.RUnlock();

	return NewFile(p[0], "|0"), NewFile(p[1], "|1"), nil
}

// Mkdir creates a new directory with the specified name and permission bits.
// It returns an error, if any.
func Mkdir(name string, perm int) Error {
	r, e := syscall.Mkdir(name, int64(perm));
	return ErrnoToError(e)
}

// Stat returns the Dir structure describing the named file. If the file
// is a symbolic link, it returns information about the file the link
// references.
// It returns the Dir and an error, if any.
func Stat(name string) (dir *Dir, err Error) {
	stat := new(syscall.Stat_t);
	r, e := syscall.Stat(name, stat);
	if e != 0 {
		return nil, ErrnoToError(e)
	}
	return dirFromStat(name, new(Dir), stat), nil
}

// Stat returns the Dir structure describing file.
// It returns the Dir and an error, if any.
func (file *File) Stat() (dir *Dir, err Error) {
	stat := new(syscall.Stat_t);
	r, e := syscall.Fstat(file.fd, stat);
	if e != 0 {
		return nil, ErrnoToError(e)
	}
	return dirFromStat(file.name, new(Dir), stat), nil
}

// Lstat returns the Dir structure describing the named file. If the file
// is a symbolic link, it returns information about the link itself.
// It returns the Dir and an error, if any.
func Lstat(name string) (dir *Dir, err Error) {
	stat := new(syscall.Stat_t);
	r, e := syscall.Lstat(name, stat);
	if e != 0 {
		return nil, ErrnoToError(e)
	}
	return dirFromStat(name, new(Dir), stat), nil
}

// Readdirnames has a non-portable implemenation so its code is separated into an
// operating-system-dependent file.
func readdirnames(file *File, count int) (names []string, err Error)

// Readdirnames reads the contents of the directory associated with file and
// returns an array of up to count names, in directory order.  Subsequent
// calls on the same file will yield further names.
// A negative count means to read until EOF.
// It returns the array and an Error, if any.
func (file *File) Readdirnames(count int) (names []string, err Error) {
	return readdirnames(file, count);
}

// Readdir reads the contents of the directory associated with file and
// returns an array of up to count Dir structures, in directory order.  Subsequent
// calls on the same file will yield further Dirs.
// A negative count means to read until EOF.
// It returns the array and an Error, if any.
func (file *File) Readdir(count int) (dirs []Dir, err Error) {
	dirname := file.name;
	if dirname == "" {
		dirname = ".";
	}
	dirname += "/";
	names, err1 := file.Readdirnames(count);
	if err1 != nil {
		return nil, err1
	}
	dirs = make([]Dir, len(names));
	for i, filename := range names {
		dirp, err := Stat(dirname + filename);
		if dirp ==  nil || err != nil {
			dirs[i].Name = filename	// rest is already zeroed out
		} else {
			dirs[i] = *dirp
		}
	}
	return
}

// Chdir changes the current working directory to the named directory.
func Chdir(dir string) Error {
	r, e := syscall.Chdir(dir);
	return ErrnoToError(e);
}

// Remove removes the named file or directory.
func Remove(name string) Error {
	// System call interface forces us to know
	// whether name is a file or directory.
	// Try both: it is cheaper on average than
	// doing a Stat plus the right one.
	r, e := syscall.Unlink(name);
	if e == 0 {
		return nil;
	}
	r1, e1 := syscall.Rmdir(name);
	if e1 == 0 {
		return nil;
	}

	// Both failed: figure out which error to return.
	// OS X and Linux differ on whether unlink(dir)
	// returns EISDIR, so can't use that.  However,
	// both agree that rmdir(file) returns ENOTDIR,
	// so we can use that to decide which error is real.
	// Rmdir might also return ENOTDIR if given a bad
	// file path, like /etc/passwd/foo, but in that case,
	// both errors will be ENOTDIR, so it's okay to
	// use the error from unlink.
	if e1 != syscall.ENOTDIR {
		e = e1;
	}
	return ErrnoToError(e);
}

<|endoftext|>


<commit_before>package weed_server

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"mime/multipart"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"

	"github.com/gorilla/mux"
)

var serverStats *stats.ServerStats
var startTime = time.Now()

func init() {
	serverStats = stats.NewServerStats()
	go serverStats.Start()
}

// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
func bodyAllowedForStatus(status int) bool {
	switch {
	case status >= 100 && status <= 199:
		return false
	case status == http.StatusNoContent:
		return false
	case status == http.StatusNotModified:
		return false
	}
	return true
}

func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
	if !bodyAllowedForStatus(httpStatus) {
		return
	}

	var bytes []byte
	if obj != nil {
		if r.FormValue("pretty") != "" {
			bytes, err = json.MarshalIndent(obj, "", "  ")
		} else {
			bytes, err = json.Marshal(obj)
		}
	}
	if err != nil {
		return
	}

	if httpStatus >= 400 {
		glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s",
			r.Method, r.URL.String(), httpStatus, string(bytes))
	}

	callback := r.FormValue("callback")
	if callback == "" {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		_, err = w.Write(bytes)
	} else {
		w.Header().Set("Content-Type", "application/javascript")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}

	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
	if err := writeJson(w, r, httpStatus, obj); err != nil {
		glog.V(0).Infof("error writing JSON status %d: %v", httpStatus, err)
		glog.V(1).Infof("JSON content: %+v", obj)
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
	m := make(map[string]interface{})
	m["error"] = err.Error()
	writeJsonQuiet(w, r, httpStatus, m)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}

func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) {
	m := make(map[string]interface{})
	if r.Method != "POST" {
		writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!"))
		return
	}

	debug("parsing upload file...")
	bytesBuffer := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(bytesBuffer)
	pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
	if pe != nil {
		writeJsonError(w, r, http.StatusBadRequest, pe)
		return
	}

	debug("assigning file id for", pu.FileName)
	r.ParseForm()
	count := uint64(1)
	if r.FormValue("count") != "" {
		count, pe = strconv.ParseUint(r.FormValue("count"), 10, 32)
		if pe != nil {
			writeJsonError(w, r, http.StatusBadRequest, pe)
			return
		}
	}
	ar := &operation.VolumeAssignRequest{
		Count:       count,
		DataCenter:  r.FormValue("dataCenter"),
		Rack:        r.FormValue("rack"),
		Replication: r.FormValue("replication"),
		Collection:  r.FormValue("collection"),
		Ttl:         r.FormValue("ttl"),
		DiskType:    r.FormValue("disk"),
	}
	assignResult, ae := operation.Assign(masterFn, grpcDialOption, ar)
	if ae != nil {
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		return
	}

	url := "http://" + assignResult.Url + "/" + assignResult.Fid
	if pu.ModifiedTime != 0 {
		url = url + "?ts=" + strconv.FormatUint(pu.ModifiedTime, 10)
	}

	debug("upload file to store", url)
	uploadOption := &operation.UploadOption{
		UploadUrl:         url,
		Filename:          pu.FileName,
		Cipher:            false,
		IsInputCompressed: pu.IsGzipped,
		MimeType:          pu.MimeType,
		PairMap:           pu.PairMap,
		Jwt:               assignResult.Auth,
	}
	uploadResult, err := operation.UploadData(pu.Data, uploadOption)
	if err != nil {
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	m["fileName"] = pu.FileName
	m["fid"] = assignResult.Fid
	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
	m["size"] = pu.OriginalDataSize
	m["eTag"] = uploadResult.ETag
	writeJsonQuiet(w, r, http.StatusCreated, m)
	return
}

func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
	switch strings.Count(path, "/") {
	case 3:
		parts := strings.Split(path, "/")
		vid, fid, filename = parts[1], parts[2], parts[3]
		ext = filepath.Ext(filename)
	case 2:
		parts := strings.Split(path, "/")
		vid, fid = parts[1], parts[2]
		dotIndex := strings.LastIndex(fid, ".")
		if dotIndex > 0 {
			ext = fid[dotIndex:]
			fid = fid[0:dotIndex]
		}
	default:
		sepIndex := strings.LastIndex(path, "/")
		commaIndex := strings.LastIndex(path[sepIndex:], ",")
		if commaIndex <= 0 {
			vid, isVolumeIdOnly = path[sepIndex+1:], true
			return
		}
		dotIndex := strings.LastIndex(path[sepIndex:], ".")
		vid = path[sepIndex+1 : commaIndex]
		fid = path[commaIndex+1:]
		ext = ""
		if dotIndex > 0 {
			fid = path[commaIndex+1 : dotIndex]
			ext = path[dotIndex:]
		}
	}
	return
}

func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	writeJsonQuiet(w, r, http.StatusOK, m)
}
func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Counters"] = serverStats
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Memory"] = stats.MemStat()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

var StaticFS fs.FS

func handleStaticResources(defaultMux *http.ServeMux) {
	defaultMux.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	defaultMux.Handle("/seaweedfsstatic/", http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func handleStaticResources2(r *mux.Router) {
	r.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
	if filename != "" {
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
}

func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
	rangeReq := r.Header.Get("Range")

	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if err := writeFn(w, 0, totalSize); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))

		w.WriteHeader(http.StatusPartialContent)
		err = writeFn(w, ra.start, ra.length)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if e = writeFn(part, ra.start, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	if _, err := io.CopyN(w, sendContent, sendSize); err != nil {
		http.Error(w, "Internal Error", http.StatusInternalServerError)
		return
	}
}
<commit_msg>s3: use "response-content-disposition" to overwrite default content-disposition<commit_after>package weed_server

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"mime/multipart"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"

	"github.com/gorilla/mux"
)

var serverStats *stats.ServerStats
var startTime = time.Now()

func init() {
	serverStats = stats.NewServerStats()
	go serverStats.Start()
}

// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
func bodyAllowedForStatus(status int) bool {
	switch {
	case status >= 100 && status <= 199:
		return false
	case status == http.StatusNoContent:
		return false
	case status == http.StatusNotModified:
		return false
	}
	return true
}

func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
	if !bodyAllowedForStatus(httpStatus) {
		return
	}

	var bytes []byte
	if obj != nil {
		if r.FormValue("pretty") != "" {
			bytes, err = json.MarshalIndent(obj, "", "  ")
		} else {
			bytes, err = json.Marshal(obj)
		}
	}
	if err != nil {
		return
	}

	if httpStatus >= 400 {
		glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s",
			r.Method, r.URL.String(), httpStatus, string(bytes))
	}

	callback := r.FormValue("callback")
	if callback == "" {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		_, err = w.Write(bytes)
	} else {
		w.Header().Set("Content-Type", "application/javascript")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}

	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
	if err := writeJson(w, r, httpStatus, obj); err != nil {
		glog.V(0).Infof("error writing JSON status %d: %v", httpStatus, err)
		glog.V(1).Infof("JSON content: %+v", obj)
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
	m := make(map[string]interface{})
	m["error"] = err.Error()
	writeJsonQuiet(w, r, httpStatus, m)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}

func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) {
	m := make(map[string]interface{})
	if r.Method != "POST" {
		writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!"))
		return
	}

	debug("parsing upload file...")
	bytesBuffer := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(bytesBuffer)
	pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
	if pe != nil {
		writeJsonError(w, r, http.StatusBadRequest, pe)
		return
	}

	debug("assigning file id for", pu.FileName)
	r.ParseForm()
	count := uint64(1)
	if r.FormValue("count") != "" {
		count, pe = strconv.ParseUint(r.FormValue("count"), 10, 32)
		if pe != nil {
			writeJsonError(w, r, http.StatusBadRequest, pe)
			return
		}
	}
	ar := &operation.VolumeAssignRequest{
		Count:       count,
		DataCenter:  r.FormValue("dataCenter"),
		Rack:        r.FormValue("rack"),
		Replication: r.FormValue("replication"),
		Collection:  r.FormValue("collection"),
		Ttl:         r.FormValue("ttl"),
		DiskType:    r.FormValue("disk"),
	}
	assignResult, ae := operation.Assign(masterFn, grpcDialOption, ar)
	if ae != nil {
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		return
	}

	url := "http://" + assignResult.Url + "/" + assignResult.Fid
	if pu.ModifiedTime != 0 {
		url = url + "?ts=" + strconv.FormatUint(pu.ModifiedTime, 10)
	}

	debug("upload file to store", url)
	uploadOption := &operation.UploadOption{
		UploadUrl:         url,
		Filename:          pu.FileName,
		Cipher:            false,
		IsInputCompressed: pu.IsGzipped,
		MimeType:          pu.MimeType,
		PairMap:           pu.PairMap,
		Jwt:               assignResult.Auth,
	}
	uploadResult, err := operation.UploadData(pu.Data, uploadOption)
	if err != nil {
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	m["fileName"] = pu.FileName
	m["fid"] = assignResult.Fid
	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
	m["size"] = pu.OriginalDataSize
	m["eTag"] = uploadResult.ETag
	writeJsonQuiet(w, r, http.StatusCreated, m)
	return
}

func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
	switch strings.Count(path, "/") {
	case 3:
		parts := strings.Split(path, "/")
		vid, fid, filename = parts[1], parts[2], parts[3]
		ext = filepath.Ext(filename)
	case 2:
		parts := strings.Split(path, "/")
		vid, fid = parts[1], parts[2]
		dotIndex := strings.LastIndex(fid, ".")
		if dotIndex > 0 {
			ext = fid[dotIndex:]
			fid = fid[0:dotIndex]
		}
	default:
		sepIndex := strings.LastIndex(path, "/")
		commaIndex := strings.LastIndex(path[sepIndex:], ",")
		if commaIndex <= 0 {
			vid, isVolumeIdOnly = path[sepIndex+1:], true
			return
		}
		dotIndex := strings.LastIndex(path[sepIndex:], ".")
		vid = path[sepIndex+1 : commaIndex]
		fid = path[commaIndex+1:]
		ext = ""
		if dotIndex > 0 {
			fid = path[commaIndex+1 : dotIndex]
			ext = path[dotIndex:]
		}
	}
	return
}

func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	writeJsonQuiet(w, r, http.StatusOK, m)
}
func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Counters"] = serverStats
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Memory"] = stats.MemStat()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

var StaticFS fs.FS

func handleStaticResources(defaultMux *http.ServeMux) {
	defaultMux.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	defaultMux.Handle("/seaweedfsstatic/", http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func handleStaticResources2(r *mux.Router) {
	r.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
	responseContentDisposition := r.FormValue("response-content-disposition")
	if responseContentDisposition != "" {
		w.Header().Set("Content-Disposition", responseContentDisposition)
		return
	}
	if filename != "" {
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
}

func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
	rangeReq := r.Header.Get("Range")

	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if err := writeFn(w, 0, totalSize); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))

		w.WriteHeader(http.StatusPartialContent)
		err = writeFn(w, ra.start, ra.length)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if e = writeFn(part, ra.start, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	if _, err := io.CopyN(w, sendContent, sendSize); err != nil {
		http.Error(w, "Internal Error", http.StatusInternalServerError)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Exported LRE interface.

package match

import (
	"fmt"
	"sync"
)

// An LRE is a compiled license regular expression.
//
// TODO: Move this comment somewhere non-internal later.
//
// A license regular expression (LRE) is a pattern syntax intended for
// describing large English texts such as software licenses, with minor
// allowed variations. The pattern syntax and the matching are word-based
// and case-insensitive; punctuation is ignored in the pattern and in
// the matched text.
//
// The valid LRE patterns are:
//
//	word            - a single case-insensitive word
//	__N__           - any sequence of up to N words
//	expr1 expr2     - concatenation
//	expr1 || expr2  - alternation
//	(( expr ))      - grouping
//	expr??          - zero or one instances of expr
//	//** text **//  - a comment
//
// To make patterns harder to misread in large texts:
//
//	- || must only appear inside (( ))
//	- ?? must only follow (( ))
//	- (( must be at the start of a line, preceded only by spaces
//	- )) must be at the end of a line, followed only by spaces and ??.
//
// For example:
//
//	//** https://en.wikipedia.org/wiki/Filler_text **//
//	Now is
//	((not))??
//	the time for all good
//	((men || women || people))
//	to come to the aid of their __1__.
//
type LRE struct {
	dict   *Dict
	file   string
	syntax *reSyntax
	prog   reProg

	onceDFA sync.Once
	dfa     reDFA
}

// ParseLRE parses the string s as a license regexp.
// The file name is used in error messages if non-empty.
func ParseLRE(d *Dict, file, s string) (*LRE, error) {
	syntax, err := reParse(d, s, true)
	if err != nil {
		return nil, err
	}
	prog, err := syntax.compile(nil, 0)
	if err != nil {
		return nil, err
	}
	return &LRE{dict: d, file: file, syntax: syntax, prog: prog}, nil
}

// Dict returns the Dict used by the LRE.
func (re *LRE) Dict() *Dict {
	return re.dict
}

// File returns the file name passed to ParseLRE.
func (re *LRE) File() string {
	return re.file
}

// Match reports whether text matches the license regexp.
func (re *LRE) match(text string) bool {
	re.onceDFA.Do(re.compile)
	match, _ := re.dfa.match(re.dict, text, re.dict.Split(text))
	return match >= 0
}

// compile initializes lre.dfa.
// It is invoked lazily (in Match) because most LREs end up only
// being inputs to a MultiLRE; we never need their DFAs directly.
func (re *LRE) compile() {
	re.dfa = reCompileDFA(re.prog)
}

// A MultiLRE matches multiple LREs simultaneously against a text.
// It is more efficient than matching each LRE in sequence against the text.
type MultiLRE struct {
	dict *Dict // dict shared by all LREs
	dfa  reDFA // compiled DFA for all LREs

	// start contains the two-word phrases
	// where a match can validly start,
	// to allow for faster scans over non-license text.
	start map[phrase]struct{}
}

// A phrase is a phrase of up to two words.
// The zero-word phrase is phrase{NoWord, NoWord}.
// A single-word phrase w is phrase{w, NoWord}.
type phrase [2]WordID

// NewMultiLRE returns a MultiLRE looking for the given LREs.
// All the LREs must have been parsed using the same Dict;
// if not, NewMultiLRE panics.
func NewMultiLRE(list []*LRE) (_ *MultiLRE, err error) {
	if len(list) == 0 {
		return &MultiLRE{}, nil
	}

	dict := list[0].dict
	for _, sub := range list[1:] {
		if sub.dict != dict {
			panic("MultiRE: LREs parsed with different Dicts")
		}
	}

	var progs []reProg
	for _, sub := range list {
		progs = append(progs, sub.prog)
	}

	start := make(map[phrase]struct{})
	for _, sub := range list {
		phrases := sub.syntax.leadingPhrases()
		if len(phrases) == 0 {
			return nil, fmt.Errorf("%s: no leading phrases", sub.File())
		}
		for _, p := range phrases {
			if p[0] == BadWord {
				return nil, fmt.Errorf("%s: invalid pattern: matches empty text", sub.File())
			}
			if p[0] == AnyWord {
				if p[1] == BadWord {
					return nil, fmt.Errorf("%s: invalid pattern: matches a single wildcard", sub.File())
				}
				if p[1] == AnyWord {
					return nil, fmt.Errorf("%s: invalid pattern: begins with two wildcards", sub.File())
				}
				return nil, fmt.Errorf("%s: invalid pattern: begins with wildcard phrase: __ %s", sub.File(), dict.Words()[p[1]])
			}
			if p[1] == BadWord {
				return nil, fmt.Errorf("%s: invalid pattern: matches single word %s", sub.File(), dict.Words()[p[0]])
			}
			if p[1] == AnyWord {
				return nil, fmt.Errorf("%s: invalid pattern: begins with wildcard phrase: %s __", sub.File(), dict.Words()[p[0]])
			}
			start[p] = struct{}{}
		}
	}

	prog := reCompileMulti(progs)
	dfa := reCompileDFA(prog)

	return &MultiLRE{dict, dfa, start}, nil
}

// Dict returns the Dict used by the MultiLRE.
func (re *MultiLRE) Dict() *Dict {
	return re.dict
}

// A Matches is a collection of all leftmost-longest, non-overlapping matches in text.
type Matches struct {
	Text  string  // the entire text
	Words []Word  // the text, split into Words
	List  []Match // the matches
}

// A Match records the position of a single match in a text.
type Match struct {
	ID    int // index of LRE in list passed to NewMultiLRE
	Start int // word index of start of match
	End   int // word index of end of match
}

// Match reports all leftmost-longest, non-overlapping matches in text.
func (re *MultiLRE) Match(text string) *Matches {
	m := &Matches{
		Text:  text,
		Words: re.dict.Split(text),
	}
	p := phrase{BadWord, BadWord}
	for i := 0; i < len(m.Words); i++ {
		p[0], p[1] = p[1], m.Words[i].ID
		if _, ok := re.start[p]; ok {
			match, end := re.dfa.match(re.dict, text, m.Words[i-1:])
			if match >= 0 && end > 0 {
				end += i - 1 // translate from index in m.Words[i-1:] to index in m.Words
				m.List = append(m.List, Match{ID: int(match), Start: i - 1, End: end})

				// Continue search at end of match.
				i = end - 1 // loop will i++
				p[0] = BadWord
				continue
			}
		}
	}
	if len(m.List) == 0 {
		return nil
	}
	return m
}
<commit_msg>internal/match: return split text always from MultiLRE.Match<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Exported LRE interface.

package match

import (
	"fmt"
	"sync"
)

// An LRE is a compiled license regular expression.
//
// TODO: Move this comment somewhere non-internal later.
//
// A license regular expression (LRE) is a pattern syntax intended for
// describing large English texts such as software licenses, with minor
// allowed variations. The pattern syntax and the matching are word-based
// and case-insensitive; punctuation is ignored in the pattern and in
// the matched text.
//
// The valid LRE patterns are:
//
//	word            - a single case-insensitive word
//	__N__           - any sequence of up to N words
//	expr1 expr2     - concatenation
//	expr1 || expr2  - alternation
//	(( expr ))      - grouping
//	expr??          - zero or one instances of expr
//	//** text **//  - a comment
//
// To make patterns harder to misread in large texts:
//
//	- || must only appear inside (( ))
//	- ?? must only follow (( ))
//	- (( must be at the start of a line, preceded only by spaces
//	- )) must be at the end of a line, followed only by spaces and ??.
//
// For example:
//
//	//** https://en.wikipedia.org/wiki/Filler_text **//
//	Now is
//	((not))??
//	the time for all good
//	((men || women || people))
//	to come to the aid of their __1__.
//
type LRE struct {
	dict   *Dict
	file   string
	syntax *reSyntax
	prog   reProg

	onceDFA sync.Once
	dfa     reDFA
}

// ParseLRE parses the string s as a license regexp.
// The file name is used in error messages if non-empty.
func ParseLRE(d *Dict, file, s string) (*LRE, error) {
	syntax, err := reParse(d, s, true)
	if err != nil {
		return nil, err
	}
	prog, err := syntax.compile(nil, 0)
	if err != nil {
		return nil, err
	}
	return &LRE{dict: d, file: file, syntax: syntax, prog: prog}, nil
}

// Dict returns the Dict used by the LRE.
func (re *LRE) Dict() *Dict {
	return re.dict
}

// File returns the file name passed to ParseLRE.
func (re *LRE) File() string {
	return re.file
}

// Match reports whether text matches the license regexp.
func (re *LRE) match(text string) bool {
	re.onceDFA.Do(re.compile)
	match, _ := re.dfa.match(re.dict, text, re.dict.Split(text))
	return match >= 0
}

// compile initializes lre.dfa.
// It is invoked lazily (in Match) because most LREs end up only
// being inputs to a MultiLRE; we never need their DFAs directly.
func (re *LRE) compile() {
	re.dfa = reCompileDFA(re.prog)
}

// A MultiLRE matches multiple LREs simultaneously against a text.
// It is more efficient than matching each LRE in sequence against the text.
type MultiLRE struct {
	dict *Dict // dict shared by all LREs
	dfa  reDFA // compiled DFA for all LREs

	// start contains the two-word phrases
	// where a match can validly start,
	// to allow for faster scans over non-license text.
	start map[phrase]struct{}
}

// A phrase is a phrase of up to two words.
// The zero-word phrase is phrase{NoWord, NoWord}.
// A single-word phrase w is phrase{w, NoWord}.
type phrase [2]WordID

// NewMultiLRE returns a MultiLRE looking for the given LREs.
// All the LREs must have been parsed using the same Dict;
// if not, NewMultiLRE panics.
func NewMultiLRE(list []*LRE) (_ *MultiLRE, err error) {
	if len(list) == 0 {
		return &MultiLRE{}, nil
	}

	dict := list[0].dict
	for _, sub := range list[1:] {
		if sub.dict != dict {
			panic("MultiRE: LREs parsed with different Dicts")
		}
	}

	var progs []reProg
	for _, sub := range list {
		progs = append(progs, sub.prog)
	}

	start := make(map[phrase]struct{})
	for _, sub := range list {
		phrases := sub.syntax.leadingPhrases()
		if len(phrases) == 0 {
			return nil, fmt.Errorf("%s: no leading phrases", sub.File())
		}
		for _, p := range phrases {
			if p[0] == BadWord {
				return nil, fmt.Errorf("%s: invalid pattern: matches empty text", sub.File())
			}
			if p[0] == AnyWord {
				if p[1] == BadWord {
					return nil, fmt.Errorf("%s: invalid pattern: matches a single wildcard", sub.File())
				}
				if p[1] == AnyWord {
					return nil, fmt.Errorf("%s: invalid pattern: begins with two wildcards", sub.File())
				}
				return nil, fmt.Errorf("%s: invalid pattern: begins with wildcard phrase: __ %s", sub.File(), dict.Words()[p[1]])
			}
			if p[1] == BadWord {
				return nil, fmt.Errorf("%s: invalid pattern: matches single word %s", sub.File(), dict.Words()[p[0]])
			}
			if p[1] == AnyWord {
				return nil, fmt.Errorf("%s: invalid pattern: begins with wildcard phrase: %s __", sub.File(), dict.Words()[p[0]])
			}
			start[p] = struct{}{}
		}
	}

	prog := reCompileMulti(progs)
	dfa := reCompileDFA(prog)

	return &MultiLRE{dict, dfa, start}, nil
}

// Dict returns the Dict used by the MultiLRE.
func (re *MultiLRE) Dict() *Dict {
	return re.dict
}

// A Matches is a collection of all leftmost-longest, non-overlapping matches in text.
type Matches struct {
	Text  string  // the entire text
	Words []Word  // the text, split into Words
	List  []Match // the matches
}

// A Match records the position of a single match in a text.
type Match struct {
	ID    int // index of LRE in list passed to NewMultiLRE
	Start int // word index of start of match
	End   int // word index of end of match
}

// Match reports all leftmost-longest, non-overlapping matches in text.
// It always returns a non-nil *Matches, in order to return the split text.
// Check len(matches.List) to see whether any matches were found.
func (re *MultiLRE) Match(text string) *Matches {
	m := &Matches{
		Text:  text,
		Words: re.dict.Split(text),
	}
	p := phrase{BadWord, BadWord}
	for i := 0; i < len(m.Words); i++ {
		p[0], p[1] = p[1], m.Words[i].ID
		if _, ok := re.start[p]; ok {
			match, end := re.dfa.match(re.dict, text, m.Words[i-1:])
			if match >= 0 && end > 0 {
				end += i - 1 // translate from index in m.Words[i-1:] to index in m.Words
				m.List = append(m.List, Match{ID: int(match), Start: i - 1, End: end})

				// Continue search at end of match.
				i = end - 1 // loop will i++
				p[0] = BadWord
				continue
			}
		}
	}
	return m
}
<|endoftext|>