<commit_before>package apps

import (
	"encoding/json"
	"io"
	"net/url"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/permissions"
	"github.com/cozy/cozy-stack/pkg/prefixer"
)

// KonnManifest contains all the informations associated with an installed
// konnector.
type KonnManifest struct {
	DocRev string `json:"_rev,omitempty"`

	Name       string `json:"name"`
	NamePrefix string `json:"name_prefix,omitempty"`
	Editor     string `json:"editor"`
	Icon       string `json:"icon"`

	Type        string           `json:"type,omitempty"`
	License     string           `json:"license,omitempty"`
	Language    string           `json:"language,omitempty"`
	VendorLink  string           `json:"vendor_link"`
	Locales     *json.RawMessage `json:"locales,omitempty"`
	Langs       *json.RawMessage `json:"langs,omitempty"`
	Platforms   *json.RawMessage `json:"platforms,omitempty"`
	Categories  *json.RawMessage `json:"categories,omitempty"`
	Developer   *json.RawMessage `json:"developer,omitempty"`
	Screenshots *json.RawMessage `json:"screenshots,omitempty"`
	Tags        *json.RawMessage `json:"tags,omitempty"`

	Frequency    string           `json:"frequency,omitempty"`
	DataTypes    *json.RawMessage `json:"data_types,omitempty"`
	Doctypes     *json.RawMessage `json:"doctypes,omitempty"`
	Fields       *json.RawMessage `json:"fields,omitempty"`
	Folders      *json.RawMessage `json:"folders,omitempty"`
	Messages     *json.RawMessage `json:"messages,omitempty"`
	OAuth        *json.RawMessage `json:"oauth,omitempty"`
	TimeInterval *json.RawMessage `json:"time_interval,omitempty"`

	Parameters    *json.RawMessage `json:"parameters,omitempty"`
	Notifications Notifications    `json:"notifications"`

	// OnDeleteAccount can be used to specify a file path which will be executed
	// when an account associated with the konnector is deleted.
	OnDeleteAccount string `json:"on_delete_account,omitempty"`

	DocSlug          string          `json:"slug"`
	DocState         State           `json:"state"`
	DocSource        string          `json:"source"`
	DocVersion       string          `json:"version"`
	DocPermissions   permissions.Set `json:"permissions"`
	AvailableVersion string          `json:"available_version,omitempty"`

	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`

	Err string `json:"error,omitempty"`
	err error
}

// ID is part of the Manifest interface
func (m *KonnManifest) ID() string { return m.DocType() + "/" + m.DocSlug }

// Rev is part of the Manifest interface
func (m *KonnManifest) Rev() string { return m.DocRev }

// DocType is part of the Manifest interface
func (m *KonnManifest) DocType() string { return consts.Konnectors }

// Clone is part of the Manifest interface
func (m *KonnManifest) Clone() couchdb.Doc {
	cloned := *m

	cloned.DocPermissions = make(permissions.Set, len(m.DocPermissions))
	copy(cloned.DocPermissions, m.DocPermissions)

	cloned.Locales = cloneRawMessage(m.Locales)
	cloned.Langs = cloneRawMessage(m.Langs)
	cloned.Platforms = cloneRawMessage(m.Platforms)
	cloned.Categories = cloneRawMessage(m.Categories)
	cloned.Developer = cloneRawMessage(m.Developer)
	cloned.Screenshots = cloneRawMessage(m.Screenshots)
	cloned.Tags = cloneRawMessage(m.Tags)
	cloned.Parameters = cloneRawMessage(m.Parameters)

	cloned.DataTypes = cloneRawMessage(m.DataTypes)
	cloned.Doctypes = cloneRawMessage(m.Doctypes)
	cloned.Fields = cloneRawMessage(m.Fields)
	cloned.Folders = cloneRawMessage(m.Folders)
	cloned.Messages = cloneRawMessage(m.Messages)
	cloned.OAuth = cloneRawMessage(m.OAuth)
	cloned.TimeInterval = cloneRawMessage(m.TimeInterval)

	cloned.Notifications = make(Notifications, len(m.Notifications))
	for k, v := range m.Notifications {
		props := (&v).Clone()
		cloned.Notifications[k] = *props
	}
	return &cloned
}

// SetID is part of the Manifest interface
func (m *KonnManifest) SetID(id string) {}

// SetRev is part of the Manifest interface
func (m *KonnManifest) SetRev(rev string) { m.DocRev = rev }

// SetSource is part of the Manifest interface
func (m *KonnManifest) SetSource(src *url.URL) { m.DocSource = src.String() }

// Source is part of the Manifest interface
func (m *KonnManifest) Source() string { return m.DocSource }

// Version is part of the Manifest interface
func (m *KonnManifest) Version() string { return m.DocVersion }

// Slug is part of the Manifest interface
func (m *KonnManifest) Slug() string { return m.DocSlug }

// State is part of the Manifest interface
func (m *KonnManifest) State() State { return m.DocState }

// LastUpdate is part of the Manifest interface
func (m *KonnManifest) LastUpdate() time.Time { return m.UpdatedAt }

// SetState is part of the Manifest interface
func (m *KonnManifest) SetState(state State) { m.DocState = state }

// SetVersion is part of the Manifest interface
func (m *KonnManifest) SetVersion(version string) { m.DocVersion = version }

// SetAvailableVersion is part of the Manifest interface
func (m *KonnManifest) SetAvailableVersion(version string) { m.AvailableVersion = version }

// AppType is part of the Manifest interface
func (m *KonnManifest) AppType() AppType { return Konnector }

// Permissions is part of the Manifest interface
func (m *KonnManifest) Permissions() permissions.Set {
	return m.DocPermissions
}

// SetError is part of the Manifest interface
func (m *KonnManifest) SetError(err error) {
	m.SetState(Errored)
	m.Err = err.Error()
	m.err = err
}

// Error is part of the Manifest interface
func (m *KonnManifest) Error() error { return m.err }

// Match is part of the Manifest interface
func (m *KonnManifest) Match(field, value string) bool {
	switch field {
	case "slug":
		return m.DocSlug == value
	case "state":
		return m.DocState == State(value)
	}
	return false
}

// ReadManifest is part of the Manifest interface
func (m *KonnManifest) ReadManifest(r io.Reader, slug, sourceURL string) (Manifest, error) {
	var newManifest KonnManifest
	if err := json.NewDecoder(r).Decode(&newManifest); err != nil {
		return nil, ErrBadManifest
	}

	newManifest.SetID(m.ID())
	newManifest.SetRev(m.Rev())
	newManifest.SetState(m.State())
	newManifest.CreatedAt = m.CreatedAt
	newManifest.DocSlug = slug
	newManifest.DocSource = sourceURL
	if newManifest.Parameters == nil {
		newManifest.Parameters = m.Parameters
	}

	return &newManifest, nil
}

// Create is part of the Manifest interface
func (m *KonnManifest) Create(db prefixer.Prefixer) error {
	m.CreatedAt = time.Now()
	m.UpdatedAt = time.Now()
	if err := couchdb.CreateNamedDocWithDB(db, m); err != nil {
		return err
	}
	_, err := permissions.CreateKonnectorSet(db, m.Slug(), m.Permissions())
	return err
}

// Update is part of the Manifest interface
func (m *KonnManifest) Update(db prefixer.Prefixer) error {
	m.UpdatedAt = time.Now()
	err := couchdb.UpdateDoc(db, m)
	if err != nil {
		return err
	}
	_, err = permissions.UpdateKonnectorSet(db, m.Slug(), m.Permissions())
	return err
}

// Delete is part of the Manifest interface
func (m *KonnManifest) Delete(db prefixer.Prefixer) error {
	err := permissions.DestroyKonnector(db, m.Slug())
	if err != nil && !couchdb.IsNotFoundError(err) {
		return err
	}
	return couchdb.DeleteDoc(db, m)
}

// GetKonnectorBySlug fetch the manifest of a konnector from the database given
// a slug.
func GetKonnectorBySlug(db prefixer.Prefixer, slug string) (*KonnManifest, error) {
	if slug == "" || !slugReg.MatchString(slug) {
		return nil, ErrInvalidSlugName
	}
	man := &KonnManifest{}
	err := couchdb.GetDoc(db, consts.Konnectors, consts.Konnectors+"/"+slug, man)
	if couchdb.IsNotFoundError(err) {
		return nil, ErrNotFound
	}
	if err != nil {
		return nil, err
	}
	return man, nil
}

// GetWebappBySlugAndUpdate fetch the KonnManifest and perform an update of
// the application if necessary and if the application was installed from the
// registry.
func GetKonnectorBySlugAndUpdate(db prefixer.Prefixer, slug string, copier Copier, registries []*url.URL) (*KonnManifest, error) {
	man, err := GetKonnectorBySlug(db, slug)
	if err != nil {
		return nil, err
	}
	return DoLazyUpdate(db, man, man.AvailableVersion, copier, registries).(*KonnManifest), nil
}

// ListKonnectors returns the list of installed konnectors applications.
//
// TODO: pagination
func ListKonnectors(db prefixer.Prefixer) ([]Manifest, error) {
	var docs []*KonnManifest
	req := &couchdb.AllDocsRequest{Limit: 100}
	err := couchdb.GetAllDocs(db, consts.Konnectors, req, &docs)
	if err != nil {
		return nil, err
	}
	mans := make([]Manifest, len(docs))
	for i, m := range docs {
		mans[i] = m
	}
	return mans, nil
}

var _ Manifest = &KonnManifest{}
<commit_msg>feat: Handles agregator accounts ✨<commit_after>package apps

import (
	"encoding/json"
	"io"
	"net/url"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/permissions"
	"github.com/cozy/cozy-stack/pkg/prefixer"
)

// KonnManifest contains all the informations associated with an installed
// konnector.
type KonnManifest struct {
	DocRev string `json:"_rev,omitempty"`

	Name       string `json:"name"`
	NamePrefix string `json:"name_prefix,omitempty"`
	Editor     string `json:"editor"`
	Icon       string `json:"icon"`

	Aggregator  *json.RawMessage `json:"aggregator,omitempty"`

	Type        string           `json:"type,omitempty"`
	License     string           `json:"license,omitempty"`
	Language    string           `json:"language,omitempty"`
	VendorLink  string           `json:"vendor_link"`
	Locales     *json.RawMessage `json:"locales,omitempty"`
	Langs       *json.RawMessage `json:"langs,omitempty"`
	Platforms   *json.RawMessage `json:"platforms,omitempty"`
	Categories  *json.RawMessage `json:"categories,omitempty"`
	Developer   *json.RawMessage `json:"developer,omitempty"`
	Screenshots *json.RawMessage `json:"screenshots,omitempty"`
	Tags        *json.RawMessage `json:"tags,omitempty"`

	Frequency    string           `json:"frequency,omitempty"`
	DataTypes    *json.RawMessage `json:"data_types,omitempty"`
	Doctypes     *json.RawMessage `json:"doctypes,omitempty"`
	Fields       *json.RawMessage `json:"fields,omitempty"`
	Folders      *json.RawMessage `json:"folders,omitempty"`
	Messages     *json.RawMessage `json:"messages,omitempty"`
	OAuth        *json.RawMessage `json:"oauth,omitempty"`
	TimeInterval *json.RawMessage `json:"time_interval,omitempty"`

	Parameters    *json.RawMessage `json:"parameters,omitempty"`
	Notifications Notifications    `json:"notifications"`

	// OnDeleteAccount can be used to specify a file path which will be executed
	// when an account associated with the konnector is deleted.
	OnDeleteAccount string `json:"on_delete_account,omitempty"`

	DocSlug          string          `json:"slug"`
	DocState         State           `json:"state"`
	DocSource        string          `json:"source"`
	DocVersion       string          `json:"version"`
	DocPermissions   permissions.Set `json:"permissions"`
	AvailableVersion string          `json:"available_version,omitempty"`

	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`

	Err string `json:"error,omitempty"`
	err error
}

// ID is part of the Manifest interface
func (m *KonnManifest) ID() string { return m.DocType() + "/" + m.DocSlug }

// Rev is part of the Manifest interface
func (m *KonnManifest) Rev() string { return m.DocRev }

// DocType is part of the Manifest interface
func (m *KonnManifest) DocType() string { return consts.Konnectors }

// Clone is part of the Manifest interface
func (m *KonnManifest) Clone() couchdb.Doc {
	cloned := *m

	cloned.DocPermissions = make(permissions.Set, len(m.DocPermissions))
	copy(cloned.DocPermissions, m.DocPermissions)

	cloned.Aggregator = cloneRawMessage(m.Aggregator)

	cloned.Locales = cloneRawMessage(m.Locales)
	cloned.Langs = cloneRawMessage(m.Langs)
	cloned.Platforms = cloneRawMessage(m.Platforms)
	cloned.Categories = cloneRawMessage(m.Categories)
	cloned.Developer = cloneRawMessage(m.Developer)
	cloned.Screenshots = cloneRawMessage(m.Screenshots)
	cloned.Tags = cloneRawMessage(m.Tags)
	cloned.Parameters = cloneRawMessage(m.Parameters)

	cloned.DataTypes = cloneRawMessage(m.DataTypes)
	cloned.Doctypes = cloneRawMessage(m.Doctypes)
	cloned.Fields = cloneRawMessage(m.Fields)
	cloned.Folders = cloneRawMessage(m.Folders)
	cloned.Messages = cloneRawMessage(m.Messages)
	cloned.OAuth = cloneRawMessage(m.OAuth)
	cloned.TimeInterval = cloneRawMessage(m.TimeInterval)

	cloned.Notifications = make(Notifications, len(m.Notifications))
	for k, v := range m.Notifications {
		props := (&v).Clone()
		cloned.Notifications[k] = *props
	}
	return &cloned
}

// SetID is part of the Manifest interface
func (m *KonnManifest) SetID(id string) {}

// SetRev is part of the Manifest interface
func (m *KonnManifest) SetRev(rev string) { m.DocRev = rev }

// SetSource is part of the Manifest interface
func (m *KonnManifest) SetSource(src *url.URL) { m.DocSource = src.String() }

// Source is part of the Manifest interface
func (m *KonnManifest) Source() string { return m.DocSource }

// Version is part of the Manifest interface
func (m *KonnManifest) Version() string { return m.DocVersion }

// Slug is part of the Manifest interface
func (m *KonnManifest) Slug() string { return m.DocSlug }

// State is part of the Manifest interface
func (m *KonnManifest) State() State { return m.DocState }

// LastUpdate is part of the Manifest interface
func (m *KonnManifest) LastUpdate() time.Time { return m.UpdatedAt }

// SetState is part of the Manifest interface
func (m *KonnManifest) SetState(state State) { m.DocState = state }

// SetVersion is part of the Manifest interface
func (m *KonnManifest) SetVersion(version string) { m.DocVersion = version }

// SetAvailableVersion is part of the Manifest interface
func (m *KonnManifest) SetAvailableVersion(version string) { m.AvailableVersion = version }

// AppType is part of the Manifest interface
func (m *KonnManifest) AppType() AppType { return Konnector }

// Permissions is part of the Manifest interface
func (m *KonnManifest) Permissions() permissions.Set {
	return m.DocPermissions
}

// SetError is part of the Manifest interface
func (m *KonnManifest) SetError(err error) {
	m.SetState(Errored)
	m.Err = err.Error()
	m.err = err
}

// Error is part of the Manifest interface
func (m *KonnManifest) Error() error { return m.err }

// Match is part of the Manifest interface
func (m *KonnManifest) Match(field, value string) bool {
	switch field {
	case "slug":
		return m.DocSlug == value
	case "state":
		return m.DocState == State(value)
	}
	return false
}

// ReadManifest is part of the Manifest interface
func (m *KonnManifest) ReadManifest(r io.Reader, slug, sourceURL string) (Manifest, error) {
	var newManifest KonnManifest
	if err := json.NewDecoder(r).Decode(&newManifest); err != nil {
		return nil, ErrBadManifest
	}

	newManifest.SetID(m.ID())
	newManifest.SetRev(m.Rev())
	newManifest.SetState(m.State())
	newManifest.CreatedAt = m.CreatedAt
	newManifest.DocSlug = slug
	newManifest.DocSource = sourceURL
	if newManifest.Parameters == nil {
		newManifest.Parameters = m.Parameters
	}

	return &newManifest, nil
}

// Create is part of the Manifest interface
func (m *KonnManifest) Create(db prefixer.Prefixer) error {
	m.CreatedAt = time.Now()
	m.UpdatedAt = time.Now()
	if err := couchdb.CreateNamedDocWithDB(db, m); err != nil {
		return err
	}
	_, err := permissions.CreateKonnectorSet(db, m.Slug(), m.Permissions())
	return err
}

// Update is part of the Manifest interface
func (m *KonnManifest) Update(db prefixer.Prefixer) error {
	m.UpdatedAt = time.Now()
	err := couchdb.UpdateDoc(db, m)
	if err != nil {
		return err
	}
	_, err = permissions.UpdateKonnectorSet(db, m.Slug(), m.Permissions())
	return err
}

// Delete is part of the Manifest interface
func (m *KonnManifest) Delete(db prefixer.Prefixer) error {
	err := permissions.DestroyKonnector(db, m.Slug())
	if err != nil && !couchdb.IsNotFoundError(err) {
		return err
	}
	return couchdb.DeleteDoc(db, m)
}

// GetKonnectorBySlug fetch the manifest of a konnector from the database given
// a slug.
func GetKonnectorBySlug(db prefixer.Prefixer, slug string) (*KonnManifest, error) {
	if slug == "" || !slugReg.MatchString(slug) {
		return nil, ErrInvalidSlugName
	}
	man := &KonnManifest{}
	err := couchdb.GetDoc(db, consts.Konnectors, consts.Konnectors+"/"+slug, man)
	if couchdb.IsNotFoundError(err) {
		return nil, ErrNotFound
	}
	if err != nil {
		return nil, err
	}
	return man, nil
}

// GetWebappBySlugAndUpdate fetch the KonnManifest and perform an update of
// the application if necessary and if the application was installed from the
// registry.
func GetKonnectorBySlugAndUpdate(db prefixer.Prefixer, slug string, copier Copier, registries []*url.URL) (*KonnManifest, error) {
	man, err := GetKonnectorBySlug(db, slug)
	if err != nil {
		return nil, err
	}
	return DoLazyUpdate(db, man, man.AvailableVersion, copier, registries).(*KonnManifest), nil
}

// ListKonnectors returns the list of installed konnectors applications.
//
// TODO: pagination
func ListKonnectors(db prefixer.Prefixer) ([]Manifest, error) {
	var docs []*KonnManifest
	req := &couchdb.AllDocsRequest{Limit: 100}
	err := couchdb.GetAllDocs(db, consts.Konnectors, req, &docs)
	if err != nil {
		return nil, err
	}
	mans := make([]Manifest, len(docs))
	for i, m := range docs {
		mans[i] = m
	}
	return mans, nil
}

var _ Manifest = &KonnManifest{}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"io/ioutil"
	"os"
	"sync"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
)

var stdoutMutex = &sync.Mutex{}

func TestMetricsNormalDeployment(t *testing.T) {
	// Run deploy normally, should see METRICS=true
	testDeploy(t, false, true, true)
}

func TestMetricsNormalDeploymentNoMetricsFlagSet(t *testing.T) {
	// Run deploy normally, should see METRICS=true
	testDeploy(t, false, false, false)
}

func TestMetricsDevDeployment(t *testing.T) {
	// Run deploy w dev flag, should see METRICS=false
	testDeploy(t, true, true, false)
}

func TestMetricsDevDeploymentNoMetricsFlagSet(t *testing.T) {
	// Run deploy w dev flag, should see METRICS=false
	testDeploy(t, true, false, false)
}

func TestPortForwardError(t *testing.T) {
	os.Setenv("PACHD_ADDRESS", "localhost:30650")
	c := tu.Cmd("pachctl", "version", "--timeout=1ns", "--no-port-forwarding")
	var errMsg bytes.Buffer
	c.Stdout = ioutil.Discard
	c.Stderr = &errMsg
	err := c.Run()
	require.YesError(t, err) // 1ns should prevent even local connections
	require.Matches(t, "port-forward", errMsg.String())
}

func TestNoPortError(t *testing.T) {
	os.Setenv("PACHD_ADDRESS", "127.127.127.0")
	c := tu.Cmd("pachctl", "version", "--timeout=1ns", "--no-port-forwarding")
	var errMsg bytes.Buffer
	c.Stdout = ioutil.Discard
	c.Stderr = &errMsg
	err := c.Run()
	require.YesError(t, err) // 1ns should prevent even local connections
	require.Matches(t, "does not seem to be host:port", errMsg.String())
}

func TestWeirdPortError(t *testing.T) {
	os.Setenv("PACHD_ADDRESS", "localhost:30560")
	c := tu.Cmd("pachctl", "version", "--timeout=1ns", "--no-port-forwarding")
	var errMsg bytes.Buffer
	c.Stdout = ioutil.Discard
	c.Stderr = &errMsg
	err := c.Run()
	require.YesError(t, err) // 1ns should prevent even local connections
	require.Matches(t, "30650", errMsg.String())
}

func testDeploy(t *testing.T, devFlag bool, metrics bool, expectedEnvValue bool) {
	//t.Parallel()
	//stdoutMutex.Lock()
	//defer stdoutMutex.Unlock()

	//// Setup user config prior to test
	//// So that stdout only contains JSON no warnings
	//_, err := config.Read()
	//require.NoError(t, err)

	//old := os.Stdout
	//r, w, _ := os.Pipe()
	//os.Stdout = w

	//os.Args = []string{
	//"deploy",
	//"local",
	//"--dry-run",
	//fmt.Sprintf("-d=%v", devFlag),
	//}
	//err = deploycmds.DeployCmd(&metrics).Execute()
	//require.NoError(t, err)
	//require.NoError(t, w.Close())
	//// restore stdout
	//os.Stdout = old

	//decoder := json.NewDecoder(r)
	//foundPachdManifest := false
	//// Loop through generated manifest until we find a
	//// ReplicationController (limit of 100 makes sure test
	//// fails quickly if there is no RC)
	//for i := 0; i < 100; i++ {
	//var manifest *extensions.Deployment
	//err = decoder.Decode(&manifest)
	//if err == io.EOF {
	//break
	//}
	//if err != nil {
	//// Not a replication controller
	//continue
	//}
	//require.NoError(t, err)
	//if manifest.ObjectMeta.Name == "pachd" && manifest.Kind == "Deployment" {
	//foundPachdManifest = true
	//expectedMetricEnvVar := api.EnvVar{
	//Name:  "METRICS",
	//Value: fmt.Sprintf("%v", expectedEnvValue),
	//}
	//var env []interface{}
	//require.Equal(t, 1, len(manifest.Spec.Template.Spec.Containers))
	//for _, value := range manifest.Spec.Template.Spec.Containers[0].Env {
	//env = append(env, value)
	//}
	//require.OneOfEquals(t, interface{}(expectedMetricEnvVar), env)
	//}
	//}
	//require.Equal(t, true, foundPachdManifest)
}
<commit_msg>Killed old test<commit_after>package cmd

import (
	"bytes"
	"io/ioutil"
	"os"
	"sync"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
)

var stdoutMutex = &sync.Mutex{}

func TestMetricsNormalDeployment(t *testing.T) {
	// Run deploy normally, should see METRICS=true
	testDeploy(t, false, true, true)
}

func TestMetricsNormalDeploymentNoMetricsFlagSet(t *testing.T) {
	// Run deploy normally, should see METRICS=true
	testDeploy(t, false, false, false)
}

func TestMetricsDevDeployment(t *testing.T) {
	// Run deploy w dev flag, should see METRICS=false
	testDeploy(t, true, true, false)
}

func TestMetricsDevDeploymentNoMetricsFlagSet(t *testing.T) {
	// Run deploy w dev flag, should see METRICS=false
	testDeploy(t, true, false, false)
}

func TestPortForwardError(t *testing.T) {
	os.Setenv("PACHD_ADDRESS", "localhost:30650")
	c := tu.Cmd("pachctl", "version", "--timeout=1ns", "--no-port-forwarding")
	var errMsg bytes.Buffer
	c.Stdout = ioutil.Discard
	c.Stderr = &errMsg
	err := c.Run()
	require.YesError(t, err) // 1ns should prevent even local connections
	require.Matches(t, "port-forward", errMsg.String())
}

func TestNoPortError(t *testing.T) {
	os.Setenv("PACHD_ADDRESS", "127.127.127.0")
	c := tu.Cmd("pachctl", "version", "--timeout=1ns", "--no-port-forwarding")
	var errMsg bytes.Buffer
	c.Stdout = ioutil.Discard
	c.Stderr = &errMsg
	err := c.Run()
	require.YesError(t, err) // 1ns should prevent even local connections
	require.Matches(t, "does not seem to be host:port", errMsg.String())
}

func TestWeirdPortError(t *testing.T) {
	os.Setenv("PACHD_ADDRESS", "localhost:30560")
	c := tu.Cmd("pachctl", "version", "--timeout=1ns", "--no-port-forwarding")
	var errMsg bytes.Buffer
	c.Stdout = ioutil.Discard
	c.Stderr = &errMsg
	err := c.Run()
	require.YesError(t, err) // 1ns should prevent even local connections
	require.Matches(t, "30650", errMsg.String())
}
<|endoftext|>


<commit_before>package worker

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim"
	"github.com/pivotal-golang/lager"
)

const ephemeralPropertyName = "concourse:ephemeral"
const volumePropertyName = "concourse:volumes"
const volumeMountsPropertyName = "concourse:volume-mounts"

type releasable interface {
	Release(*time.Duration)
}

type gardenContainerSpecFactory struct {
	logger             lager.Logger
	baggageclaimClient baggageclaim.Client
	imageFetcher       ImageFetcher
	releaseAfterCreate []releasable
	db                 GardenWorkerDB
}

func NewGardenContainerSpecFactory(logger lager.Logger, baggageclaimClient baggageclaim.Client, imageFetcher ImageFetcher, db GardenWorkerDB) gardenContainerSpecFactory {
	return gardenContainerSpecFactory{
		logger:             logger,
		baggageclaimClient: baggageclaimClient,
		imageFetcher:       imageFetcher,
		releaseAfterCreate: []releasable{},
		db:                 db,
	}
}

func (factory *gardenContainerSpecFactory) BuildContainerSpec(
	spec ContainerSpec,
	resourceTypes []atc.WorkerResourceType,
	workerTags atc.Tags,
	cancel <-chan os.Signal,
	delegate ImageFetchingDelegate,
	id Identifier,
	metadata Metadata,
	workerClient Client,
	customTypes atc.ResourceTypes,
) (garden.ContainerSpec, error) {
	var volumeHandles []string
	var user string

	imageResourceConfig, hasImageResource := spec.ImageResource()
	var gardenSpec garden.ContainerSpec
	if hasImageResource {
		image, err := factory.imageFetcher.FetchImage(
			factory.logger,
			imageResourceConfig,
			cancel,
			id,
			metadata,
			delegate,
			workerClient,
			workerTags,
			customTypes,
		)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		imageVolume := image.Volume()

		volumeHandles = append(volumeHandles, imageVolume.Handle())
		factory.releaseAfterCreate = append(factory.releaseAfterCreate, image)
		user = image.Metadata().User

		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
			RootFSPath: path.Join(imageVolume.Path(), "rootfs"),
			Env:        image.Metadata().Env,
		}
	} else {
		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
		}
	}

	volumeMountPaths := map[baggageclaim.Volume]string{}
	var volumeMounts []VolumeMount

dance:
	switch s := spec.(type) {
	case ResourceTypeContainerSpec:
		if len(s.Mounts) > 0 && s.Cache.Volume != nil {
			return garden.ContainerSpec{}, errors.New("a container may not have mounts and a cache")
		}

		gardenSpec.Privileged = true
		gardenSpec.Env = append(gardenSpec.Env, s.Env...)

		if s.Ephemeral {
			gardenSpec.Properties[ephemeralPropertyName] = "true"
		}

		if s.Cache.Volume != nil && s.Cache.MountPath != "" {
			volumeHandles = append(volumeHandles, s.Cache.Volume.Handle())
			volumeMountPaths[s.Cache.Volume] = s.Cache.MountPath
		}

		volumeMounts = s.Mounts

		if s.ImageResourcePointer == nil {
			for _, t := range resourceTypes {
				if t.Type == s.Type {
					gardenSpec.RootFSPath = t.Image
					break dance
				}
			}

			return garden.ContainerSpec{}, ErrUnsupportedResourceType
		}
	case TaskContainerSpec:
		if s.ImageResourcePointer == nil {
			gardenSpec.RootFSPath = s.Image
		}

		gardenSpec.Privileged = s.Privileged

		volumeMounts = s.Inputs

		for _, mount := range s.Outputs {
			volume := mount.Volume
			volumeHandles = append(volumeHandles, volume.Handle())
			volumeMountPaths[volume] = mount.MountPath
		}
	default:
		return garden.ContainerSpec{}, fmt.Errorf("unknown container spec type: %T (%#v)", s, s)
	}

	newVolumeHandles, newVolumeMountPaths, err := factory.createVolumes(gardenSpec, volumeMounts)
	if err != nil {
		return garden.ContainerSpec{}, err
	}

	for _, h := range newVolumeHandles {
		volumeHandles = append(volumeHandles, h)
	}

	for volume, mountPath := range newVolumeMountPaths {
		volumeMountPaths[volume] = mountPath
	}

	for volume, mount := range volumeMountPaths {
		gardenSpec.BindMounts = append(gardenSpec.BindMounts, garden.BindMount{
			SrcPath: volume.Path(),
			DstPath: mount,
			Mode:    garden.BindMountModeRW,
		})
	}

	if len(volumeHandles) > 0 {
		volumesJSON, err := json.Marshal(volumeHandles)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumePropertyName] = string(volumesJSON)

		volumeHandleMounts := map[string]string{}

		for k, v := range volumeMountPaths {
			volumeHandleMounts[k.Handle()] = v
		}

		mountsJSON, err := json.Marshal(volumeHandleMounts)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumeMountsPropertyName] = string(mountsJSON)
	}

	gardenSpec.Properties["user"] = user

	return gardenSpec, nil
}

func (factory *gardenContainerSpecFactory) ReleaseVolumes() {
	for _, releasable := range factory.releaseAfterCreate {
		releasable.Release(nil)
	}
}

func (factory *gardenContainerSpecFactory) createVolumes(containerSpec garden.ContainerSpec, mounts []VolumeMount) ([]string, map[baggageclaim.Volume]string, error) {
	var volumeHandles []string
	volumeMountPaths := map[baggageclaim.Volume]string{}

	for _, mount := range mounts {
		cowVolume, err := factory.baggageclaimClient.CreateVolume(factory.logger, baggageclaim.VolumeSpec{
			Strategy: baggageclaim.COWStrategy{
				Parent: mount.Volume,
			},
			Privileged: containerSpec.Privileged,
			TTL:        VolumeTTL,
		})
		if err != nil {
			return []string{}, map[baggageclaim.Volume]string{}, err
		}

		factory.releaseAfterCreate = append(factory.releaseAfterCreate, cowVolume)

		err = factory.db.InsertCOWVolume(mount.Volume.Handle(), cowVolume.Handle(), VolumeTTL)
		if err != nil {
			return []string{}, map[baggageclaim.Volume]string{}, err
		}

		volumeHandles = append(volumeHandles, cowVolume.Handle())
		volumeMountPaths[cowVolume] = mount.MountPath

		factory.logger.Info("created-cow-volume", lager.Data{
			"original-volume-handle": mount.Volume.Handle(),
			"cow-volume-handle":      cowVolume.Handle(),
		})
	}

	return volumeHandles, volumeMountPaths, nil
}
<commit_msg>whitespace<commit_after>package worker

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim"
	"github.com/pivotal-golang/lager"
)

const ephemeralPropertyName = "concourse:ephemeral"
const volumePropertyName = "concourse:volumes"
const volumeMountsPropertyName = "concourse:volume-mounts"

type releasable interface {
	Release(*time.Duration)
}

type gardenContainerSpecFactory struct {
	logger             lager.Logger
	baggageclaimClient baggageclaim.Client
	imageFetcher       ImageFetcher
	releaseAfterCreate []releasable
	db                 GardenWorkerDB
}

func NewGardenContainerSpecFactory(logger lager.Logger, baggageclaimClient baggageclaim.Client, imageFetcher ImageFetcher, db GardenWorkerDB) gardenContainerSpecFactory {
	return gardenContainerSpecFactory{
		logger:             logger,
		baggageclaimClient: baggageclaimClient,
		imageFetcher:       imageFetcher,
		releaseAfterCreate: []releasable{},
		db:                 db,
	}
}

func (factory *gardenContainerSpecFactory) BuildContainerSpec(
	spec ContainerSpec,
	resourceTypes []atc.WorkerResourceType,
	workerTags atc.Tags,
	cancel <-chan os.Signal,
	delegate ImageFetchingDelegate,
	id Identifier,
	metadata Metadata,
	workerClient Client,
	customTypes atc.ResourceTypes,
) (garden.ContainerSpec, error) {
	var volumeHandles []string
	var user string

	imageResourceConfig, hasImageResource := spec.ImageResource()
	var gardenSpec garden.ContainerSpec
	if hasImageResource {
		image, err := factory.imageFetcher.FetchImage(
			factory.logger,
			imageResourceConfig,
			cancel,
			id,
			metadata,
			delegate,
			workerClient,
			workerTags,
			customTypes,
		)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		imageVolume := image.Volume()
		volumeHandles = append(volumeHandles, imageVolume.Handle())
		factory.releaseAfterCreate = append(factory.releaseAfterCreate, image)
		user = image.Metadata().User

		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
			RootFSPath: path.Join(imageVolume.Path(), "rootfs"),
			Env:        image.Metadata().Env,
		}
	} else {
		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
		}
	}

	volumeMountPaths := map[baggageclaim.Volume]string{}
	var volumeMounts []VolumeMount

dance:
	switch s := spec.(type) {
	case ResourceTypeContainerSpec:
		if len(s.Mounts) > 0 && s.Cache.Volume != nil {
			return garden.ContainerSpec{}, errors.New("a container may not have mounts and a cache")
		}

		gardenSpec.Privileged = true
		gardenSpec.Env = append(gardenSpec.Env, s.Env...)

		if s.Ephemeral {
			gardenSpec.Properties[ephemeralPropertyName] = "true"
		}

		if s.Cache.Volume != nil && s.Cache.MountPath != "" {
			volumeHandles = append(volumeHandles, s.Cache.Volume.Handle())
			volumeMountPaths[s.Cache.Volume] = s.Cache.MountPath
		}

		volumeMounts = s.Mounts

		if s.ImageResourcePointer == nil {
			for _, t := range resourceTypes {
				if t.Type == s.Type {
					gardenSpec.RootFSPath = t.Image
					break dance
				}
			}

			return garden.ContainerSpec{}, ErrUnsupportedResourceType
		}
	case TaskContainerSpec:
		if s.ImageResourcePointer == nil {
			gardenSpec.RootFSPath = s.Image
		}

		gardenSpec.Privileged = s.Privileged

		volumeMounts = s.Inputs

		for _, mount := range s.Outputs {
			volume := mount.Volume
			volumeHandles = append(volumeHandles, volume.Handle())
			volumeMountPaths[volume] = mount.MountPath
		}
	default:
		return garden.ContainerSpec{}, fmt.Errorf("unknown container spec type: %T (%#v)", s, s)
	}

	newVolumeHandles, newVolumeMountPaths, err := factory.createVolumes(gardenSpec, volumeMounts)
	if err != nil {
		return garden.ContainerSpec{}, err
	}

	for _, h := range newVolumeHandles {
		volumeHandles = append(volumeHandles, h)
	}

	for volume, mountPath := range newVolumeMountPaths {
		volumeMountPaths[volume] = mountPath
	}

	for volume, mount := range volumeMountPaths {
		gardenSpec.BindMounts = append(gardenSpec.BindMounts, garden.BindMount{
			SrcPath: volume.Path(),
			DstPath: mount,
			Mode:    garden.BindMountModeRW,
		})
	}

	if len(volumeHandles) > 0 {
		volumesJSON, err := json.Marshal(volumeHandles)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumePropertyName] = string(volumesJSON)

		volumeHandleMounts := map[string]string{}

		for k, v := range volumeMountPaths {
			volumeHandleMounts[k.Handle()] = v
		}

		mountsJSON, err := json.Marshal(volumeHandleMounts)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumeMountsPropertyName] = string(mountsJSON)
	}

	gardenSpec.Properties["user"] = user

	return gardenSpec, nil
}

func (factory *gardenContainerSpecFactory) ReleaseVolumes() {
	for _, releasable := range factory.releaseAfterCreate {
		releasable.Release(nil)
	}
}

func (factory *gardenContainerSpecFactory) createVolumes(containerSpec garden.ContainerSpec, mounts []VolumeMount) ([]string, map[baggageclaim.Volume]string, error) {
	var volumeHandles []string
	volumeMountPaths := map[baggageclaim.Volume]string{}

	for _, mount := range mounts {
		cowVolume, err := factory.baggageclaimClient.CreateVolume(factory.logger, baggageclaim.VolumeSpec{
			Strategy: baggageclaim.COWStrategy{
				Parent: mount.Volume,
			},
			Privileged: containerSpec.Privileged,
			TTL:        VolumeTTL,
		})
		if err != nil {
			return []string{}, map[baggageclaim.Volume]string{}, err
		}

		factory.releaseAfterCreate = append(factory.releaseAfterCreate, cowVolume)

		err = factory.db.InsertCOWVolume(mount.Volume.Handle(), cowVolume.Handle(), VolumeTTL)
		if err != nil {
			return []string{}, map[baggageclaim.Volume]string{}, err
		}

		volumeHandles = append(volumeHandles, cowVolume.Handle())
		volumeMountPaths[cowVolume] = mount.MountPath

		factory.logger.Info("created-cow-volume", lager.Data{
			"original-volume-handle": mount.Volume.Handle(),
			"cow-volume-handle":      cowVolume.Handle(),
		})
	}

	return volumeHandles, volumeMountPaths, nil
}
<|endoftext|>


<commit_before>package agent

import (
	"net/http"

	"github.com/Dataman-Cloud/swan-janitor/src"
	"github.com/Dataman-Cloud/swan-resolver/nameserver"
	"github.com/Dataman-Cloud/swan/src/apiserver/metrics"
	"github.com/Dataman-Cloud/swan/src/config"
	"github.com/Dataman-Cloud/swan/src/event"
	"github.com/Dataman-Cloud/swan/src/types"
	"github.com/Dataman-Cloud/swan/src/utils"

	"github.com/Sirupsen/logrus"
	restful "github.com/emicklei/go-restful"
)

type AgentApi struct {
	agent *Agent
}

func (api *AgentApi) Register(container *restful.Container) {
	ws := new(restful.WebService)
	ws.
		ApiVersion(config.API_PREFIX).
		Path(config.API_PREFIX + "/agent").
		Doc("agent server api").
		Consumes(restful.MIME_JSON).
		Produces("*/*")

	ws.Route(ws.POST("/init").To(metrics.InstrumentRouteFunc("POST", "Agent Init", api.Init)).
		Doc("Init Agent").
		Operation("InitAgent").
		Returns(201, "OK", nil).
		Returns(400, "BadRequest", nil).
		Reads([]nameserver.RecordGeneratorChangeEvent{}).
		Writes(nil))

	ws.Route(ws.POST("/resolver/event").To(metrics.InstrumentRouteFunc("POST", "Resolver Event", api.ResolverEventHandler)).
		Doc("Resolver Event Handler").
		Operation("Handle Resolver Event").
		Returns(201, "OK", nil).
		Returns(400, "BadRequest", nil).
		Reads(event.Event{}).
		Writes(nil))

	ws.Route(ws.POST("/janitor/event").To(metrics.InstrumentRouteFunc("POST", "Janitor Event", api.JanitorEventHandler)).
		Doc("Janitor Event Handler").
		Operation("Handler Janitor Event").
		Returns(201, "OK", nil).
		Returns(400, "BadRequest", nil).
		Reads(event.Event{}).
		Writes(nil))

	container.Add(ws)
}

func (api *AgentApi) Init(request *restful.Request, response *restful.Response) {
	var events []*event.Event
	if err := request.ReadEntity(&events); err != nil {
		logrus.Errorf("init agent failed. Error: %s", err.Error())
		response.WriteError(http.StatusBadRequest, err)
		return
	}

	go api.dispenseEvents(events)

	response.WriteHeaderAndEntity(http.StatusCreated, nil)
	return
}

func (api *AgentApi) dispenseEvents(events []*event.Event) {
	for _, taskEvent := range events {
		// taskEvent.Payload is type of interface{} can not auto unmarshal
		eventInfo := &types.TaskInfoEvent{}
		err := utils.FillStruct(eventInfo, taskEvent.Payload.(map[string]interface{}))
		if err != nil {
			logrus.Warnf("agent got resolver event %+v", err)
			return
		}

		taskEvent.Payload = eventInfo

		resolverEvent, err := event.BuildResolverEvent(taskEvent)
		if err == nil {
			logrus.Infof("agent got resolver event %+v", resolverEvent)
			api.agent.resolver.RecordGeneratorChangeChan() <- resolverEvent
		} else {
			logrus.Errorf("Build resolver event got error: %s", err.Error())
		}

		janitorEvent, err := event.BuildJanitorEvent(taskEvent)
		if err == nil {
			logrus.Infof("agent got resolver event %+v", janitorEvent)
			api.agent.janitorServer.SwanEventChan() <- janitorEvent
		} else {
			logrus.Errorf("Build janitor event got error: %s", err.Error())
		}
	}
}

func (api *AgentApi) ResolverEventHandler(request *restful.Request, response *restful.Response) {
	var resolverEvent nameserver.RecordGeneratorChangeEvent
	if err := request.ReadEntity(&resolverEvent); err != nil {
		logrus.Errorf("handle resolver event failed. Error: %s", err.Error())
		response.WriteError(http.StatusBadRequest, err)
		return
	}

	logrus.Infof("agent got resolver event %+v", resolverEvent)
	api.agent.resolver.RecordGeneratorChangeChan() <- &resolverEvent

	response.WriteHeaderAndEntity(http.StatusCreated, nil)
	return
}

func (api *AgentApi) JanitorEventHandler(request *restful.Request, response *restful.Response) {
	var janitorEvent janitor.TargetChangeEvent
	if err := request.ReadEntity(&janitorEvent); err != nil {
		logrus.Errorf("handle janitor event failed. Error: %s", err.Error())
		response.WriteError(http.StatusBadRequest, err)
		return
	}

	logrus.Infof("agent got janitor event %+v", janitorEvent)
	api.agent.janitorServer.SwanEventChan() <- &janitorEvent

	response.WriteHeaderAndEntity(http.StatusCreated, nil)
	return

}
<commit_msg>rename function name from dispenseEvent to watchEvents<commit_after>package agent

import (
	"net/http"

	"github.com/Dataman-Cloud/swan-janitor/src"
	"github.com/Dataman-Cloud/swan-resolver/nameserver"
	"github.com/Dataman-Cloud/swan/src/apiserver/metrics"
	"github.com/Dataman-Cloud/swan/src/config"
	"github.com/Dataman-Cloud/swan/src/event"
	"github.com/Dataman-Cloud/swan/src/types"
	"github.com/Dataman-Cloud/swan/src/utils"

	"github.com/Sirupsen/logrus"
	restful "github.com/emicklei/go-restful"
)

type AgentApi struct {
	agent *Agent
}

func (api *AgentApi) Register(container *restful.Container) {
	ws := new(restful.WebService)
	ws.
		ApiVersion(config.API_PREFIX).
		Path(config.API_PREFIX + "/agent").
		Doc("agent server api").
		Consumes(restful.MIME_JSON).
		Produces("*/*")

	ws.Route(ws.POST("/init").To(metrics.InstrumentRouteFunc("POST", "Agent Init", api.Init)).
		Doc("Init Agent").
		Operation("InitAgent").
		Returns(201, "OK", nil).
		Returns(400, "BadRequest", nil).
		Reads([]nameserver.RecordGeneratorChangeEvent{}).
		Writes(nil))

	ws.Route(ws.POST("/resolver/event").To(metrics.InstrumentRouteFunc("POST", "Resolver Event", api.ResolverEventHandler)).
		Doc("Resolver Event Handler").
		Operation("Handle Resolver Event").
		Returns(201, "OK", nil).
		Returns(400, "BadRequest", nil).
		Reads(event.Event{}).
		Writes(nil))

	ws.Route(ws.POST("/janitor/event").To(metrics.InstrumentRouteFunc("POST", "Janitor Event", api.JanitorEventHandler)).
		Doc("Janitor Event Handler").
		Operation("Handler Janitor Event").
		Returns(201, "OK", nil).
		Returns(400, "BadRequest", nil).
		Reads(event.Event{}).
		Writes(nil))

	container.Add(ws)
}

func (api *AgentApi) Init(request *restful.Request, response *restful.Response) {
	var events []*event.Event
	if err := request.ReadEntity(&events); err != nil {
		logrus.Errorf("init agent failed. Error: %s", err.Error())
		response.WriteError(http.StatusBadRequest, err)
		return
	}

	go api.watchEvents(events)

	response.WriteHeaderAndEntity(http.StatusCreated, nil)
	return
}

func (api *AgentApi) watchEvents(events []*event.Event) {
	for _, taskEvent := range events {
		// taskEvent.Payload is type of interface{} can not auto unmarshal
		eventInfo := &types.TaskInfoEvent{}
		err := utils.FillStruct(eventInfo, taskEvent.Payload.(map[string]interface{}))
		if err != nil {
			logrus.Warnf("agent got resolver event %+v", err)
			return
		}

		taskEvent.Payload = eventInfo

		resolverEvent, err := event.BuildResolverEvent(taskEvent)
		if err == nil {
			logrus.Infof("agent got resolver event %+v", resolverEvent)
			api.agent.resolver.RecordGeneratorChangeChan() <- resolverEvent
		} else {
			logrus.Errorf("Build resolver event got error: %s", err.Error())
		}

		janitorEvent, err := event.BuildJanitorEvent(taskEvent)
		if err == nil {
			logrus.Infof("agent got resolver event %+v", janitorEvent)
			api.agent.janitorServer.SwanEventChan() <- janitorEvent
		} else {
			logrus.Errorf("Build janitor event got error: %s", err.Error())
		}
	}
}

func (api *AgentApi) ResolverEventHandler(request *restful.Request, response *restful.Response) {
	var resolverEvent nameserver.RecordGeneratorChangeEvent
	if err := request.ReadEntity(&resolverEvent); err != nil {
		logrus.Errorf("handle resolver event failed. Error: %s", err.Error())
		response.WriteError(http.StatusBadRequest, err)
		return
	}

	logrus.Infof("agent got resolver event %+v", resolverEvent)
	api.agent.resolver.RecordGeneratorChangeChan() <- &resolverEvent

	response.WriteHeaderAndEntity(http.StatusCreated, nil)
	return
}

func (api *AgentApi) JanitorEventHandler(request *restful.Request, response *restful.Response) {
	var janitorEvent janitor.TargetChangeEvent
	if err := request.ReadEntity(&janitorEvent); err != nil {
		logrus.Errorf("handle janitor event failed. Error: %s", err.Error())
		response.WriteError(http.StatusBadRequest, err)
		return
	}

	logrus.Infof("agent got janitor event %+v", janitorEvent)
	api.agent.janitorServer.SwanEventChan() <- &janitorEvent

	response.WriteHeaderAndEntity(http.StatusCreated, nil)
	return

}
<|endoftext|>


<commit_before>package expand_url

import (
	"fmt"

	"os"

	"github.com/codegangsta/cli"
	"github.com/syou6162/go-active-learning/lib/cache"
	"github.com/syou6162/go-active-learning/lib/db"
	"github.com/syou6162/go-active-learning/lib/util/file"
)

func doExpandURL(c *cli.Context) error {
	inputFilename := c.String("input-filename")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "expand-url")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	conn, err := db.CreateDBConnection()
	if err != nil {
		return err
	}
	defer conn.Close()

	cache, err := cache.NewCache()
	if err != nil {
		return err
	}
	defer cache.Close()

	examples, err := file.ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	cache.AttachMetaData(examples)

	for _, e := range examples {
		_, err = db.InsertOrUpdateExample(conn, e)
		if err != nil {
			return err
		}

		url := e.Url
		if e.FinalUrl != "" {
			url = e.FinalUrl
		}
		fmt.Println(fmt.Sprintf("%s\t%d", url, e.Label))
	}

	return nil
}

var CommandExpandURL = cli.Command{
	Name:  "expand-url",
	Usage: "Expand shortened url",
	Description: `
Expand shortened url.
`,
	Action: doExpandURL,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
	},
}
<commit_msg>unused<commit_after>package expand_url

import (
	"fmt"

	"github.com/codegangsta/cli"
	"github.com/syou6162/go-active-learning/lib/cache"
	"github.com/syou6162/go-active-learning/lib/db"
	"github.com/syou6162/go-active-learning/lib/util/file"
)

func doExpandURL(c *cli.Context) error {
	inputFilename := c.String("input-filename")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "expand-url")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	conn, err := db.CreateDBConnection()
	if err != nil {
		return err
	}
	defer conn.Close()

	cache, err := cache.NewCache()
	if err != nil {
		return err
	}
	defer cache.Close()

	examples, err := file.ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	cache.AttachMetaData(examples)

	for _, e := range examples {
		_, err = db.InsertOrUpdateExample(conn, e)
		if err != nil {
			return err
		}

		url := e.Url
		if e.FinalUrl != "" {
			url = e.FinalUrl
		}
		fmt.Println(fmt.Sprintf("%s\t%d", url, e.Label))
	}

	return nil
}

var CommandExpandURL = cli.Command{
	Name:  "expand-url",
	Usage: "Expand shortened url",
	Description: `
Expand shortened url.
`,
	Action: doExpandURL,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
	},
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"os/exec"
	"time"

	. "./equtils"

	"./historystorage"
	"./explorepolicy"

	"github.com/mitchellh/cli"
)

func createCmd(scriptPath, workingDirPath, materialsDirPath string) *exec.Cmd {
	cmd := exec.Command("sh", "-c", scriptPath)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd.Env = append(cmd.Env, "EQ_WORKING_DIR="+workingDirPath)
	cmd.Env = append(cmd.Env, "EQ_MATERIALS_DIR="+materialsDirPath)

	return cmd
}

func run(args []string) {
	if len(args) != 1 {
		fmt.Printf("specify <storage dir path>\n")
		os.Exit(1)
	}

	storagePath := args[0]
	confPath := storagePath + "/" + historystorage.StorageConfigPath

	vcfg, err := ParseConfigFile(confPath)
	if err != nil {
		fmt.Printf("failed to parse config file %s: %s\n", confPath, err)
		os.Exit(1)
	}

	storage := historystorage.New(vcfg.GetString("storageType"), storagePath)
	storage.Init()

	policy := explorepolicy.CreatePolicy(vcfg.GetString("explorePolicy"))
	if policy == nil {
		fmt.Printf("invalid policy name: %s", vcfg.GetString("explorePolicy"))
		os.Exit(1)
	}
	policy.Init(storage, vcfg.GetStringMap("explorePolicyParam"))

	nextDir := storage.CreateNewWorkingDir()
	InitLog(nextDir + "/earthquake.log")
	AddLogTee(os.Stdout)

	end := make(chan interface{})
	newTraceCh := make(chan *SingleTrace)

	go orchestrate(end, policy, newTraceCh)

	materialsDir := storagePath + "/" + storageMaterialsPath
	runScriptPath := materialsDir + "/" + vcfg.GetString("run")

	cleanScriptPath := ""
	if vcfg.GetString("clean") != "" {
		cleanScriptPath = materialsDir + "/" + vcfg.GetString("clean")
	}

	validateScriptPath := ""
	if vcfg.GetString("validate") != "" {
		validateScriptPath = materialsDir + "/" + vcfg.GetString("validate")
	}

	runCmd := createCmd(runScriptPath, nextDir, materialsDir)

	startTime := time.Now()

	rerr := runCmd.Run()
	if rerr != nil {
		fmt.Printf("failed to execute run script %s: %s\n", runScriptPath, rerr)
		os.Exit(1)
	}

	end <- true
	newTrace := <-newTraceCh

	endTime := time.Now()
	requiredTime := endTime.Sub(startTime)

	storage.RecordNewTrace(newTrace)

	if validateScriptPath != "" {
		validateCmd := createCmd(validateScriptPath, nextDir, materialsDir)

		rerr = validateCmd.Run()
		if rerr != nil {
			fmt.Printf("validation failed: %s\n", rerr)
			// TODO: detailed check of error
			// e.g. handle a case like permission denied, noent, etc
			storage.RecordResult(false, requiredTime)
		} else {
			fmt.Printf("validation succeed\n")
			storage.RecordResult(true, requiredTime)
		}
	}

	if cleanScriptPath != "" {
		cleanCmd := createCmd(cleanScriptPath, nextDir, materialsDir)

		rerr = cleanCmd.Run()
		if rerr != nil {
			fmt.Printf("failed to execute clean script %s: %s\n", cleanScriptPath, rerr)
			os.Exit(1)
		}
	}
}

type runCmd struct {
}

func (cmd runCmd) Help() string {
	return "run help (todo)"
}

func (cmd runCmd) Run(args []string) int {
	run(args)
	return 0
}

func (cmd runCmd) Synopsis() string {
	return "run subcommand"
}

func runCommandFactory() (cli.Command, error) {
	return runCmd{}, nil
}
<commit_msg>run: set rlimit for bunch of connections<commit_after>// Copyright (C) 2015 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	. "./equtils"

	"./explorepolicy"
	"./historystorage"

	"github.com/mitchellh/cli"
)

func init() {
	var rLimit syscall.Rlimit
	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
	if err != nil {
		fmt.Println("Error Getting Rlimit ", err)
	}

	rLimit.Max = 999999
	rLimit.Cur = 999999
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
	if err != nil {
		fmt.Println("Error Setting Rlimit ", err)
	}
}

func createCmd(scriptPath, workingDirPath, materialsDirPath string) *exec.Cmd {
	cmd := exec.Command("sh", "-c", scriptPath)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd.Env = append(cmd.Env, "EQ_WORKING_DIR="+workingDirPath)
	cmd.Env = append(cmd.Env, "EQ_MATERIALS_DIR="+materialsDirPath)

	return cmd
}

func run(args []string) {
	if len(args) != 1 {
		fmt.Printf("specify <storage dir path>\n")
		os.Exit(1)
	}

	storagePath := args[0]
	confPath := storagePath + "/" + historystorage.StorageConfigPath

	vcfg, err := ParseConfigFile(confPath)
	if err != nil {
		fmt.Printf("failed to parse config file %s: %s\n", confPath, err)
		os.Exit(1)
	}

	storage := historystorage.New(vcfg.GetString("storageType"), storagePath)
	storage.Init()

	policy := explorepolicy.CreatePolicy(vcfg.GetString("explorePolicy"))
	if policy == nil {
		fmt.Printf("invalid policy name: %s", vcfg.GetString("explorePolicy"))
		os.Exit(1)
	}
	policy.Init(storage, vcfg.GetStringMap("explorePolicyParam"))

	nextDir := storage.CreateNewWorkingDir()
	InitLog(nextDir + "/earthquake.log")
	AddLogTee(os.Stdout)

	end := make(chan interface{})
	newTraceCh := make(chan *SingleTrace)

	go orchestrate(end, policy, newTraceCh)

	materialsDir := storagePath + "/" + storageMaterialsPath
	runScriptPath := materialsDir + "/" + vcfg.GetString("run")

	cleanScriptPath := ""
	if vcfg.GetString("clean") != "" {
		cleanScriptPath = materialsDir + "/" + vcfg.GetString("clean")
	}

	validateScriptPath := ""
	if vcfg.GetString("validate") != "" {
		validateScriptPath = materialsDir + "/" + vcfg.GetString("validate")
	}

	runCmd := createCmd(runScriptPath, nextDir, materialsDir)

	startTime := time.Now()

	rerr := runCmd.Run()
	if rerr != nil {
		fmt.Printf("failed to execute run script %s: %s\n", runScriptPath, rerr)
		os.Exit(1)
	}

	end <- true
	newTrace := <-newTraceCh

	endTime := time.Now()
	requiredTime := endTime.Sub(startTime)

	storage.RecordNewTrace(newTrace)

	if validateScriptPath != "" {
		validateCmd := createCmd(validateScriptPath, nextDir, materialsDir)

		rerr = validateCmd.Run()
		if rerr != nil {
			fmt.Printf("validation failed: %s\n", rerr)
			// TODO: detailed check of error
			// e.g. handle a case like permission denied, noent, etc
			storage.RecordResult(false, requiredTime)
		} else {
			fmt.Printf("validation succeed\n")
			storage.RecordResult(true, requiredTime)
		}
	}

	if cleanScriptPath != "" {
		cleanCmd := createCmd(cleanScriptPath, nextDir, materialsDir)

		rerr = cleanCmd.Run()
		if rerr != nil {
			fmt.Printf("failed to execute clean script %s: %s\n", cleanScriptPath, rerr)
			os.Exit(1)
		}
	}
}

type runCmd struct {
}

func (cmd runCmd) Help() string {
	return "run help (todo)"
}

func (cmd runCmd) Run(args []string) int {
	run(args)
	return 0
}

func (cmd runCmd) Synopsis() string {
	return "run subcommand"
}

func runCommandFactory() (cli.Command, error) {
	return runCmd{}, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"errors"
	"os"
	"os/signal"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/concurrency"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
)

// NewLockCommand returns the cobra command for "lock".
func NewLockCommand() *cobra.Command {
	c := &cobra.Command{
		Use:   "lock <lockname>",
		Short: "Acquires a named lock",
		Run:   lockCommandFunc,
	}
	return c
}

func lockCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, errors.New("lock takes one lock name argument."))
	}
	c := mustClientFromCmd(cmd)
	if err := lockUntilSignal(c, args[0]); err != nil {
		ExitWithError(ExitError, err)
	}
}

func lockUntilSignal(c *clientv3.Client, lockname string) error {
	s, err := concurrency.NewSession(c)
	if err != nil {
		return err
	}

	m := concurrency.NewMutex(s, lockname)
	ctx, cancel := context.WithCancel(context.TODO())

	// unlock in case of ordinary shutdown
	donec := make(chan struct{})
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt, os.Kill)
	go func() {
		<-sigc
		cancel()
		close(donec)
	}()

	s, serr := concurrency.NewSession(c)
	if serr != nil {
		return serr
	}

	if err := m.Lock(ctx); err != nil {
		return err
	}

	k, kerr := c.Get(ctx, m.Key())
	if kerr != nil {
		return kerr
	}
	if len(k.Kvs) == 0 {
		return errors.New("lock lost on init")
	}

	display.Get(*k)

	select {
	case <-donec:
		return m.Unlock(context.TODO())
	case <-s.Done():
	}

	return errors.New("session expired")
}
<commit_msg>etcdctl: remove superfluous session in lock command<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"errors"
	"os"
	"os/signal"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/concurrency"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
)

// NewLockCommand returns the cobra command for "lock".
func NewLockCommand() *cobra.Command {
	c := &cobra.Command{
		Use:   "lock <lockname>",
		Short: "Acquires a named lock",
		Run:   lockCommandFunc,
	}
	return c
}

func lockCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, errors.New("lock takes one lock name argument."))
	}
	c := mustClientFromCmd(cmd)
	if err := lockUntilSignal(c, args[0]); err != nil {
		ExitWithError(ExitError, err)
	}
}

func lockUntilSignal(c *clientv3.Client, lockname string) error {
	s, err := concurrency.NewSession(c)
	if err != nil {
		return err
	}

	m := concurrency.NewMutex(s, lockname)
	ctx, cancel := context.WithCancel(context.TODO())

	// unlock in case of ordinary shutdown
	donec := make(chan struct{})
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt, os.Kill)
	go func() {
		<-sigc
		cancel()
		close(donec)
	}()

	if err := m.Lock(ctx); err != nil {
		return err
	}

	k, kerr := c.Get(ctx, m.Key())
	if kerr != nil {
		return kerr
	}
	if len(k.Kvs) == 0 {
		return errors.New("lock lost on init")
	}

	display.Get(*k)

	select {
	case <-donec:
		return m.Unlock(context.TODO())
	case <-s.Done():
	}

	return errors.New("session expired")
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/zlib"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idSize    = 8 // should be between 6 and 256
	indexTmpl = "index.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl     = flag.String("u", "http://localhost:8080", "URL of the site")
	listen      = flag.String("l", "localhost:8080", "Host and port to listen to")
	dataDir     = flag.String("d", "data", "Directory to store all the pastes in")
	lifeTimeStr = flag.String("t", "12h", "Lifetime of the pastes (units: s,m,h)")
	maxSizeStr  = flag.String("s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")

	lifeTime time.Duration
	maxSize  ByteSize

	validId       = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.FormatInt(idSize, 10) + "}$")
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
)

func pathId(id string) string {
	return path.Join(id[0:2], id[2:4], id[4:])
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func randomId() string {
	s := make([]byte, idSize)
	var offset uint = 0
MainLoop:
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				break MainLoop
			}
			r >>= 8
		}
	}
	return string(s)
}

func endLife(path string) {
	err := os.Remove(path)
	if err == nil {
		log.Printf("Removed paste: %s", path)
	} else {
		log.Printf("Could not end the life of %s: %s", path, err)
		programDeath(path, 2*time.Minute)
	}
}

func programDeath(path string, after time.Duration) {
	timer := time.NewTimer(after)
	go func() {
		<-timer.C
		endLife(path)
	}()
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		var id, pastePath string
		id = r.URL.Path[1:]
		if len(id) == 0 {
			indexTemplate.Execute(w, *siteUrl)
			return
		}
		if !validId.MatchString(id) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id = strings.ToLower(id)
		pastePath = pathId(id)
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		compReader, err := zlib.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		io.Copy(w, compReader)
		compReader.Close()
		pasteFile.Close()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var id, pastePath string
		for {
			id = randomId()
			pastePath = pathId(id)
			if _, err := os.Stat(pastePath); os.IsNotExist(err) {
				break
			}
		}
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		var content string
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		programDeath(pastePath, lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		compWriter := zlib.NewWriter(pasteFile)
		b, err := io.WriteString(compWriter, content)
		compWriter.Close()
		pasteFile.Close()
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		log.Printf("Created a new paste: %s (%s)", pastePath, writtenSize)
		fmt.Fprintf(w, "%s/%s\n", *siteUrl, id)
	}
}

func walkFunc(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.IsDir() {
		return nil
	}
	deathTime := info.ModTime().Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go endLife(path)
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	log.Printf("Recovered paste %s has %s left", path, lifeLeft)
	programDeath(path, lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if lifeTime, err = time.ParseDuration(*lifeTimeStr); err != nil {
		log.Fatalf("Invalid lifetime '%s': %s", *lifeTimeStr, err)
	}
	if maxSize, err = parseByteSize(*maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", *maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if err = os.MkdirAll(*dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", *dataDir, err)
	}
	if err = os.Chdir(*dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", *dataDir, err)
	}
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", *dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", *siteUrl)
	log.Printf("listen   = %s", *listen)
	log.Printf("dataDir  = %s", *dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe(*listen, nil))
}
<commit_msg>Eventually give up on finding unused ids<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/zlib"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idSize    = 8 // should be between 6 and 256
	indexTmpl = "index.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl     = flag.String("u", "http://localhost:8080", "URL of the site")
	listen      = flag.String("l", "localhost:8080", "Host and port to listen to")
	dataDir     = flag.String("d", "data", "Directory to store all the pastes in")
	lifeTimeStr = flag.String("t", "12h", "Lifetime of the pastes (units: s,m,h)")
	maxSizeStr  = flag.String("s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")

	lifeTime time.Duration
	maxSize  ByteSize

	validId       = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.FormatInt(idSize, 10) + "}$")
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
)

func pathId(id string) string {
	return path.Join(id[0:2], id[2:4], id[4:])
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func randomId() string {
	s := make([]byte, idSize)
	var offset uint = 0
MainLoop:
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				break MainLoop
			}
			r >>= 8
		}
	}
	return string(s)
}

func endLife(path string) {
	err := os.Remove(path)
	if err == nil {
		log.Printf("Removed paste: %s", path)
	} else {
		log.Printf("Could not end the life of %s: %s", path, err)
		programDeath(path, 2*time.Minute)
	}
}

func programDeath(path string, after time.Duration) {
	timer := time.NewTimer(after)
	go func() {
		<-timer.C
		endLife(path)
	}()
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		var id, pastePath string
		id = r.URL.Path[1:]
		if len(id) == 0 {
			indexTemplate.Execute(w, *siteUrl)
			return
		}
		if !validId.MatchString(id) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id = strings.ToLower(id)
		pastePath = pathId(id)
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		compReader, err := zlib.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		io.Copy(w, compReader)
		compReader.Close()
		pasteFile.Close()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var id, pastePath string
		found := false
		for i := 0; i < 10; i++ {
			id = randomId()
			pastePath = pathId(id)
			if _, err := os.Stat(pastePath); os.IsNotExist(err) {
				found = true
				break
			}
		}
		if !found {
			log.Printf("Gave up trying to find an unused random id.")
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		var content string
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		programDeath(pastePath, lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		compWriter := zlib.NewWriter(pasteFile)
		b, err := io.WriteString(compWriter, content)
		compWriter.Close()
		pasteFile.Close()
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		log.Printf("Created a new paste: %s (%s)", pastePath, writtenSize)
		fmt.Fprintf(w, "%s/%s\n", *siteUrl, id)
	}
}

func walkFunc(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.IsDir() {
		return nil
	}
	deathTime := info.ModTime().Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go endLife(path)
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	log.Printf("Recovered paste %s has %s left", path, lifeLeft)
	programDeath(path, lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if lifeTime, err = time.ParseDuration(*lifeTimeStr); err != nil {
		log.Fatalf("Invalid lifetime '%s': %s", *lifeTimeStr, err)
	}
	if maxSize, err = parseByteSize(*maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", *maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if err = os.MkdirAll(*dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", *dataDir, err)
	}
	if err = os.Chdir(*dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", *dataDir, err)
	}
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", *dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", *siteUrl)
	log.Printf("listen   = %s", *listen)
	log.Printf("dataDir  = %s", *dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe(*listen, nil))
}
<|endoftext|>


<commit_before>package datastore

import (
	"log"
	"os"
)

// OpenFile opens and returns a file handler for a CSV file
func OpenFile(inputFilePath string) (*os.File, error) {
	_, err := os.Stat(inputFilePath)
	if err != nil {
		log.Fatal("Input error: ", err)
	}

	file, err := os.Open(inputFilePath)
	if err != nil {
		return nil, err
	}

	return file, err
}
<commit_msg>Removes OpenFile form datastore<commit_after><|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"github.com/keybase/kbfs/tlf"
	"golang.org/x/net/context"
)

// BlockOpsStandard implements the BlockOps interface by relaying
// requests to the block server.
type BlockOpsStandard struct {
	config  Config
	queue   *blockRetrievalQueue
	workers []*blockRetrievalWorker
}

var _ BlockOps = (*BlockOpsStandard)(nil)

func NewBlockOpsStandard(config Config) *BlockOpsStandard {
	bops := &BlockOpsStandard{
		config:  config,
		queue:   newBlockRetrievalQueue(defaultBlockRetrievalWorkerQueueSize),
		workers: make([]*blockRetrievalWorker, 0, defaultBlockRetrievalWorkerQueueSize),
	}
	bg := &realBlockGetter{config: config}
	for i := 0; i < defaultBlockRetrievalWorkerQueueSize; i++ {
		bops.workers = append(bops.workers, newBlockRetrievalWorker(bg, bops.queue))
	}
	return bops
}

// Get implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Get(ctx context.Context, kmd KeyMetadata,
	blockPtr BlockPointer, block Block) error {
	errCh := b.queue.Request(ctx, defaultOnDemandRequestPriority, kmd, blockPtr, block)
	return <-errCh
}

// Ready implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Ready(ctx context.Context, kmd KeyMetadata,
	block Block) (id BlockID, plainSize int, readyBlockData ReadyBlockData,
	err error) {
	defer func() {
		if err != nil {
			id = BlockID{}
			plainSize = 0
			readyBlockData = ReadyBlockData{}
		}
	}()

	crypto := b.config.Crypto()

	tlfCryptKey, err := b.config.KeyManager().
		GetTLFCryptKeyForEncryption(ctx, kmd)
	if err != nil {
		return
	}

	// New server key half for the block.
	serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
	if err != nil {
		return
	}

	blockKey, err := crypto.UnmaskBlockCryptKey(serverHalf, tlfCryptKey)
	if err != nil {
		return
	}

	plainSize, encryptedBlock, err := crypto.EncryptBlock(block, blockKey)
	if err != nil {
		return
	}

	buf, err := b.config.Codec().Encode(encryptedBlock)
	if err != nil {
		return
	}

	readyBlockData = ReadyBlockData{
		buf:        buf,
		serverHalf: serverHalf,
	}

	encodedSize := readyBlockData.GetEncodedSize()
	if encodedSize < plainSize {
		err = TooLowByteCountError{
			ExpectedMinByteCount: plainSize,
			ByteCount:            encodedSize,
		}
		return
	}

	id, err = crypto.MakePermanentBlockID(buf)
	if err != nil {
		return
	}

	// Cache the encoded size.
	block.SetEncodedSize(uint32(encodedSize))

	return
}

// Delete implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Delete(ctx context.Context, tlfID tlf.ID,
	ptrs []BlockPointer) (liveCounts map[BlockID]int, err error) {
	contexts := make(map[BlockID][]BlockContext)
	for _, ptr := range ptrs {
		contexts[ptr.ID] = append(contexts[ptr.ID], ptr.BlockContext)
	}
	return b.config.BlockServer().RemoveBlockReferences(ctx, tlfID, contexts)
}

// Archive implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Archive(ctx context.Context, tlfID tlf.ID,
	ptrs []BlockPointer) error {
	contexts := make(map[BlockID][]BlockContext)
	for _, ptr := range ptrs {
		contexts[ptr.ID] = append(contexts[ptr.ID], ptr.BlockContext)
	}

	return b.config.BlockServer().ArchiveBlockReferences(ctx, tlfID, contexts)
}

// Shutdown implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Shutdown() {
	b.queue.Shutdown()
	for _, w := range b.workers {
		w.Shutdown()
	}
}
<commit_msg>Fix bugs from rebase<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"github.com/keybase/kbfs/tlf"
	"golang.org/x/net/context"
)

// BlockOpsStandard implements the BlockOps interface by relaying
// requests to the block server.
type BlockOpsStandard struct {
	config  Config
	queue   *blockRetrievalQueue
	workers []*blockRetrievalWorker
}

var _ BlockOps = (*BlockOpsStandard)(nil)

// NewBlockOpsStandard creates a new BlockOpsStandard
func NewBlockOpsStandard(config Config) *BlockOpsStandard {
	bops := &BlockOpsStandard{
		config:  config,
		queue:   newBlockRetrievalQueue(defaultBlockRetrievalWorkerQueueSize, config.Codec()),
		workers: make([]*blockRetrievalWorker, 0, defaultBlockRetrievalWorkerQueueSize),
	}
	bg := &realBlockGetter{config: config}
	for i := 0; i < defaultBlockRetrievalWorkerQueueSize; i++ {
		bops.workers = append(bops.workers, newBlockRetrievalWorker(bg, bops.queue))
	}
	return bops
}

// Get implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Get(ctx context.Context, kmd KeyMetadata,
	blockPtr BlockPointer, block Block) error {
	errCh := b.queue.Request(ctx, defaultOnDemandRequestPriority, kmd, blockPtr, block)
	return <-errCh
}

// Ready implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Ready(ctx context.Context, kmd KeyMetadata,
	block Block) (id BlockID, plainSize int, readyBlockData ReadyBlockData,
	err error) {
	defer func() {
		if err != nil {
			id = BlockID{}
			plainSize = 0
			readyBlockData = ReadyBlockData{}
		}
	}()

	crypto := b.config.Crypto()

	tlfCryptKey, err := b.config.KeyManager().
		GetTLFCryptKeyForEncryption(ctx, kmd)
	if err != nil {
		return
	}

	// New server key half for the block.
	serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
	if err != nil {
		return
	}

	blockKey, err := crypto.UnmaskBlockCryptKey(serverHalf, tlfCryptKey)
	if err != nil {
		return
	}

	plainSize, encryptedBlock, err := crypto.EncryptBlock(block, blockKey)
	if err != nil {
		return
	}

	buf, err := b.config.Codec().Encode(encryptedBlock)
	if err != nil {
		return
	}

	readyBlockData = ReadyBlockData{
		buf:        buf,
		serverHalf: serverHalf,
	}

	encodedSize := readyBlockData.GetEncodedSize()
	if encodedSize < plainSize {
		err = TooLowByteCountError{
			ExpectedMinByteCount: plainSize,
			ByteCount:            encodedSize,
		}
		return
	}

	id, err = crypto.MakePermanentBlockID(buf)
	if err != nil {
		return
	}

	// Cache the encoded size.
	block.SetEncodedSize(uint32(encodedSize))

	return
}

// Delete implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Delete(ctx context.Context, tlfID tlf.ID,
	ptrs []BlockPointer) (liveCounts map[BlockID]int, err error) {
	contexts := make(map[BlockID][]BlockContext)
	for _, ptr := range ptrs {
		contexts[ptr.ID] = append(contexts[ptr.ID], ptr.BlockContext)
	}
	return b.config.BlockServer().RemoveBlockReferences(ctx, tlfID, contexts)
}

// Archive implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Archive(ctx context.Context, tlfID tlf.ID,
	ptrs []BlockPointer) error {
	contexts := make(map[BlockID][]BlockContext)
	for _, ptr := range ptrs {
		contexts[ptr.ID] = append(contexts[ptr.ID], ptr.BlockContext)
	}

	return b.config.BlockServer().ArchiveBlockReferences(ctx, tlfID, contexts)
}

// Shutdown implements the BlockOps interface for BlockOpsStandard.
func (b *BlockOpsStandard) Shutdown() {
	b.queue.Shutdown()
	for _, w := range b.workers {
		w.Shutdown()
	}
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

                 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbutil

import (
	"database/sql"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/cloudflare/cfssl/log"
	"github.com/go-sql-driver/mysql"
	"github.com/hyperledger/fabric-ca/lib/tls"
	"github.com/jmoiron/sqlx"
)

// NewUserRegistrySQLLite3 returns a pointer to a sqlite database
func NewUserRegistrySQLLite3(datasource string) (*sqlx.DB, bool, error) {
	log.Debugf("Using sqlite database, connect to database in home (%s) directory", datasource)

	datasource = filepath.Join(datasource)
	var exists bool

	if datasource != "" {
		// Check if database exists if not create it and bootstrap it based on the config file
		if _, err := os.Stat(datasource); err != nil {
			if os.IsNotExist(err) {
				log.Debugf("Database (%s) does not exist", datasource)
				exists = false
				err2 := createSQLiteDBTables(datasource)
				if err2 != nil {
					return nil, false, err2
				}
			} else {
				log.Debug("Database (%s) exists", datasource)
				exists = true
			}
		}
	}

	db, err := sqlx.Open("sqlite3", datasource)
	if err != nil {
		return nil, false, err
	}

	log.Debug("Successfully opened sqlite3 DB")

	return db, exists, nil
}

func createSQLiteDBTables(datasource string) error {
	log.Debug("Creating SQLite Database...")
	log.Debug("Database location: ", datasource)
	db, err := sqlx.Open("sqlite3", datasource)
	if err != nil {
		return fmt.Errorf("Failed to open database: %s", err)
	}

	log.Debug("Creating tables...")
	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(64), token bytea, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER,  max_enrollments INTEGER)"); err != nil {
		return err
	}
	log.Debug("Created users table")

	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(64), prekey VARCHAR(64))"); err != nil {
		return err
	}
	log.Debug("Created affiliation table")

	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(64), serial_number bytea NOT NULL, authority_key_identifier bytea NOT NULL, ca_label bytea, status bytea NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem bytea NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
		return err
	}
	log.Debug("Created certificates table")

	return nil
}

// NewUserRegistryPostgres opens a connecton to a postgres database
func NewUserRegistryPostgres(datasource string, clientTLSConfig *tls.ClientTLSConfig) (*sqlx.DB, bool, error) {
	log.Debugf("Using postgres database, connecting to database...")

	var exists bool
	dbName := getDBName(datasource)
	log.Debug("Database Name: ", dbName)

	if strings.Contains(dbName, "-") || strings.HasSuffix(dbName, ".db") {
		return nil, false, fmt.Errorf("Database name %s cannot contain any '-' or end with '.db'", dbName)
	}

	connStr := getConnStr(datasource)

	if clientTLSConfig.Enabled {
		if len(clientTLSConfig.CertFilesList) > 0 {
			root := clientTLSConfig.CertFilesList[0]
			connStr = fmt.Sprintf("%s sslrootcert=%s", connStr, root)
		}

		cert := clientTLSConfig.Client.CertFile
		key := clientTLSConfig.Client.KeyFile
		connStr = fmt.Sprintf("%s sslcert=%s sslkey=%s", connStr, cert, key)
	}

	log.Debug("Connection String: ", connStr)

	db, err := sqlx.Open("postgres", connStr)
	if err != nil {
		return nil, false, fmt.Errorf("Failed to open database: %s", err)
	}

	err = db.Ping()
	if err != nil {
		log.Errorf("Failed to connect to Postgres database [error: %s]", err)
		return nil, false, err
	}

	// Check if database exists
	r, err2 := db.Exec("SELECT * FROM pg_catalog.pg_database where datname=$1", dbName)
	if err2 != nil {
		return nil, false, fmt.Errorf("Failed to query 'pg_database' table: %s", err2)
	}

	found, _ := r.RowsAffected()
	if found == 0 {
		log.Debugf("Database (%s) does not exist", dbName)
		exists = false
		connStr = connStr + " dbname=" + dbName
		err = createPostgresDBTables(connStr, dbName, db)
		if err != nil {
			return nil, false, err
		}
	} else {
		log.Debugf("Database (%s) exists", dbName)
		exists = true
	}

	db, err = sqlx.Open("postgres", datasource)
	if err != nil {
		return nil, false, err
	}

	return db, exists, nil
}

// createPostgresDB creates postgres database
func createPostgresDBTables(datasource string, dbName string, db *sqlx.DB) error {
	log.Debugf("Creating Postgres Database (%s)...", dbName)
	query := "CREATE DATABASE " + dbName
	_, err := db.Exec(query)
	if err != nil {
		return fmt.Errorf("Failed to create Postgres database: %s", err)
	}

	database, err := sqlx.Open("postgres", datasource)
	if err != nil {
		log.Errorf("Failed to open database (%s)", dbName)
	}

	log.Debug("Creating Tables...")
	if _, err := database.Exec("CREATE TABLE users (id VARCHAR(64), token bytea, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER,  max_enrollments INTEGER)"); err != nil {
		log.Errorf("Error creating users table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE affiliations (name VARCHAR(64), prekey VARCHAR(64))"); err != nil {
		log.Errorf("Error creating affiliations table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE certificates (id VARCHAR(64), serial_number bytea NOT NULL, authority_key_identifier bytea NOT NULL, ca_label bytea, status bytea NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem bytea NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
		log.Errorf("Error creating certificates table [error: %s] ", err)
		return err
	}
	return nil
}

// NewUserRegistryMySQL opens a connecton to a postgres database
func NewUserRegistryMySQL(datasource string, clientTLSConfig *tls.ClientTLSConfig) (*sqlx.DB, bool, error) {
	log.Debugf("Using MySQL database, connecting to database...")

	var exists bool
	dbName := getDBName(datasource)
	log.Debug("Database Name: ", dbName)

	re := regexp.MustCompile(`\/([a-zA-z]+)`)
	connStr := re.ReplaceAllString(datasource, "/")

	if clientTLSConfig.Enabled {
		tlsConfig, err := tls.GetClientTLSConfig(clientTLSConfig)
		if err != nil {
			log.Errorf("Failed to create TLS configuration [error: %s]", err)
		}

		mysql.RegisterTLSConfig("custom", tlsConfig)
	}

	log.Debug("Connection String: ", connStr)
	db, err := sqlx.Open("mysql", connStr)
	if err != nil {
		return nil, false, fmt.Errorf("Failed to open database: %s", err)
	}

	err = db.Ping()
	if err != nil {
		log.Errorf("Failed to connect to MySQL database [error: %s]", err)
	}

	// Check if database exists
	var name string
	err = db.QueryRow("SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = ?", dbName).Scan(&name)
	if err != nil {
		if err == sql.ErrNoRows {
			log.Debugf("Database (%s) does not exist", dbName)
			exists = false
		} else {
			return nil, false, fmt.Errorf("Failed to query 'INFORMATION_SCHEMA.SCHEMATA table: %s", err)
		}
	}

	if name == "" {
		createMySQLTables(datasource, dbName, db)
	} else {
		log.Debugf("Database (%s) exists", dbName)
		exists = true
	}

	db, err = sqlx.Open("mysql", datasource)
	if err != nil {
		return nil, false, err
	}

	return db, exists, nil
}

func createMySQLTables(datasource string, dbName string, db *sqlx.DB) error {
	log.Debugf("Creating MySQL Database (%s)...", dbName)

	_, err := db.Exec("CREATE DATABASE " + dbName)
	if err != nil {
		panic(err)
	}

	database, err := sqlx.Open("mysql", datasource)
	if err != nil {
		log.Errorf("Failed to open database (%s), err: %s", dbName, err)
	}
	log.Debug("Creating Tables...")
	if _, err := database.Exec("CREATE TABLE users (id VARCHAR(64) NOT NULL, token blob, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER, max_enrollments INTEGER, PRIMARY KEY (id))"); err != nil {
		log.Errorf("Error creating users table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE affiliations (name VARCHAR(64), prekey VARCHAR(64))"); err != nil {
		log.Errorf("Error creating affiliations table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE certificates (id VARCHAR(64), serial_number varbinary(128) NOT NULL, authority_key_identifier varbinary(128) NOT NULL, ca_label varbinary(128), status varbinary(128) NOT NULL, reason int, expiry timestamp DEFAULT '1970-01-01 00:00:01', revoked_at timestamp DEFAULT '1970-01-01 00:00:01', pem varbinary(4096) NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
		log.Errorf("Error creating certificates table [error: %s] ", err)
		return err
	}

	return nil
}

// GetDBName gets database name from connection string
func getDBName(datasource string) string {
	var dbName string
	datasource = strings.ToLower(datasource)

	re := regexp.MustCompile(`(?:\/([^\/?]+))|(?:dbname=([^\s]+))`)
	getName := re.FindStringSubmatch(datasource)
	if getName != nil {
		dbName = getName[1]
		if dbName == "" {
			dbName = getName[2]
		}
	}

	return dbName
}

// GetConnStr gets connection string without database
func getConnStr(datasource string) string {
	re := regexp.MustCompile(`(dbname=)([^\s]+)`)
	connStr := re.ReplaceAllString(datasource, "")
	return connStr
}
<commit_msg>Case sensitivity for MySQL users table<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

                 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbutil

import (
	"database/sql"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/cloudflare/cfssl/log"
	"github.com/go-sql-driver/mysql"
	"github.com/hyperledger/fabric-ca/lib/tls"
	"github.com/jmoiron/sqlx"
)

// NewUserRegistrySQLLite3 returns a pointer to a sqlite database
func NewUserRegistrySQLLite3(datasource string) (*sqlx.DB, bool, error) {
	log.Debugf("Using sqlite database, connect to database in home (%s) directory", datasource)

	datasource = filepath.Join(datasource)
	var exists bool

	if datasource != "" {
		// Check if database exists if not create it and bootstrap it based on the config file
		if _, err := os.Stat(datasource); err != nil {
			if os.IsNotExist(err) {
				log.Debugf("Database (%s) does not exist", datasource)
				exists = false
				err2 := createSQLiteDBTables(datasource)
				if err2 != nil {
					return nil, false, err2
				}
			} else {
				log.Debug("Database (%s) exists", datasource)
				exists = true
			}
		}
	}

	db, err := sqlx.Open("sqlite3", datasource)
	if err != nil {
		return nil, false, err
	}

	log.Debug("Successfully opened sqlite3 DB")

	return db, exists, nil
}

func createSQLiteDBTables(datasource string) error {
	log.Debug("Creating SQLite Database...")
	log.Debug("Database location: ", datasource)
	db, err := sqlx.Open("sqlite3", datasource)
	if err != nil {
		return fmt.Errorf("Failed to open database: %s", err)
	}

	log.Debug("Creating tables...")
	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(64), token bytea, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER,  max_enrollments INTEGER)"); err != nil {
		return err
	}
	log.Debug("Created users table")

	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(64), prekey VARCHAR(64))"); err != nil {
		return err
	}
	log.Debug("Created affiliation table")

	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(64), serial_number bytea NOT NULL, authority_key_identifier bytea NOT NULL, ca_label bytea, status bytea NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem bytea NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
		return err
	}
	log.Debug("Created certificates table")

	return nil
}

// NewUserRegistryPostgres opens a connecton to a postgres database
func NewUserRegistryPostgres(datasource string, clientTLSConfig *tls.ClientTLSConfig) (*sqlx.DB, bool, error) {
	log.Debugf("Using postgres database, connecting to database...")

	var exists bool
	dbName := getDBName(datasource)
	log.Debug("Database Name: ", dbName)

	if strings.Contains(dbName, "-") || strings.HasSuffix(dbName, ".db") {
		return nil, false, fmt.Errorf("Database name %s cannot contain any '-' or end with '.db'", dbName)
	}

	connStr := getConnStr(datasource)

	if clientTLSConfig.Enabled {
		if len(clientTLSConfig.CertFilesList) > 0 {
			root := clientTLSConfig.CertFilesList[0]
			connStr = fmt.Sprintf("%s sslrootcert=%s", connStr, root)
		}

		cert := clientTLSConfig.Client.CertFile
		key := clientTLSConfig.Client.KeyFile
		connStr = fmt.Sprintf("%s sslcert=%s sslkey=%s", connStr, cert, key)
	}

	log.Debug("Connection String: ", connStr)

	db, err := sqlx.Open("postgres", connStr)
	if err != nil {
		return nil, false, fmt.Errorf("Failed to open database: %s", err)
	}

	err = db.Ping()
	if err != nil {
		log.Errorf("Failed to connect to Postgres database [error: %s]", err)
		return nil, false, err
	}

	// Check if database exists
	r, err2 := db.Exec("SELECT * FROM pg_catalog.pg_database where datname=$1", dbName)
	if err2 != nil {
		return nil, false, fmt.Errorf("Failed to query 'pg_database' table: %s", err2)
	}

	found, _ := r.RowsAffected()
	if found == 0 {
		log.Debugf("Database (%s) does not exist", dbName)
		exists = false
		connStr = connStr + " dbname=" + dbName
		err = createPostgresDBTables(connStr, dbName, db)
		if err != nil {
			return nil, false, err
		}
	} else {
		log.Debugf("Database (%s) exists", dbName)
		exists = true
	}

	db, err = sqlx.Open("postgres", datasource)
	if err != nil {
		return nil, false, err
	}

	return db, exists, nil
}

// createPostgresDB creates postgres database
func createPostgresDBTables(datasource string, dbName string, db *sqlx.DB) error {
	log.Debugf("Creating Postgres Database (%s)...", dbName)
	query := "CREATE DATABASE " + dbName
	_, err := db.Exec(query)
	if err != nil {
		return fmt.Errorf("Failed to create Postgres database: %s", err)
	}

	database, err := sqlx.Open("postgres", datasource)
	if err != nil {
		log.Errorf("Failed to open database (%s)", dbName)
	}

	log.Debug("Creating Tables...")
	if _, err := database.Exec("CREATE TABLE users (id VARCHAR(64), token bytea, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER,  max_enrollments INTEGER)"); err != nil {
		log.Errorf("Error creating users table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE affiliations (name VARCHAR(64), prekey VARCHAR(64))"); err != nil {
		log.Errorf("Error creating affiliations table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE certificates (id VARCHAR(64), serial_number bytea NOT NULL, authority_key_identifier bytea NOT NULL, ca_label bytea, status bytea NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem bytea NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
		log.Errorf("Error creating certificates table [error: %s] ", err)
		return err
	}
	return nil
}

// NewUserRegistryMySQL opens a connecton to a postgres database
func NewUserRegistryMySQL(datasource string, clientTLSConfig *tls.ClientTLSConfig) (*sqlx.DB, bool, error) {
	log.Debugf("Using MySQL database, connecting to database...")

	var exists bool
	dbName := getDBName(datasource)
	log.Debug("Database Name: ", dbName)

	re := regexp.MustCompile(`\/([a-zA-z]+)`)
	connStr := re.ReplaceAllString(datasource, "/")

	if clientTLSConfig.Enabled {
		tlsConfig, err := tls.GetClientTLSConfig(clientTLSConfig)
		if err != nil {
			log.Errorf("Failed to create TLS configuration [error: %s]", err)
		}

		mysql.RegisterTLSConfig("custom", tlsConfig)
	}

	log.Debug("Connection String: ", connStr)
	db, err := sqlx.Open("mysql", connStr)
	if err != nil {
		return nil, false, fmt.Errorf("Failed to open database: %s", err)
	}

	err = db.Ping()
	if err != nil {
		log.Errorf("Failed to connect to MySQL database [error: %s]", err)
	}

	// Check if database exists
	var name string
	err = db.QueryRow("SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = ?", dbName).Scan(&name)
	if err != nil {
		if err == sql.ErrNoRows {
			log.Debugf("Database (%s) does not exist", dbName)
			exists = false
		} else {
			return nil, false, fmt.Errorf("Failed to query 'INFORMATION_SCHEMA.SCHEMATA table: %s", err)
		}
	}

	if name == "" {
		createMySQLTables(datasource, dbName, db)
	} else {
		log.Debugf("Database (%s) exists", dbName)
		exists = true
	}

	db, err = sqlx.Open("mysql", datasource)
	if err != nil {
		return nil, false, err
	}

	return db, exists, nil
}

func createMySQLTables(datasource string, dbName string, db *sqlx.DB) error {
	log.Debugf("Creating MySQL Database (%s)...", dbName)

	_, err := db.Exec("CREATE DATABASE " + dbName)
	if err != nil {
		panic(err)
	}

	database, err := sqlx.Open("mysql", datasource)
	if err != nil {
		log.Errorf("Failed to open database (%s), err: %s", dbName, err)
	}
	log.Debug("Creating Tables...")
	if _, err := database.Exec("CREATE TABLE users (id VARCHAR(64) NOT NULL, token blob, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER, max_enrollments INTEGER, PRIMARY KEY (id)) DEFAULT CHARSET=utf8 COLLATE utf8_bin"); err != nil {
		log.Errorf("Error creating users table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE affiliations (name VARCHAR(64), prekey VARCHAR(64))"); err != nil {
		log.Errorf("Error creating affiliations table [error: %s] ", err)
		return err
	}
	if _, err := database.Exec("CREATE TABLE certificates (id VARCHAR(64), serial_number varbinary(128) NOT NULL, authority_key_identifier varbinary(128) NOT NULL, ca_label varbinary(128), status varbinary(128) NOT NULL, reason int, expiry timestamp DEFAULT '1970-01-01 00:00:01', revoked_at timestamp DEFAULT '1970-01-01 00:00:01', pem varbinary(4096) NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier)) DEFAULT CHARSET=utf8 COLLATE utf8_bin"); err != nil {
		log.Errorf("Error creating certificates table [error: %s] ", err)
		return err
	}

	return nil
}

// GetDBName gets database name from connection string
func getDBName(datasource string) string {
	var dbName string
	datasource = strings.ToLower(datasource)

	re := regexp.MustCompile(`(?:\/([^\/?]+))|(?:dbname=([^\s]+))`)
	getName := re.FindStringSubmatch(datasource)
	if getName != nil {
		dbName = getName[1]
		if dbName == "" {
			dbName = getName[2]
		}
	}

	return dbName
}

// GetConnStr gets connection string without database
func getConnStr(datasource string) string {
	re := regexp.MustCompile(`(dbname=)([^\s]+)`)
	connStr := re.ReplaceAllString(datasource, "")
	return connStr
}
<|endoftext|>