<commit_before>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aliyun/aliyun-oss-go-sdk/oss"
	"sqlflow.org/goalisa"
	"sqlflow.org/gomaxcompute"
	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/ir"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/pai"
)

var resourceName = "job.tar.gz"
var entryFile = "entry.py"
var reOSS = regexp.MustCompile(`oss://([^/]+).*host=([^&]+)`)

type alisaSubmitter struct {
	*defaultSubmitter
}

func (s *alisaSubmitter) submitAlisaTask(code, resourceURL string) error {
	_, dsName, err := database.ParseURL(s.Session.DbConnStr)
	if err != nil {
		return err
	}
	cfg, e := goalisa.ParseDSN(dsName)
	if e != nil {
		return e
	}

	cfg.Env["RES_DOWNLOAD_URL"] = fmt.Sprintf(`[{\"downloadUrl\":\"%s\", \"resourceName\":\"%s\"}]`, resourceURL, resourceName)
	cfg.Verbose = true
	newDatasource := cfg.FormatDSN()

	alisa, e := database.OpenDB(fmt.Sprintf("alisa://%s", newDatasource))
	if e != nil {
		return e
	}
	_, e = alisa.Exec(code)
	return e
}

func (s *alisaSubmitter) ExecuteTrain(ts *ir.TrainStmt) (e error) {
	ts.TmpTrainTable, ts.TmpValidateTable, e = createTempTrainAndValTable(ts.Select, ts.ValidationSelect, s.Session.DbConnStr)
	if e != nil {
		return e
	}
	defer dropTmpTables([]string{ts.TmpTrainTable, ts.TmpValidateTable}, s.Session.DbConnStr)

	ossModelPath, e := getModelPath(ts.Into, s.Session)
	if e != nil {
		return e
	}
	currProject, e := database.GetDatabaseName(s.Session.DbConnStr)
	if e != nil {
		return e
	}
	// cleanup saved model on OSS before training
	modelBucket, e := getModelBucket(currProject)
	if e != nil {
		return e
	}
	if e := modelBucket.DeleteObject(ossModelPath); e != nil {
		return e
	}

	// Alisa resource should be prefix with @@, alisa source would replace it with the RES_DOWN_URL.resourceName in alisa env.
	scriptPath := fmt.Sprintf("file://@@%s", resourceName)
	if e = pai.CleanupPAIModel(ts, s.Session); e != nil {
		return e
	}

	code, paiCmd, requirements, e := pai.Train(ts, s.Session, scriptPath, ts.Into, ossModelPath, s.Cwd)
	if e != nil {
		return e
	}
	// upload generated program to OSS and submit an Alisa task.
	return s.uploadResourceAndSubmitAlisaTask(code, requirements, paiCmd)
}

func (s *alisaSubmitter) ExecutePredict(ps *ir.PredictStmt) error {
	dbName, tableName, err := createTmpTableFromSelect(ps.Select, s.Session.DbConnStr)
	if err != nil {
		return err
	}
	ps.TmpPredictTable = strings.Join([]string{dbName, tableName}, ".")
	defer dropTmpTables([]string{ps.TmpPredictTable}, s.Session.DbConnStr)

	if e := createPredictionTableFromIR(ps, s.Db, s.Session); e != nil {
		return e
	}

	ossModelPath, e := getModelPath(ps.Using, s.Session)
	if e != nil {
		return e
	}
	// NOTE(typhoonzero): current project may differ from the project from SELECT statement.
	currProject, e := database.GetDatabaseName(s.Session.DbConnStr)
	if e != nil {
		return e
	}
	modelType, _, e := getOSSSavedModelType(ossModelPath, currProject)
	if e != nil {
		return e
	}

	scriptPath := fmt.Sprintf("file://@@%s", resourceName)
	code, paiCmd, requirements, e := pai.Predict(ps, s.Session, scriptPath, ps.Using, ossModelPath, s.Cwd, modelType)
	if e != nil {
		return e
	}
	return s.uploadResourceAndSubmitAlisaTask(code, requirements, paiCmd)
}

func (s *alisaSubmitter) uploadResourceAndSubmitAlisaTask(entryCode, requirements, alisaExecCode string) error {
	// achieve and upload alisa Resource
	ossObjectName := randStringRunes(16)
	alisaBucket, e := getAlisaBucket()
	if e != nil {
		return e
	}
	resourceURL, e := tarAndUploadResource(s.Cwd, entryCode, requirements, ossObjectName, alisaBucket)
	if e != nil {
		return e
	}
	defer alisaBucket.DeleteObject(ossObjectName)
	// upload generated program to OSS and submit an Alisa task.
	return s.submitAlisaTask(alisaExecCode, resourceURL)
}

func (s *alisaSubmitter) ExecuteExplain(cl *ir.ExplainStmt) error {
	dbName, tableName, err := createTmpTableFromSelect(cl.Select, s.Session.DbConnStr)
	if err != nil {
		return err
	}
	cl.TmpExplainTable = strings.Join([]string{dbName, tableName}, ".")
	defer dropTmpTables([]string{cl.TmpExplainTable}, s.Session.DbConnStr)

	currProject, err := database.GetDatabaseName(s.Session.DbConnStr)
	if err != nil {
		return err
	}
	ossModelPath, e := getModelPath(cl.ModelName, s.Session)
	if e != nil {
		return e
	}
	modelType, estimator, e := getOSSSavedModelType(ossModelPath, currProject)
	if e != nil {
		return e
	}
	if cl.Into != "" {
		if e := createExplainResultTable(s.Db, cl, cl.Into, modelType, estimator); e != nil {
			return e
		}
	}

	scriptPath := fmt.Sprintf("file://@@%s", resourceName)
	expn, e := pai.Explain(cl, s.Session, scriptPath, cl.ModelName, ossModelPath, s.Cwd, modelType)
	if e != nil {
		return e
	}
	if e = s.uploadResourceAndSubmitAlisaTask(expn.Code, expn.Requirements, expn.PaiCmd); e != nil {
		return e
	}
	if img, e := expn.Draw(); e == nil {
		s.Writer.Write(Figures{img, ""})
	}
	return e
}

func (s *alisaSubmitter) GetTrainStmtFromModel() bool { return false }

func findPyModulePath(pyModuleName string) (string, error) {
	cmd := exec.Command("python", "-c", fmt.Sprintf(`import %s;print(%s.__path__[0])`, pyModuleName, pyModuleName))
	out, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("failed %s, %v", cmd, err)
	}
	return strings.TrimSpace(string(out)), nil
}

func getModelBucket(project string) (*oss.Bucket, error) {
	ossCkptDir, err := pai.GetOSSCheckpointDir(project)
	if err != nil {
		return nil, err
	}
	ak := os.Getenv("SQLFLOW_OSS_AK")
	sk := os.Getenv("SQLFLOW_OSS_SK")
	ep := os.Getenv("SQLFLOW_OSS_MODEL_ENDPOINT")
	if ak == "" || sk == "" || ep == "" || ossCkptDir == "" {
		return nil, fmt.Errorf("should define SQLFLOW_OSS_MODEL_ENDPOINT, SQLFLOW_OSS_CHECKPOINT_DIR, SQLFLOW_OSS_AK, SQLFLOW_OSS_SK when using submitter alisa")
	}

	sub := reOSS.FindStringSubmatch(ossCkptDir)
	if len(sub) != 3 {
		return nil, fmt.Errorf("SQLFLOW_OSS_CHECKPOINT_DIR should be format: oss://bucket/?role_arn=xxx&host=xxx")
	}
	bucketName := sub[1]
	cli, e := oss.New(ep, ak, sk)
	if e != nil {
		return nil, e
	}
	return cli.Bucket(bucketName)
}

func getAlisaBucket() (*oss.Bucket, error) {
	ep := os.Getenv("SQLFLOW_OSS_ALISA_ENDPOINT")
	ak := os.Getenv("SQLFLOW_OSS_AK")
	sk := os.Getenv("SQLFLOW_OSS_SK")
	bucketName := os.Getenv("SQLFLOW_OSS_ALISA_BUCKET")

	if ep == "" || ak == "" || sk == "" {
		return nil, fmt.Errorf("should define SQLFLOW_OSS_ALISA_ENDPOINT, SQLFLOW_OSS_ALISA_BUCKET, SQLFLOW_OSS_AK, SQLFLOW_OSS_SK when using submitter alisa")
	}

	cli, err := oss.New(ep, ak, sk)
	if err != nil {
		return nil, err
	}
	return cli.Bucket(bucketName)
}

func writeFile(filePath, program string) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("create python code failed")
	}
	defer f.Close()
	f.WriteString(program)
	return nil
}

func getModelPath(modelName string, session *pb.Session) (string, error) {
	driverName, dsName, e := database.ParseURL(session.DbConnStr)
	if e != nil {
		return "", e
	}
	userID := session.UserId
	var projectName string
	if driverName == "maxcompute" {
		cfg, e := gomaxcompute.ParseDSN(dsName)
		if e != nil {
			return "", e
		}
		projectName = cfg.Project
	} else if driverName == "alisa" {
		cfg, e := goalisa.ParseDSN(dsName)
		if e != nil {
			return "", e
		}
		projectName = cfg.Project
	}
	if userID == "" {
		userID = "unknown"
	}
	return strings.Join([]string{projectName, userID, modelName}, "/"), nil
}

func tarAndUploadResource(cwd, entryCode, requirements, ossObjectName string, bucket *oss.Bucket) (string, error) {
	tarball := "job.tar.gz"
	if e := achieveResource(cwd, entryCode, requirements, tarball); e != nil {
		return "", e
	}
	resourceURL := fmt.Sprintf("https://%s.%s/%s", bucket.BucketName, bucket.Client.Config.Endpoint, ossObjectName)

	if e := bucket.PutObjectFromFile(ossObjectName, filepath.Join(cwd, tarball)); e != nil {
		return "", e
	}
	return resourceURL, nil
}
<commit_msg>use database.GetDatabaseName (#1907)<commit_after>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aliyun/aliyun-oss-go-sdk/oss"
	"sqlflow.org/goalisa"
	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/ir"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/pai"
)

var resourceName = "job.tar.gz"
var entryFile = "entry.py"
var reOSS = regexp.MustCompile(`oss://([^/]+).*host=([^&]+)`)

type alisaSubmitter struct {
	*defaultSubmitter
}

func (s *alisaSubmitter) submitAlisaTask(code, resourceURL string) error {
	_, dsName, err := database.ParseURL(s.Session.DbConnStr)
	if err != nil {
		return err
	}
	cfg, e := goalisa.ParseDSN(dsName)
	if e != nil {
		return e
	}

	cfg.Env["RES_DOWNLOAD_URL"] = fmt.Sprintf(`[{\"downloadUrl\":\"%s\", \"resourceName\":\"%s\"}]`, resourceURL, resourceName)
	cfg.Verbose = true
	newDatasource := cfg.FormatDSN()

	alisa, e := database.OpenDB(fmt.Sprintf("alisa://%s", newDatasource))
	if e != nil {
		return e
	}
	_, e = alisa.Exec(code)
	return e
}

func (s *alisaSubmitter) ExecuteTrain(ts *ir.TrainStmt) (e error) {
	ts.TmpTrainTable, ts.TmpValidateTable, e = createTempTrainAndValTable(ts.Select, ts.ValidationSelect, s.Session.DbConnStr)
	if e != nil {
		return e
	}
	defer dropTmpTables([]string{ts.TmpTrainTable, ts.TmpValidateTable}, s.Session.DbConnStr)

	ossModelPath, e := getModelPath(ts.Into, s.Session)
	if e != nil {
		return e
	}
	currProject, e := database.GetDatabaseName(s.Session.DbConnStr)
	if e != nil {
		return e
	}
	// cleanup saved model on OSS before training
	modelBucket, e := getModelBucket(currProject)
	if e != nil {
		return e
	}
	if e := modelBucket.DeleteObject(ossModelPath); e != nil {
		return e
	}

	// Alisa resource should be prefix with @@, alisa source would replace it with the RES_DOWN_URL.resourceName in alisa env.
	scriptPath := fmt.Sprintf("file://@@%s", resourceName)
	if e = pai.CleanupPAIModel(ts, s.Session); e != nil {
		return e
	}

	code, paiCmd, requirements, e := pai.Train(ts, s.Session, scriptPath, ts.Into, ossModelPath, s.Cwd)
	if e != nil {
		return e
	}
	// upload generated program to OSS and submit an Alisa task.
	return s.uploadResourceAndSubmitAlisaTask(code, requirements, paiCmd)
}

func (s *alisaSubmitter) ExecutePredict(ps *ir.PredictStmt) error {
	dbName, tableName, err := createTmpTableFromSelect(ps.Select, s.Session.DbConnStr)
	if err != nil {
		return err
	}
	ps.TmpPredictTable = strings.Join([]string{dbName, tableName}, ".")
	defer dropTmpTables([]string{ps.TmpPredictTable}, s.Session.DbConnStr)

	if e := createPredictionTableFromIR(ps, s.Db, s.Session); e != nil {
		return e
	}

	ossModelPath, e := getModelPath(ps.Using, s.Session)
	if e != nil {
		return e
	}
	// NOTE(typhoonzero): current project may differ from the project from SELECT statement.
	currProject, e := database.GetDatabaseName(s.Session.DbConnStr)
	if e != nil {
		return e
	}
	modelType, _, e := getOSSSavedModelType(ossModelPath, currProject)
	if e != nil {
		return e
	}

	scriptPath := fmt.Sprintf("file://@@%s", resourceName)
	code, paiCmd, requirements, e := pai.Predict(ps, s.Session, scriptPath, ps.Using, ossModelPath, s.Cwd, modelType)
	if e != nil {
		return e
	}
	return s.uploadResourceAndSubmitAlisaTask(code, requirements, paiCmd)
}

func (s *alisaSubmitter) uploadResourceAndSubmitAlisaTask(entryCode, requirements, alisaExecCode string) error {
	// achieve and upload alisa Resource
	ossObjectName := randStringRunes(16)
	alisaBucket, e := getAlisaBucket()
	if e != nil {
		return e
	}
	resourceURL, e := tarAndUploadResource(s.Cwd, entryCode, requirements, ossObjectName, alisaBucket)
	if e != nil {
		return e
	}
	defer alisaBucket.DeleteObject(ossObjectName)
	// upload generated program to OSS and submit an Alisa task.
	return s.submitAlisaTask(alisaExecCode, resourceURL)
}

func (s *alisaSubmitter) ExecuteExplain(cl *ir.ExplainStmt) error {
	dbName, tableName, err := createTmpTableFromSelect(cl.Select, s.Session.DbConnStr)
	if err != nil {
		return err
	}
	cl.TmpExplainTable = strings.Join([]string{dbName, tableName}, ".")
	defer dropTmpTables([]string{cl.TmpExplainTable}, s.Session.DbConnStr)

	currProject, err := database.GetDatabaseName(s.Session.DbConnStr)
	if err != nil {
		return err
	}
	ossModelPath, e := getModelPath(cl.ModelName, s.Session)
	if e != nil {
		return e
	}
	modelType, estimator, e := getOSSSavedModelType(ossModelPath, currProject)
	if e != nil {
		return e
	}
	if cl.Into != "" {
		if e := createExplainResultTable(s.Db, cl, cl.Into, modelType, estimator); e != nil {
			return e
		}
	}

	scriptPath := fmt.Sprintf("file://@@%s", resourceName)
	expn, e := pai.Explain(cl, s.Session, scriptPath, cl.ModelName, ossModelPath, s.Cwd, modelType)
	if e != nil {
		return e
	}
	if e = s.uploadResourceAndSubmitAlisaTask(expn.Code, expn.Requirements, expn.PaiCmd); e != nil {
		return e
	}
	if img, e := expn.Draw(); e == nil {
		s.Writer.Write(Figures{img, ""})
	}
	return e
}

func (s *alisaSubmitter) GetTrainStmtFromModel() bool { return false }

func findPyModulePath(pyModuleName string) (string, error) {
	cmd := exec.Command("python", "-c", fmt.Sprintf(`import %s;print(%s.__path__[0])`, pyModuleName, pyModuleName))
	out, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("failed %s, %v", cmd, err)
	}
	return strings.TrimSpace(string(out)), nil
}

func getModelBucket(project string) (*oss.Bucket, error) {
	ossCkptDir, err := pai.GetOSSCheckpointDir(project)
	if err != nil {
		return nil, err
	}
	ak := os.Getenv("SQLFLOW_OSS_AK")
	sk := os.Getenv("SQLFLOW_OSS_SK")
	ep := os.Getenv("SQLFLOW_OSS_MODEL_ENDPOINT")
	if ak == "" || sk == "" || ep == "" || ossCkptDir == "" {
		return nil, fmt.Errorf("should define SQLFLOW_OSS_MODEL_ENDPOINT, SQLFLOW_OSS_CHECKPOINT_DIR, SQLFLOW_OSS_AK, SQLFLOW_OSS_SK when using submitter alisa")
	}

	sub := reOSS.FindStringSubmatch(ossCkptDir)
	if len(sub) != 3 {
		return nil, fmt.Errorf("SQLFLOW_OSS_CHECKPOINT_DIR should be format: oss://bucket/?role_arn=xxx&host=xxx")
	}
	bucketName := sub[1]
	cli, e := oss.New(ep, ak, sk)
	if e != nil {
		return nil, e
	}
	return cli.Bucket(bucketName)
}

func getAlisaBucket() (*oss.Bucket, error) {
	ep := os.Getenv("SQLFLOW_OSS_ALISA_ENDPOINT")
	ak := os.Getenv("SQLFLOW_OSS_AK")
	sk := os.Getenv("SQLFLOW_OSS_SK")
	bucketName := os.Getenv("SQLFLOW_OSS_ALISA_BUCKET")

	if ep == "" || ak == "" || sk == "" {
		return nil, fmt.Errorf("should define SQLFLOW_OSS_ALISA_ENDPOINT, SQLFLOW_OSS_ALISA_BUCKET, SQLFLOW_OSS_AK, SQLFLOW_OSS_SK when using submitter alisa")
	}

	cli, err := oss.New(ep, ak, sk)
	if err != nil {
		return nil, err
	}
	return cli.Bucket(bucketName)
}

func writeFile(filePath, program string) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("create python code failed")
	}
	defer f.Close()
	f.WriteString(program)
	return nil
}

func getModelPath(modelName string, session *pb.Session) (string, error) {
	userID := session.UserId
	projectName, err := database.GetDatabaseName(session.DbConnStr)
	if err != nil {
		return "", err
	}
	if userID == "" {
		userID = "unknown"
	}
	return strings.Join([]string{projectName, userID, modelName}, "/"), nil
}

func tarAndUploadResource(cwd, entryCode, requirements, ossObjectName string, bucket *oss.Bucket) (string, error) {
	tarball := "job.tar.gz"
	if e := achieveResource(cwd, entryCode, requirements, tarball); e != nil {
		return "", e
	}
	resourceURL := fmt.Sprintf("https://%s.%s/%s", bucket.BucketName, bucket.Client.Config.Endpoint, ossObjectName)

	if e := bucket.PutObjectFromFile(ossObjectName, filepath.Join(cwd, tarball)); e != nil {
		return "", e
	}
	return resourceURL, nil
}
<|endoftext|>


<commit_before>package watcher

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestConf(t *testing.T) {
	watcherConf := `{
    	"points": [
        	{
            	"name": "post list",
            	"desc": "post list",
            	"type": "http",
            	"conf": "\n{\n    \"task\": {\n        \"path\": \"https://api-prod.wallstreetcn.com/apiv1/content/articles\",\n        \"method\": \"get\"\n    },\n    \"assertions\": [\n        {\n            \"source\": \"status\",\n            \"operator\": \"equal\",\n            \"value\": 200\n        },\n        {\n            \"source\": \"body\",\n            \"operator\": \"equal\",\n            \"key\": \"code\",\n            \"value\": 20000,\n            \"source_encoding\": \"json\"\n        }\n    ],\n    \"variables\": [\n        {\n            \"source\": \"body\",\n            \"key\": \"code\",\n            \"source_encoding\": \"json\"\n        }\n    ]\n}\n"
        	}
    	]
	}
`
	c := Config{}
	json.Unmarshal([]byte(watcherConf), &c)
	assert.NotNil(t, c.Points)
	assert.Len(t, c.Points, 1)
	fmt.Println(c)
}
<commit_msg>remove log<commit_after>package watcher

import (
	"encoding/json"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestConf(t *testing.T) {
	watcherConf := `{
    	"points": [
        	{
            	"name": "post list",
            	"desc": "post list",
            	"type": "http",
            	"conf": "\n{\n    \"task\": {\n        \"path\": \"https://api-prod.wallstreetcn.com/apiv1/content/articles\",\n        \"method\": \"get\"\n    },\n    \"assertions\": [\n        {\n            \"source\": \"status\",\n            \"operator\": \"equal\",\n            \"value\": 200\n        },\n        {\n            \"source\": \"body\",\n            \"operator\": \"equal\",\n            \"key\": \"code\",\n            \"value\": 20000,\n            \"source_encoding\": \"json\"\n        }\n    ],\n    \"variables\": [\n        {\n            \"source\": \"body\",\n            \"key\": \"code\",\n            \"source_encoding\": \"json\"\n        }\n    ]\n}\n"
        	}
    	]
	}
`
	c := Config{}
	json.Unmarshal([]byte(watcherConf), &c)
	assert.NotNil(t, c.Points)
	assert.Len(t, c.Points, 1)
}
<|endoftext|>


<commit_before>// Package consumer implements command line crawl consumer from nsq.
package consumer

import (
	"errors"
	"os"
	"os/signal"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"

	"github.com/crackcomm/crawl"
	"github.com/crackcomm/crawl/nsq/nsqcrawl"
)

// Spider - Spider registrator.
type Spider func(crawl.Crawler)

// App - Consumer command line application structure.
type App struct {
	// Ctx - Cli context, set on action.
	Ctx *cli.Context

	// Queue - NSQ queue. Constructed on first Action() call.
	*nsqcrawl.Queue

	// before - Flag requirements checking.
	before func(c *App) error

	// crawler - Accessed using Crawler() which constructs it on first call
	// using parameters from commmand line.
	crawler crawl.Crawler

	// crawlerConstructor - Constructs a crawler. Called only once in Crawler().
	// It can be changed using WithCrawlerConstructor()
	crawlerConstructor func(*App) crawl.Crawler

	// opts - Options which are applied on Action() call with all required
	// parameters from the command line context.
	opts []Option

	// spiderConstructors - List of functions which use flags to construct spider.
	spiderConstructors []func(*App) Spider
}

// Flags - Consumer app flags.
var Flags = []cli.Flag{
	cli.StringFlag{
		Name:   "topic",
		EnvVar: "TOPIC",
		Usage:  "crawl requests nsq topic (required)",
		Value:  "crawl_requests",
	},
	cli.StringFlag{
		Name:   "channel",
		EnvVar: "CHANNEL",
		Usage:  "crawl requests nsq channel (required)",
		Value:  "default",
	},
	cli.StringSliceFlag{
		Name:   "nsq-addr",
		EnvVar: "NSQ_ADDR",
	},
	cli.StringSliceFlag{
		Name:   "nsqlookup-addr",
		EnvVar: "NSQLOOKUP_ADDR",
	},
	cli.IntFlag{
		Name:   "concurrency",
		Value:  100,
		EnvVar: "CONCURRENCY",
	},
}

// New - Creates nsq consumer app.
func New(opts ...Option) *cli.App {
	app := &App{opts: opts}
	cliapp := cli.NewApp()
	cliapp.Name = "crawler"
	cliapp.HelpName = cliapp.Name
	cliapp.Version = "0.0.1"
	cliapp.Usage = "nsq crawl consumer"
	cliapp.Flags = Flags
	cliapp.Before = app.Before
	cliapp.Action = app.Action
	return cliapp
}

// Action - Command line action.
func (app *App) Action(c *cli.Context) {
	app.Queue = nsqcrawl.NewQueue(c.String("topic"), c.String("channel"), c.Int("concurrency"))

	for _, opt := range app.opts {
		opt(app)
	}

	crawler := app.Crawler()

	nsqAddr := c.StringSlice("nsq-addr")[0]
	if err := app.Queue.Producer.Connect(nsqAddr); err != nil {
		glog.Fatalf("Error connecting producer to %q: %v", nsqAddr, err)
	}

	if addrs := c.StringSlice("nsq-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq %s", addr)
			if err := app.Queue.Consumer.Connect(addr); err != nil {
				glog.Fatalf("Error connecting to nsq %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq %s", addr)
		}
	}

	if addrs := c.StringSlice("nsqlookup-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq lookup %s", addr)
			if err := app.Queue.Consumer.ConnectLookupd(addr); err != nil {
				glog.Fatalf("Error connecting to nsq lookup %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq lookup %s", addr)
		}
	}

	for _, spiderConstructor := range app.spiderConstructors {
		spiderConstructor(app)(crawler)
	}

	go func() {
		for err := range crawler.Errors() {
			glog.Warningf("Crawl error: %v", err)
		}
	}()

	done := make(chan bool, 1)
	go func() {
		crawler.Start()
		done <- true
	}()

	glog.Infof("Started crawler (topic=%q)", c.String("topic"))

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	for {
		select {
		case <-done:
			glog.Info("Crawler closed")
			return
		case s := <-sig:
			glog.Infof("Received signal %v, closing crawler", s)
			if err := app.Queue.Close(); err != nil {
				glog.Fatalf("Error closing queue: %v", err)
			}
		}
	}
}

// Before - Executed before action.
func (app *App) Before(c *cli.Context) error {
	// Set application context
	app.Ctx = c

	// Use customized before if any
	if app.before != nil {
		return app.before(app)
	}
	return beforeApp(c)
}

func beforeApp(c *cli.Context) error {
	if c.String("topic") == "" {
		return errors.New("Flag --topic cannot be empty.")
	}
	if len(c.StringSlice("nsq-addr")) == 0 && len(c.StringSlice("nsqlookup-addr")) == 0 {
		return errors.New("At least one --nsq-addr or --nsqlookup-addr is required")
	}
	return nil
}

// Crawler - Returns app crawler. Constructs if empty.
func (app *App) Crawler() crawl.Crawler {
	if app.crawler == nil {
		if app.crawlerConstructor != nil {
			app.crawler = app.crawlerConstructor(app)
		} else {
			app.crawler = crawlerConstructor(app)
		}
	}
	return app.crawler
}

func crawlerConstructor(app *App) crawl.Crawler {
	return crawl.New(
		crawl.WithQueue(app.Queue),
		crawl.WithConcurrency(app.Ctx.Int("concurrency")),
	)
}
<commit_msg>nsq consumer: tolerate empty spider returned from constructor<commit_after>// Package consumer implements command line crawl consumer from nsq.
package consumer

import (
	"errors"
	"os"
	"os/signal"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"

	"github.com/crackcomm/crawl"
	"github.com/crackcomm/crawl/nsq/nsqcrawl"
)

// Spider - Spider registrator.
type Spider func(crawl.Crawler)

// App - Consumer command line application structure.
type App struct {
	// Ctx - Cli context, set on action.
	Ctx *cli.Context

	// Queue - NSQ queue. Constructed on first Action() call.
	*nsqcrawl.Queue

	// before - Flag requirements checking.
	before func(c *App) error

	// crawler - Accessed using Crawler() which constructs it on first call
	// using parameters from commmand line.
	crawler crawl.Crawler

	// crawlerConstructor - Constructs a crawler. Called only once in Crawler().
	// It can be changed using WithCrawlerConstructor()
	crawlerConstructor func(*App) crawl.Crawler

	// opts - Options which are applied on Action() call with all required
	// parameters from the command line context.
	opts []Option

	// spiderConstructors - List of functions which use flags to construct spider.
	spiderConstructors []func(*App) Spider
}

// Flags - Consumer app flags.
var Flags = []cli.Flag{
	cli.StringFlag{
		Name:   "topic",
		EnvVar: "TOPIC",
		Usage:  "crawl requests nsq topic (required)",
		Value:  "crawl_requests",
	},
	cli.StringFlag{
		Name:   "channel",
		EnvVar: "CHANNEL",
		Usage:  "crawl requests nsq channel (required)",
		Value:  "default",
	},
	cli.StringSliceFlag{
		Name:   "nsq-addr",
		EnvVar: "NSQ_ADDR",
	},
	cli.StringSliceFlag{
		Name:   "nsqlookup-addr",
		EnvVar: "NSQLOOKUP_ADDR",
	},
	cli.IntFlag{
		Name:   "concurrency",
		Value:  100,
		EnvVar: "CONCURRENCY",
	},
}

// New - Creates nsq consumer app.
func New(opts ...Option) *cli.App {
	app := &App{opts: opts}
	cliapp := cli.NewApp()
	cliapp.Name = "crawler"
	cliapp.HelpName = cliapp.Name
	cliapp.Version = "0.0.1"
	cliapp.Usage = "nsq crawl consumer"
	cliapp.Flags = Flags
	cliapp.Before = app.Before
	cliapp.Action = app.Action
	return cliapp
}

// Action - Command line action.
func (app *App) Action(c *cli.Context) {
	app.Queue = nsqcrawl.NewQueue(c.String("topic"), c.String("channel"), c.Int("concurrency"))

	for _, opt := range app.opts {
		opt(app)
	}

	crawler := app.Crawler()

	nsqAddr := c.StringSlice("nsq-addr")[0]
	if err := app.Queue.Producer.Connect(nsqAddr); err != nil {
		glog.Fatalf("Error connecting producer to %q: %v", nsqAddr, err)
	}

	if addrs := c.StringSlice("nsq-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq %s", addr)
			if err := app.Queue.Consumer.Connect(addr); err != nil {
				glog.Fatalf("Error connecting to nsq %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq %s", addr)
		}
	}

	if addrs := c.StringSlice("nsqlookup-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq lookup %s", addr)
			if err := app.Queue.Consumer.ConnectLookupd(addr); err != nil {
				glog.Fatalf("Error connecting to nsq lookup %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq lookup %s", addr)
		}
	}

	for _, spiderConstructor := range app.spiderConstructors {
		if spider := spiderConstructor(app); spider != nil {
			spider(crawler)
		}
	}

	go func() {
		for err := range crawler.Errors() {
			glog.Warningf("Crawl error: %v", err)
		}
	}()

	done := make(chan bool, 1)
	go func() {
		crawler.Start()
		done <- true
	}()

	glog.Infof("Started crawler (topic=%q)", c.String("topic"))

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	for {
		select {
		case <-done:
			glog.Info("Crawler closed")
			return
		case s := <-sig:
			glog.Infof("Received signal %v, closing crawler", s)
			if err := app.Queue.Close(); err != nil {
				glog.Fatalf("Error closing queue: %v", err)
			}
		}
	}
}

// Before - Executed before action.
func (app *App) Before(c *cli.Context) error {
	// Set application context
	app.Ctx = c

	// Use customized before if any
	if app.before != nil {
		return app.before(app)
	}
	return beforeApp(c)
}

func beforeApp(c *cli.Context) error {
	if c.String("topic") == "" {
		return errors.New("Flag --topic cannot be empty.")
	}
	if len(c.StringSlice("nsq-addr")) == 0 && len(c.StringSlice("nsqlookup-addr")) == 0 {
		return errors.New("At least one --nsq-addr or --nsqlookup-addr is required")
	}
	return nil
}

// Crawler - Returns app crawler. Constructs if empty.
func (app *App) Crawler() crawl.Crawler {
	if app.crawler == nil {
		if app.crawlerConstructor != nil {
			app.crawler = app.crawlerConstructor(app)
		} else {
			app.crawler = crawlerConstructor(app)
		}
	}
	return app.crawler
}

func crawlerConstructor(app *App) crawl.Crawler {
	return crawl.New(
		crawl.WithQueue(app.Queue),
		crawl.WithConcurrency(app.Ctx.Int("concurrency")),
	)
}
<|endoftext|>


<commit_before>package obj

import (
	"reflect"
	"testing"

	"github.com/polydawn/go-xlate/obj/atlas"
	. "github.com/polydawn/go-xlate/testutil"
	. "github.com/polydawn/go-xlate/tok"
)

func TestMarshaller(t *testing.T) {
	type NN struct {
		F int
		X string
	}
	type BB struct {
		Z string
	}
	type AA struct {
		X string
		Y BB
	}
	type DD struct {
		A *AA
		F *int
		Z *string
	}

	tt := []struct {
		title       string
		targetFn    func() interface{} // func returns target, so test source looks like your call param
		suite       *Suite
		expectSeq   []Token
		expectErr   error
		expectPanic error
		errString   string
	}{
		{
			title:    "simple literal",
			targetFn: func() interface{} { i := 4; return &i },
			expectSeq: []Token{
				TokInt(4),
			},
		},
		{
			title: "simple struct of literals",
			targetFn: func() interface{} {
				return &NN{
					7, "s",
				}
			},
			suite: (&Suite{}).
				Add(NN{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(NN{}),
					Fields: []atlas.Entry{
						{Name: "F", FieldName: atlas.FieldName{"F"}},
						{Name: "X", FieldName: atlas.FieldName{"X"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("F"), TokInt(7),
				TokStr("X"), TokStr("s"),
				Token_MapClose,
			},
		},
		{
			title: "nested structs and literals",
			targetFn: func() interface{} {
				return &AA{
					"s",
					BB{},
				}
			},
			suite: (&Suite{}).
				Add(NN{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Fields: []atlas.Entry{ /* this should be extraneous */ },
				})).
				Add(AA{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(AA{}),
					Fields: []atlas.Entry{
						{Name: "a.y", FieldName: atlas.FieldName{"Y"}},
						{Name: "a.x", FieldName: atlas.FieldName{"X"}},
					},
				})).
				Add(BB{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(BB{}),
					Fields: []atlas.Entry{
						{Name: "zee", FieldName: atlas.FieldName{"Z"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("a.y"), Token_MapOpen,
				TokStr("zee"), TokStr(""),
				Token_MapClose,
				TokStr("a.x"), TokStr("s"),
				Token_MapClose,
			},
		},
		{
			title: "struct with fields missing a handler",
			targetFn: func() interface{} {
				return &AA{
					"s",
					BB{},
				}
			},
			suite: (&Suite{}).
				Add(AA{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(AA{}),
					Fields: []atlas.Entry{
						{Name: "a.y", FieldName: atlas.FieldName{"Y"}},
						{Name: "a.x", FieldName: atlas.FieldName{"X"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("a.y"), nil, // last step panics
			},
			expectPanic: ErrNoHandler{},
			errString:   "no machine available in suite for struct of type obj.BB",
		},
		{
			title: "nested structs and ptrs",
			targetFn: func() interface{} {
				f := 2
				return &DD{
					&AA{
						"X",
						BB{"B"},
					},
					&f,
					nil,
				}
			},
			suite: (&Suite{}).
				Add(DD{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(DD{}),
					Fields: []atlas.Entry{
						{Name: "1", FieldName: atlas.FieldName{"A"}},
						{Name: "3", FieldName: atlas.FieldName{"Z"}},
						{Name: "2", FieldName: atlas.FieldName{"F"}},
					},
				})).
				Add(AA{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(AA{}),
					Fields: []atlas.Entry{
						{Name: "a.y", FieldName: atlas.FieldName{"Y"}},
					},
				})).
				Add(BB{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(BB{}),
					Fields: []atlas.Entry{
						{Name: "zee", FieldName: atlas.FieldName{"Z"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("1"), Token_MapOpen,
				/**/ TokStr("a.y"), Token_MapOpen,
				/**/ /**/ TokStr("zee"), TokStr("B"),
				/**/ Token_MapClose,
				Token_MapClose,
				TokStr("3"), nil,
				TokStr("2"), TokInt(2),
				Token_MapClose,
			},
		},
		// TODO following doesn't work yet because of type-loss issues when converting away from reflect.Value
		//  (which are in turn blocked from easily resolution because of the tricky detail that map vals are not addressable..).
		//{
		//	title: "wildcard map of literals",
		//	targetFn: func() interface{} {
		//		return &map[string]int{
		//			"a": 1,
		//		}
		//	},
		//	expectSeq: []Token{
		//		Token_MapOpen,
		//		"a", 1,
		//		Token_MapClose,
		//	},
		//},
	}
	for _, tr := range tt {
		if tr.suite == nil {
			tr.suite = &Suite{}
		}
		err := CapturePanics(func() {
			marshaller := NewMarshaler(tr.suite, tr.targetFn())

			// Run steps.
			var done bool
			var err error
			var tok Token
			for n, expectTok := range tr.expectSeq {
				done, err = marshaller.Step(&tok)
				if !IsTokenEqual(expectTok, tok) {
					t.Errorf("test %q failed: step %d yielded wrong token: expected %s, got %s",
						tr.title, n, TokenToString(expectTok), TokenToString(tok))
				}
				if err != nil {
					t.Errorf("test %q failed: step %d (expecting %#v) errored: %s",
						tr.title, n, expectTok, err)
				}
				if done && n != len(tr.expectSeq)-1 {
					t.Errorf("test %q failed: done early! on step %d out of %d tokens",
						tr.title, n, len(tr.expectSeq))
				}
			}
			if !done {
				t.Errorf("test %q failed: still not done after %d tokens!",
					tr.title, len(tr.expectSeq))
			}
		})
		if tr.expectPanic == nil && err == nil {
			t.Logf("test %q halted correctly and passed", tr.title)
		} else if err == nil {
			t.Errorf("test %q failed: expected panic of %T",
				tr.title, tr.expectPanic)
		} else {
			ok := true
			if reflect.TypeOf(tr.expectPanic) != reflect.TypeOf(err) {
				t.Errorf("test %q failed: expected panic of type %T, but got %T",
					tr.title, tr.expectPanic, err)
				ok = false
			}
			if tr.errString != err.Error() {
				t.Errorf("test %q failed: expected panic of string of %q, but got %q",
					tr.title, tr.errString, err)
				ok = false
			}
			if ok {
				t.Logf("test %q panicked correctly and passed", tr.title)
			}
		}
	}
}
<commit_msg>Test for deeper ptrs.  Yayy, it works!<commit_after>package obj

import (
	"reflect"
	"testing"

	"github.com/polydawn/go-xlate/obj/atlas"
	. "github.com/polydawn/go-xlate/testutil"
	. "github.com/polydawn/go-xlate/tok"
)

func TestMarshaller(t *testing.T) {
	type NN struct {
		F int
		X string
	}
	type BB struct {
		Z string
	}
	type AA struct {
		X string
		Y BB
	}
	type DD struct {
		A *AA
		F *int
		Z *string
	}
	type DDD struct {
		A **AA
		F *int
		Z *string
	}

	tt := []struct {
		title       string
		targetFn    func() interface{} // func returns target, so test source looks like your call param
		suite       *Suite
		expectSeq   []Token
		expectErr   error
		expectPanic error
		errString   string
	}{
		{
			title:    "simple literal",
			targetFn: func() interface{} { i := 4; return &i },
			expectSeq: []Token{
				TokInt(4),
			},
		},
		{
			title: "simple struct of literals",
			targetFn: func() interface{} {
				return &NN{
					7, "s",
				}
			},
			suite: (&Suite{}).
				Add(NN{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(NN{}),
					Fields: []atlas.Entry{
						{Name: "F", FieldName: atlas.FieldName{"F"}},
						{Name: "X", FieldName: atlas.FieldName{"X"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("F"), TokInt(7),
				TokStr("X"), TokStr("s"),
				Token_MapClose,
			},
		},
		{
			title: "nested structs and literals",
			targetFn: func() interface{} {
				return &AA{
					"s",
					BB{},
				}
			},
			suite: (&Suite{}).
				Add(NN{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Fields: []atlas.Entry{ /* this should be extraneous */ },
				})).
				Add(AA{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(AA{}),
					Fields: []atlas.Entry{
						{Name: "a.y", FieldName: atlas.FieldName{"Y"}},
						{Name: "a.x", FieldName: atlas.FieldName{"X"}},
					},
				})).
				Add(BB{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(BB{}),
					Fields: []atlas.Entry{
						{Name: "zee", FieldName: atlas.FieldName{"Z"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("a.y"), Token_MapOpen,
				TokStr("zee"), TokStr(""),
				Token_MapClose,
				TokStr("a.x"), TokStr("s"),
				Token_MapClose,
			},
		},
		{
			title: "struct with fields missing a handler",
			targetFn: func() interface{} {
				return &AA{
					"s",
					BB{},
				}
			},
			suite: (&Suite{}).
				Add(AA{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(AA{}),
					Fields: []atlas.Entry{
						{Name: "a.y", FieldName: atlas.FieldName{"Y"}},
						{Name: "a.x", FieldName: atlas.FieldName{"X"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("a.y"), nil, // last step panics
			},
			expectPanic: ErrNoHandler{},
			errString:   "no machine available in suite for struct of type obj.BB",
		},
		{
			title: "nested structs and ptrs",
			targetFn: func() interface{} {
				f := 2
				return &DD{
					&AA{
						"X",
						BB{"B"},
					},
					&f,
					nil,
				}
			},
			suite: (&Suite{}).
				Add(DD{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(DD{}),
					Fields: []atlas.Entry{
						{Name: "1", FieldName: atlas.FieldName{"A"}},
						{Name: "3", FieldName: atlas.FieldName{"Z"}},
						{Name: "2", FieldName: atlas.FieldName{"F"}},
					},
				})).
				Add(AA{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(AA{}),
					Fields: []atlas.Entry{
						{Name: "a.y", FieldName: atlas.FieldName{"Y"}},
					},
				})).
				Add(BB{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(BB{}),
					Fields: []atlas.Entry{
						{Name: "zee", FieldName: atlas.FieldName{"Z"}},
					},
				})),
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("1"), Token_MapOpen,
				/**/ TokStr("a.y"), Token_MapOpen,
				/**/ /**/ TokStr("zee"), TokStr("B"),
				/**/ Token_MapClose,
				Token_MapClose,
				TokStr("3"), nil,
				TokStr("2"), TokInt(2),
				Token_MapClose,
			},
		},
		{
			title: "nested structs and deep ptrs",
			targetFn: func() interface{} {
				f := 2
				ap := &AA{
					"X",
					BB{"B"},
				}
				return &DDD{
					&ap,
					&f,
					nil,
				}
			},
			suite: (&Suite{}).
				Add(DDD{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(DDD{}),
					Fields: []atlas.Entry{
						{Name: "1", FieldName: atlas.FieldName{"A"}},
						{Name: "3", FieldName: atlas.FieldName{"Z"}},
						{Name: "2", FieldName: atlas.FieldName{"F"}},
					},
				})).
				Add(AA{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(AA{}),
					Fields: []atlas.Entry{
						{Name: "a.y", FieldName: atlas.FieldName{"Y"}},
					},
				})).
				Add(BB{}, NewMarshalMachineStructAtlas(atlas.Atlas{
					Type: reflect.TypeOf(BB{}),
					Fields: []atlas.Entry{
						{Name: "zee", FieldName: atlas.FieldName{"Z"}},
					},
				})),
			// should serialize exact same way as previous test:
			expectSeq: []Token{
				Token_MapOpen,
				TokStr("1"), Token_MapOpen,
				/**/ TokStr("a.y"), Token_MapOpen,
				/**/ /**/ TokStr("zee"), TokStr("B"),
				/**/ Token_MapClose,
				Token_MapClose,
				TokStr("3"), nil,
				TokStr("2"), TokInt(2),
				Token_MapClose,
			},
		},
		// TODO following doesn't work yet because of type-loss issues when converting away from reflect.Value
		//  (which are in turn blocked from easily resolution because of the tricky detail that map vals are not addressable..).
		//{
		//	title: "wildcard map of literals",
		//	targetFn: func() interface{} {
		//		return &map[string]int{
		//			"a": 1,
		//		}
		//	},
		//	expectSeq: []Token{
		//		Token_MapOpen,
		//		"a", 1,
		//		Token_MapClose,
		//	},
		//},
	}
	for _, tr := range tt {
		if tr.suite == nil {
			tr.suite = &Suite{}
		}
		err := CapturePanics(func() {
			marshaller := NewMarshaler(tr.suite, tr.targetFn())

			// Run steps.
			var done bool
			var err error
			var tok Token
			for n, expectTok := range tr.expectSeq {
				done, err = marshaller.Step(&tok)
				if !IsTokenEqual(expectTok, tok) {
					t.Errorf("test %q failed: step %d yielded wrong token: expected %s, got %s",
						tr.title, n, TokenToString(expectTok), TokenToString(tok))
				}
				if err != nil {
					t.Errorf("test %q failed: step %d (expecting %#v) errored: %s",
						tr.title, n, expectTok, err)
				}
				if done && n != len(tr.expectSeq)-1 {
					t.Errorf("test %q failed: done early! on step %d out of %d tokens",
						tr.title, n, len(tr.expectSeq))
				}
			}
			if !done {
				t.Errorf("test %q failed: still not done after %d tokens!",
					tr.title, len(tr.expectSeq))
			}
		})
		if tr.expectPanic == nil && err == nil {
			t.Logf("test %q halted correctly and passed", tr.title)
		} else if err == nil {
			t.Errorf("test %q failed: expected panic of %T",
				tr.title, tr.expectPanic)
		} else {
			ok := true
			if reflect.TypeOf(tr.expectPanic) != reflect.TypeOf(err) {
				t.Errorf("test %q failed: expected panic of type %T, but got %T",
					tr.title, tr.expectPanic, err)
				ok = false
			}
			if tr.errString != err.Error() {
				t.Errorf("test %q failed: expected panic of string of %q, but got %q",
					tr.title, tr.errString, err)
				ok = false
			}
			if ok {
				t.Logf("test %q panicked correctly and passed", tr.title)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"syscall"

	"github.com/nulayer/qmd/common"
	"github.com/rcrowley/go-tigertonic"
)

var (
	listen = flag.String("listen", "0.0.0.0:8080", "listen address")
	queue  = flag.String("queue", "0.0.0.0:8181", "queue address")
	mux    *tigertonic.TrieServeMux
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: qmd-server [-listen=<listen>] [-queue=<queue>]")
		flag.PrintDefaults()
	}

	// Register the endpoints.
	mux = tigertonic.NewTrieServeMux()
	mux.Handle("POST", "/job", tigertonic.Marshaled(create))
	mux.Handle("GET", "/job/{id}", tigertonic.Marshaled(get))
}

func main() {
	flag.Parse()

	server := tigertonic.NewServer(*listen, mux)
	fmt.Printf("Listening on %s\n", *listen)
	go func() {
		var err error
		err = server.ListenAndServe()
		if nil != err {
			log.Println(err)
		}
	}()
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
	log.Println(<-ch)
	server.Close()
}

// POST /job
func create(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {
	for _, script := range rq.Scripts {
		for _, params := range script.Params {
			fmt.Println(params)
		}
	}

	// TODO: Figure out how to push to the queue.

	return http.StatusCreated, http.Header{
		"Content-Location": {fmt.Sprintf(
			"%s://%s/1.0/job/%s",
			u.Scheme,
			u.Host,
			rq.ID,
		)},
	}, &common.JobResponse{rq.ID, rq.Scripts}, nil
}

// GET /job/{id}
func get(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {

	// TODO: Figure out how to request a job through the queue.

	return http.StatusOK, nil, &common.JobResponse{
		u.Query().Get("id"),
		nil,
	}, nil
}
<commit_msg>Added NSQ code.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"syscall"

	"github.com/bitly/go-nsq"
	"github.com/nulayer/qmd/common"
	"github.com/rcrowley/go-tigertonic"
)

var (
	listen   = flag.String("listen", "0.0.0.0:8080", "listen address")
	queue    = flag.String("queue", "0.0.0.0:8181", "queue address")
	mux      *tigertonic.TrieServeMux
	producer *nsq.Producer
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: qmd-server [-listen=<listen>] [-queue=<queue>]")
		flag.PrintDefaults()
	}

	// Register the endpoints.
	mux = tigertonic.NewTrieServeMux()
	mux.Handle("POST", "/job", tigertonic.Marshaled(create))
	mux.Handle("GET", "/job/{id}", tigertonic.Marshaled(get))
}

func main() {
	flag.Parse()

	producer = nsq.NewProducer(*queue, nsq.NewConfig())
	fmt.Printf("Sending to %s\n", *queue)

	server := tigertonic.NewServer(*listen, mux)
	fmt.Printf("Listening on %s\n", *listen)
	go func() {
		var err error
		err = server.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
	}()
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
	log.Println(<-ch)
	server.Close()
}

// POST /job
func create(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {
	for _, script := range rq.Scripts {
		for _, params := range script.Params {
			fmt.Println(params)
		}
	}

	var err error

	// TODO: Figure out how to push to the queue.
	data, err := json.Marshal(rq.Scripts)
	if err != nil {
		fmt.Println(err)
	}

	err = producer.Publish("jobs", data)
	if err != nil {
		fmt.Println(err)
	}

	return http.StatusCreated, http.Header{
		"Content-Location": {fmt.Sprintf(
			"%s://%s/1.0/job/%s",
			u.Scheme,
			u.Host,
			rq.ID,
		)},
	}, &common.JobResponse{rq.ID, rq.Scripts}, nil
}

// GET /job/{id}
func get(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {

	// TODO: Figure out how to request a job through the queue.

	return http.StatusOK, nil, &common.JobResponse{
		u.Query().Get("id"),
		nil,
	}, nil
}
<|endoftext|>


<commit_before>package lazyquicktime

import "fmt"
import "image"
import "errors"

import "time"
import "log"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/go-prores-ffmpeg"

type LazyQuicktime struct {
	file lazyfs.FileSource
	Tree quicktime.AtomArray
	Trak quicktime.TRAKAtom
	Stbl *quicktime.STBLAtom
	Mvhd quicktime.MVHDAtom

	FileSize int64
}

func LoadMovMetadata(file lazyfs.FileSource) (*LazyQuicktime, error) {

	mov := &LazyQuicktime{file: file}

	sz, err := file.FileSize()
	if sz < 0 || err != nil {
		return mov, fmt.Errorf("Unable to retrieve file size.")
	}

	mov.FileSize = sz

	set_eagerload := func(conf *quicktime.BuildTreeConfig) {
		conf.EagerloadTypes = []string{"moov"}
	}

	//fmt.Println("Reading Mov of size ", mov.FileSize)
	tree, err := quicktime.BuildTree(file, uint64(mov.FileSize), set_eagerload)

	if err != nil {
		return mov, err
	}
	mov.Tree = tree

	moov := mov.Tree.FindAtom("moov")
	if moov == nil {
		return mov, errors.New("Can't find MOOV atom")
	}

	mvhd := moov.FindAtom("mvhd")
	if mvhd == nil {
		return mov, errors.New("Couldn't find MVHD in the moov atom")
	}
	mov.Mvhd, _ = quicktime.ParseMVHD(mvhd)

	tracks := moov.FindAtoms("trak")
	if tracks == nil || len(tracks) == 0 {
		return mov, errors.New("Couldn't find any TRAKs in the MOOV")
	}

	var track *quicktime.Atom = nil
	for i, t := range tracks {
		mdia := t.FindAtom("mdia")
		if mdia == nil {
			fmt.Println("No mdia track", i)
			continue
		}

		minf := mdia.FindAtom("minf")
		if minf == nil {
			fmt.Println("No minf track", i)
			continue
		}

		if minf.FindAtom("vmhd") != nil {
			track = t
			break
		}
	}

	if track == nil {
		return mov, errors.New("Couldn't identify the Video track")
	}

	mov.Trak, err = quicktime.ParseTRAK(track)
	if err != nil {
		return mov, errors.New(fmt.Sprintf("Unable to parse TRAK atom: %s", err.Error()))
	}

	mov.Stbl = &mov.Trak.Mdia.Minf.Stbl // Just an alias

	return mov, nil
}

func (mov *LazyQuicktime) NumFrames() int {
	return mov.Stbl.NumFrames()
}

func (mov *LazyQuicktime) Duration() float32 {
	return mov.Mvhd.Duration()
}

func (mov *LazyQuicktime) ExtractFrame(frame int) (image.Image, error) {

	frame_offset, frame_size, _ := mov.Stbl.SampleOffsetSize(frame)

	//fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

	buf := make([]byte, frame_size)

	if buf == nil {
		return nil, fmt.Errorf("Couldn't make buffer of size %d", frame_size)
	}

	startRead := time.Now()
	n, _ := mov.file.ReadAt(buf, frame_offset)
	log.Printf("HTTP read took %s", time.Since(startRead))

	if n != frame_size {
		return nil, fmt.Errorf("Tried to read %d bytes but got %d instead", frame_size, n)
	}

	width, height := int(mov.Trak.Tkhd.Width), int(mov.Trak.Tkhd.Height)

	startDecode := time.Now()
	img, err := prores.DecodeProRes(buf, width, height)
	log.Printf("Prores decode took %s", time.Since(startDecode))

	return img, err

}
<commit_msg>Added version string.<commit_after>package lazyquicktime

import "fmt"
import "image"
import "errors"

import "time"
import "log"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/go-prores-ffmpeg"

const Version = "v0.1.0"

type LazyQuicktime struct {
	file lazyfs.FileSource
	Tree quicktime.AtomArray
	Trak quicktime.TRAKAtom
	Stbl *quicktime.STBLAtom
	Mvhd quicktime.MVHDAtom

	FileSize int64
}

func LoadMovMetadata(file lazyfs.FileSource) (*LazyQuicktime, error) {

	mov := &LazyQuicktime{file: file}

	sz, err := file.FileSize()
	if sz < 0 || err != nil {
		return mov, fmt.Errorf("Unable to retrieve file size.")
	}

	mov.FileSize = sz

	set_eagerload := func(conf *quicktime.BuildTreeConfig) {
		conf.EagerloadTypes = []string{"moov"}
	}

	//fmt.Println("Reading Mov of size ", mov.FileSize)
	tree, err := quicktime.BuildTree(file, uint64(mov.FileSize), set_eagerload)

	if err != nil {
		return mov, err
	}
	mov.Tree = tree

	moov := mov.Tree.FindAtom("moov")
	if moov == nil {
		return mov, errors.New("Can't find MOOV atom")
	}

	mvhd := moov.FindAtom("mvhd")
	if mvhd == nil {
		return mov, errors.New("Couldn't find MVHD in the moov atom")
	}
	mov.Mvhd, _ = quicktime.ParseMVHD(mvhd)

	tracks := moov.FindAtoms("trak")
	if tracks == nil || len(tracks) == 0 {
		return mov, errors.New("Couldn't find any TRAKs in the MOOV")
	}

	var track *quicktime.Atom = nil
	for i, t := range tracks {
		mdia := t.FindAtom("mdia")
		if mdia == nil {
			fmt.Println("No mdia track", i)
			continue
		}

		minf := mdia.FindAtom("minf")
		if minf == nil {
			fmt.Println("No minf track", i)
			continue
		}

		if minf.FindAtom("vmhd") != nil {
			track = t
			break
		}
	}

	if track == nil {
		return mov, errors.New("Couldn't identify the Video track")
	}

	mov.Trak, err = quicktime.ParseTRAK(track)
	if err != nil {
		return mov, errors.New(fmt.Sprintf("Unable to parse TRAK atom: %s", err.Error()))
	}

	mov.Stbl = &mov.Trak.Mdia.Minf.Stbl // Just an alias

	return mov, nil
}

func (mov *LazyQuicktime) NumFrames() int {
	return mov.Stbl.NumFrames()
}

func (mov *LazyQuicktime) Duration() float32 {
	return mov.Mvhd.Duration()
}

func (mov *LazyQuicktime) ExtractFrame(frame int) (image.Image, error) {

	frame_offset, frame_size, _ := mov.Stbl.SampleOffsetSize(frame)

	//fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

	buf := make([]byte, frame_size)

	if buf == nil {
		return nil, fmt.Errorf("Couldn't make buffer of size %d", frame_size)
	}

	startRead := time.Now()
	n, _ := mov.file.ReadAt(buf, frame_offset)
	log.Printf("HTTP read took %s", time.Since(startRead))

	if n != frame_size {
		return nil, fmt.Errorf("Tried to read %d bytes but got %d instead", frame_size, n)
	}

	width, height := int(mov.Trak.Tkhd.Width), int(mov.Trak.Tkhd.Height)

	startDecode := time.Now()
	img, err := prores.DecodeProRes(buf, width, height)
	log.Printf("Prores decode took %s", time.Since(startDecode))

	return img, err

}
<|endoftext|>


<commit_before>package daemon

import (
	"github.com/docker/docker/pkg/graphdb"
	"github.com/docker/docker/pkg/truncindex"
	"os"
	"path"
	"testing"
)

//
// https://github.com/docker/docker/issues/8069
//

func TestGet(t *testing.T) {
	c1 := &Container{
		CommonContainer: CommonContainer{
			ID:   "5a4ff6a163ad4533d22d69a2b8960bf7fafdcba06e72d2febdba229008b0bf57",
			Name: "tender_bardeen",
		},
	}

	c2 := &Container{
		CommonContainer: CommonContainer{
			ID:   "3cdbd1aa394fd68559fd1441d6eff2ab7c1e6363582c82febfaa8045df3bd8de",
			Name: "drunk_hawking",
		},
	}

	c3 := &Container{
		CommonContainer: CommonContainer{
			ID:   "3cdbd1aa394fd68559fd1441d6eff2abfafdcba06e72d2febdba229008b0bf57",
			Name: "3cdbd1aa",
		},
	}

	c4 := &Container{
		CommonContainer: CommonContainer{
			ID:   "75fb0b800922abdbef2d27e60abcdfaf7fb0698b2a96d22d3354da361a6ff4a5",
			Name: "5a4ff6a163ad4533d22d69a2b8960bf7fafdcba06e72d2febdba229008b0bf57",
		},
	}

	c5 := &Container{
		CommonContainer: CommonContainer{
			ID:   "d22d69a2b8960bf7fafdcba06e72d2febdba960bf7fafdcba06e72d2f9008b060b",
			Name: "d22d69a2b896",
		},
	}

	store := &contStore{
		s: map[string]*Container{
			c1.ID: c1,
			c2.ID: c2,
			c3.ID: c3,
			c4.ID: c4,
			c5.ID: c5,
		},
	}

	index := truncindex.NewTruncIndex([]string{})
	index.Add(c1.ID)
	index.Add(c2.ID)
	index.Add(c3.ID)
	index.Add(c4.ID)
	index.Add(c5.ID)

	daemonTestDbPath := path.Join(os.TempDir(), "daemon_test.db")
	graph, err := graphdb.NewSqliteConn(daemonTestDbPath)
	if err != nil {
		t.Fatalf("Failed to create daemon test sqlite database at %s", daemonTestDbPath)
	}
	graph.Set(c1.Name, c1.ID)
	graph.Set(c2.Name, c2.ID)
	graph.Set(c3.Name, c3.ID)
	graph.Set(c4.Name, c4.ID)
	graph.Set(c5.Name, c5.ID)

	daemon := &Daemon{
		containers:     store,
		idIndex:        index,
		containerGraph: graph,
	}

	if container, _ := daemon.Get("3cdbd1aa394fd68559fd1441d6eff2ab7c1e6363582c82febfaa8045df3bd8de"); container != c2 {
		t.Fatal("Should explicitly match full container IDs")
	}

	if container, _ := daemon.Get("75fb0b8009"); container != c4 {
		t.Fatal("Should match a partial ID")
	}

	if container, _ := daemon.Get("drunk_hawking"); container != c2 {
		t.Fatal("Should match a full name")
	}

	// c3.Name is a partial match for both c3.ID and c2.ID
	if c, _ := daemon.Get("3cdbd1aa"); c != c3 {
		t.Fatal("Should match a full name even though it collides with another container's ID")
	}

	if container, _ := daemon.Get("d22d69a2b896"); container != c5 {
		t.Fatal("Should match a container where the provided prefix is an exact match to the it's name, and is also a prefix for it's ID")
	}

	if _, err := daemon.Get("3cdbd1"); err == nil {
		t.Fatal("Should return an error when provided a prefix that partially matches multiple container ID's")
	}

	if _, err := daemon.Get("nothing"); err == nil {
		t.Fatal("Should return an error when provided a prefix that is neither a name or a partial match to an ID")
	}

	os.Remove(daemonTestDbPath)
}
<commit_msg>Add regression test to make sure we can load old containers with volumes.<commit_after>package daemon

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"testing"

	"github.com/docker/docker/pkg/graphdb"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/pkg/truncindex"
	"github.com/docker/docker/volume"
)

//
// https://github.com/docker/docker/issues/8069
//

func TestGet(t *testing.T) {
	c1 := &Container{
		CommonContainer: CommonContainer{
			ID:   "5a4ff6a163ad4533d22d69a2b8960bf7fafdcba06e72d2febdba229008b0bf57",
			Name: "tender_bardeen",
		},
	}

	c2 := &Container{
		CommonContainer: CommonContainer{
			ID:   "3cdbd1aa394fd68559fd1441d6eff2ab7c1e6363582c82febfaa8045df3bd8de",
			Name: "drunk_hawking",
		},
	}

	c3 := &Container{
		CommonContainer: CommonContainer{
			ID:   "3cdbd1aa394fd68559fd1441d6eff2abfafdcba06e72d2febdba229008b0bf57",
			Name: "3cdbd1aa",
		},
	}

	c4 := &Container{
		CommonContainer: CommonContainer{
			ID:   "75fb0b800922abdbef2d27e60abcdfaf7fb0698b2a96d22d3354da361a6ff4a5",
			Name: "5a4ff6a163ad4533d22d69a2b8960bf7fafdcba06e72d2febdba229008b0bf57",
		},
	}

	c5 := &Container{
		CommonContainer: CommonContainer{
			ID:   "d22d69a2b8960bf7fafdcba06e72d2febdba960bf7fafdcba06e72d2f9008b060b",
			Name: "d22d69a2b896",
		},
	}

	store := &contStore{
		s: map[string]*Container{
			c1.ID: c1,
			c2.ID: c2,
			c3.ID: c3,
			c4.ID: c4,
			c5.ID: c5,
		},
	}

	index := truncindex.NewTruncIndex([]string{})
	index.Add(c1.ID)
	index.Add(c2.ID)
	index.Add(c3.ID)
	index.Add(c4.ID)
	index.Add(c5.ID)

	daemonTestDbPath := path.Join(os.TempDir(), "daemon_test.db")
	graph, err := graphdb.NewSqliteConn(daemonTestDbPath)
	if err != nil {
		t.Fatalf("Failed to create daemon test sqlite database at %s", daemonTestDbPath)
	}
	graph.Set(c1.Name, c1.ID)
	graph.Set(c2.Name, c2.ID)
	graph.Set(c3.Name, c3.ID)
	graph.Set(c4.Name, c4.ID)
	graph.Set(c5.Name, c5.ID)

	daemon := &Daemon{
		containers:     store,
		idIndex:        index,
		containerGraph: graph,
	}

	if container, _ := daemon.Get("3cdbd1aa394fd68559fd1441d6eff2ab7c1e6363582c82febfaa8045df3bd8de"); container != c2 {
		t.Fatal("Should explicitly match full container IDs")
	}

	if container, _ := daemon.Get("75fb0b8009"); container != c4 {
		t.Fatal("Should match a partial ID")
	}

	if container, _ := daemon.Get("drunk_hawking"); container != c2 {
		t.Fatal("Should match a full name")
	}

	// c3.Name is a partial match for both c3.ID and c2.ID
	if c, _ := daemon.Get("3cdbd1aa"); c != c3 {
		t.Fatal("Should match a full name even though it collides with another container's ID")
	}

	if container, _ := daemon.Get("d22d69a2b896"); container != c5 {
		t.Fatal("Should match a container where the provided prefix is an exact match to the it's name, and is also a prefix for it's ID")
	}

	if _, err := daemon.Get("3cdbd1"); err == nil {
		t.Fatal("Should return an error when provided a prefix that partially matches multiple container ID's")
	}

	if _, err := daemon.Get("nothing"); err == nil {
		t.Fatal("Should return an error when provided a prefix that is neither a name or a partial match to an ID")
	}

	os.Remove(daemonTestDbPath)
}

func TestLoadWithVolume(t *testing.T) {
	tmp, err := ioutil.TempDir("", "docker-daemon-test-")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmp)

	containerId := "d59df5276e7b219d510fe70565e0404bc06350e0d4b43fe961f22f339980170e"
	containerPath := filepath.Join(tmp, containerId)
	if err = os.MkdirAll(containerPath, 0755); err != nil {
		t.Fatal(err)
	}

	hostVolumeId := stringid.GenerateRandomID()
	volumePath := filepath.Join(tmp, "vfs", "dir", hostVolumeId)

	config := `{"State":{"Running":true,"Paused":false,"Restarting":false,"OOMKilled":false,"Dead":false,"Pid":2464,"ExitCode":0,
"Error":"","StartedAt":"2015-05-26T16:48:53.869308965Z","FinishedAt":"0001-01-01T00:00:00Z"},
"ID":"d59df5276e7b219d510fe70565e0404bc06350e0d4b43fe961f22f339980170e","Created":"2015-05-26T16:48:53.7987917Z","Path":"top",
"Args":[],"Config":{"Hostname":"d59df5276e7b","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"",
"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":true,"OpenStdin":true,
"StdinOnce":false,"Env":null,"Cmd":["top"],"Image":"ubuntu:latest","Volumes":null,"WorkingDir":"","Entrypoint":null,
"NetworkDisabled":false,"MacAddress":"","OnBuild":null,"Labels":{}},"Image":"07f8e8c5e66084bef8f848877857537ffe1c47edd01a93af27e7161672ad0e95",
"NetworkSettings":{"IPAddress":"172.17.0.1","IPPrefixLen":16,"MacAddress":"02:42:ac:11:00:01","LinkLocalIPv6Address":"fe80::42:acff:fe11:1",
"LinkLocalIPv6PrefixLen":64,"GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"Gateway":"172.17.42.1","IPv6Gateway":"","Bridge":"docker0","PortMapping":null,"Ports":{}},
"ResolvConfPath":"/var/lib/docker/containers/d59df5276e7b219d510fe70565e0404bc06350e0d4b43fe961f22f339980170e/resolv.conf",
"HostnamePath":"/var/lib/docker/containers/d59df5276e7b219d510fe70565e0404bc06350e0d4b43fe961f22f339980170e/hostname",
"HostsPath":"/var/lib/docker/containers/d59df5276e7b219d510fe70565e0404bc06350e0d4b43fe961f22f339980170e/hosts",
"LogPath":"/var/lib/docker/containers/d59df5276e7b219d510fe70565e0404bc06350e0d4b43fe961f22f339980170e/d59df5276e7b219d510fe70565e0404bc06350e0d4b43fe961f22f339980170e-json.log",
"Name":"/ubuntu","Driver":"aufs","ExecDriver":"native-0.2","MountLabel":"","ProcessLabel":"","AppArmorProfile":"","RestartCount":0,
"UpdateDns":false,"Volumes":{"/vol1":"%s"},"VolumesRW":{"/vol1":true},"AppliedVolumesFrom":null}`

	cfg := fmt.Sprintf(config, volumePath)
	if err = ioutil.WriteFile(filepath.Join(containerPath, "config.json"), []byte(cfg), 0644); err != nil {
		t.Fatal(err)
	}

	hostConfig := `{"Binds":[],"ContainerIDFile":"","LxcConf":[],"Memory":0,"MemorySwap":0,"CpuShares":0,"CpusetCpus":"",
"Privileged":false,"PortBindings":{},"Links":null,"PublishAllPorts":false,"Dns":null,"DnsSearch":null,"ExtraHosts":null,"VolumesFrom":null,
"Devices":[],"NetworkMode":"bridge","IpcMode":"","PidMode":"","CapAdd":null,"CapDrop":null,"RestartPolicy":{"Name":"no","MaximumRetryCount":0},
"SecurityOpt":null,"ReadonlyRootfs":false,"Ulimits":null,"LogConfig":{"Type":"","Config":null},"CgroupParent":""}`
	if err = ioutil.WriteFile(filepath.Join(containerPath, "hostconfig.json"), []byte(hostConfig), 0644); err != nil {
		t.Fatal(err)
	}

	if err = os.MkdirAll(volumePath, 0755); err != nil {
		t.Fatal(err)
	}

	daemon := &Daemon{
		repository: tmp,
		root:       tmp,
	}

	c, err := daemon.load(containerId)
	if err != nil {
		t.Fatal(err)
	}

	err = daemon.verifyOldVolumesInfo(c)
	if err != nil {
		t.Fatal(err)
	}

	if len(c.MountPoints) != 1 {
		t.Fatalf("Expected 1 volume mounted, was 0\n")
	}

	m := c.MountPoints["/vol1"]
	if m.Name != hostVolumeId {
		t.Fatalf("Expected mount name to be %s, was %s\n", hostVolumeId, m.Name)
	}

	if m.Destination != "/vol1" {
		t.Fatalf("Expected mount destination /vol1, was %s\n", m.Destination)
	}

	if !m.RW {
		t.Fatalf("Expected mount point to be RW but it was not\n")
	}

	if m.Driver != volume.DefaultDriverName {
		t.Fatalf("Expected mount driver local, was %s\n", m.Driver)
	}
}
<|endoftext|>


<commit_before>package rabbitsmpp

import (
	"github.com/streadway/amqp"
	"github.com/stretchr/testify/mock"
)

// MockConsumer is an autogenerated mock type for the Consumer type
type MockConsumer struct {
	mock.Mock
}

// Consume provides a mock function with given fields:
func (_m *MockConsumer) Consume() (<-chan Job, <-chan error, error) {
	ret := _m.Called()

	var r0 <-chan Job
	if rf, ok := ret.Get(0).(func() <-chan Job); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(chan Job)
		}
	}

	var r1 <-chan error
	if rf, ok := ret.Get(1).(func() <-chan error); ok {
		r1 = rf()
	} else {
		if ret.Get(1) != nil {
			r1 = ret.Get(1).(chan error)
		}
	}

	var r2 error
	if rf, ok := ret.Get(2).(func() error); ok {
		r2 = rf()
	} else {
		r2 = ret.Error(2)
	}

	return r0, r1, r2
}

// ID provides a mock function with given fields:
func (_m *MockConsumer) ID() string {
	ret := _m.Called()

	var r0 string
	if rf, ok := ret.Get(0).(func() string); ok {
		r0 = rf()
	} else {
		r0 = ret.Get(0).(string)
	}

	return r0
}

// Stop provides a mock function with given fields:
func (_m *MockConsumer) Stop() error {
	ret := _m.Called()

	var r0 error
	if rf, ok := ret.Get(0).(func() error); ok {
		r0 = rf()
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// MockChannel is an autogenerated mock type for the Channel type
type MockChannel struct {
	mock.Mock
}

// Close provides a mock function with given fields:
func (_m *MockChannel) Close() error {
	ret := _m.Called()

	var r0 error
	if rf, ok := ret.Get(0).(func() error); ok {
		r0 = rf()
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// Consume provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5, _a6
func (_m *MockChannel) Consume(_a0 string, _a1 string, _a2 bool, _a3 bool, _a4 bool, _a5 bool, _a6 amqp.Table) (<-chan amqp.Delivery, error) {
	ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5, _a6)

	var r0 <-chan amqp.Delivery
	if rf, ok := ret.Get(0).(func(string, string, bool, bool, bool, bool, amqp.Table) <-chan amqp.Delivery); ok {
		r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6)
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(<-chan amqp.Delivery)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string, string, bool, bool, bool, bool, amqp.Table) error); ok {
		r1 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// ExchangeDeclare provides a mock function with given fields: name, kind, durable, autoDelete, internal, noWait, args
func (_m *MockChannel) ExchangeDeclare(name string, kind string, durable bool, autoDelete bool, internal bool, noWait bool, args amqp.Table) error {
	ret := _m.Called(name, kind, durable, autoDelete, internal, noWait, args)

	var r0 error
	if rf, ok := ret.Get(0).(func(string, string, bool, bool, bool, bool, amqp.Table) error); ok {
		r0 = rf(name, kind, durable, autoDelete, internal, noWait, args)
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// Publish provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4
func (_m *MockChannel) Publish(_a0 string, _a1 string, _a2 bool, _a3 bool, _a4 amqp.Publishing) error {
	ret := _m.Called(_a0, _a1, _a2, _a3, _a4)

	var r0 error
	if rf, ok := ret.Get(0).(func(string, string, bool, bool, amqp.Publishing) error); ok {
		r0 = rf(_a0, _a1, _a2, _a3, _a4)
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// QueueBind provides a mock function with given fields: name, key, exchange, noWait, args
func (_m *MockChannel) QueueBind(name string, key string, exchange string, noWait bool, args amqp.Table) error {
	ret := _m.Called(name, key, exchange, noWait, args)

	var r0 error
	if rf, ok := ret.Get(0).(func(string, string, string, bool, amqp.Table) error); ok {
		r0 = rf(name, key, exchange, noWait, args)
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// QueueDeclare provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5
func (_m *MockChannel) QueueDeclare(_a0 string, _a1 bool, _a2 bool, _a3 bool, _a4 bool, _a5 amqp.Table) (amqp.Queue, error) {
	ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5)

	var r0 amqp.Queue
	if rf, ok := ret.Get(0).(func(string, bool, bool, bool, bool, amqp.Table) amqp.Queue); ok {
		r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5)
	} else {
		r0 = ret.Get(0).(amqp.Queue)
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string, bool, bool, bool, bool, amqp.Table) error); ok {
		r1 = rf(_a0, _a1, _a2, _a3, _a4, _a5)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// QueueInspect provides a mock function with given fields: _a0
func (_m *MockChannel) QueueInspect(_a0 string) (amqp.Queue, error) {
	ret := _m.Called(_a0)

	var r0 amqp.Queue
	if rf, ok := ret.Get(0).(func(string) amqp.Queue); ok {
		r0 = rf(_a0)
	} else {
		r0 = ret.Get(0).(amqp.Queue)
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string) error); ok {
		r1 = rf(_a0)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// MockPublisherClient is an autogenerated mock type for the PublisherClient type
type MockPublisherClient struct {
	mock.Mock
}

// Bind provides a mock function with given fields:
func (_m *MockPublisherClient) Bind() (chan *amqp.Error, error) {
	ret := _m.Called()

	var r0 chan *amqp.Error
	if rf, ok := ret.Get(0).(func() chan *amqp.Error); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(chan *amqp.Error)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func() error); ok {
		r1 = rf()
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// Channel provides a mock function with given fields:
func (_m *MockPublisherClient) Channel() (Channel, error) {
	ret := _m.Called()

	var r0 Channel
	if rf, ok := ret.Get(0).(func() Channel); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(Channel)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func() error); ok {
		r1 = rf()
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// Close provides a mock function with given fields:
func (_m *MockPublisherClient) Close() error {
	ret := _m.Called()

	var r0 error
	if rf, ok := ret.Get(0).(func() error); ok {
		r0 = rf()
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// QueueName provides a mock function with given fields:
func (_m *MockPublisherClient) QueueName() string {
	ret := _m.Called()

	var r0 string
	if rf, ok := ret.Get(0).(func() string); ok {
		r0 = rf()
	} else {
		r0 = ret.Get(0).(string)
	}

	return r0
}

// ReBind provides a mock function with given fields:
func (_m *MockPublisherClient) ReBind() (PublisherClient, chan *amqp.Error, error) {
	ret := _m.Called()

	var r0 PublisherClient
	if rf, ok := ret.Get(0).(func() PublisherClient); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(PublisherClient)
		}
	}

	var r1 chan *amqp.Error
	if rf, ok := ret.Get(1).(func() chan *amqp.Error); ok {
		r1 = rf()
	} else {
		if ret.Get(1) != nil {
			r1 = ret.Get(1).(chan *amqp.Error)
		}
	}

	var r2 error
	if rf, ok := ret.Get(2).(func() error); ok {
		r2 = rf()
	} else {
		r2 = ret.Error(2)
	}

	return r0, r1, r2
}
<commit_msg>Mock channell<commit_after>package rabbitsmpp

import (
	"github.com/streadway/amqp"
	"github.com/stretchr/testify/mock"
)

// MockConsumer is an autogenerated mock type for the Consumer type
type MockConsumer struct {
	mock.Mock
}

// Consume provides a mock function with given fields:
func (_m *MockConsumer) Consume() (<-chan Job, <-chan error, error) {
	ret := _m.Called()

	var r0 <-chan Job
	if rf, ok := ret.Get(0).(func() <-chan Job); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(chan Job)
		}
	}

	var r1 <-chan error
	if rf, ok := ret.Get(1).(func() <-chan error); ok {
		r1 = rf()
	} else {
		if ret.Get(1) != nil {
			r1 = ret.Get(1).(chan error)
		}
	}

	var r2 error
	if rf, ok := ret.Get(2).(func() error); ok {
		r2 = rf()
	} else {
		r2 = ret.Error(2)
	}

	return r0, r1, r2
}

// ID provides a mock function with given fields:
func (_m *MockConsumer) ID() string {
	ret := _m.Called()

	var r0 string
	if rf, ok := ret.Get(0).(func() string); ok {
		r0 = rf()
	} else {
		r0 = ret.Get(0).(string)
	}

	return r0
}

// Stop provides a mock function with given fields:
func (_m *MockConsumer) Stop() error {
	ret := _m.Called()

	var r0 error
	if rf, ok := ret.Get(0).(func() error); ok {
		r0 = rf()
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// MockChannel is an autogenerated mock type for the Channel type
type MockChannel struct {
	mock.Mock
}

// Close provides a mock function with given fields:
func (_m *MockChannel) Close() error {
	ret := _m.Called()

	var r0 error
	if rf, ok := ret.Get(0).(func() error); ok {
		r0 = rf()
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// Consume provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5, _a6
func (_m *MockChannel) Consume(_a0 string, _a1 string, _a2 bool, _a3 bool, _a4 bool, _a5 bool, _a6 amqp.Table) (<-chan amqp.Delivery, error) {
	ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5, _a6)

	var r0 <-chan amqp.Delivery
	if rf, ok := ret.Get(0).(func(string, string, bool, bool, bool, bool, amqp.Table) <-chan amqp.Delivery); ok {
		r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6)
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(<-chan amqp.Delivery)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string, string, bool, bool, bool, bool, amqp.Table) error); ok {
		r1 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// ExchangeDeclare provides a mock function with given fields: name, kind, durable, autoDelete, internal, noWait, args
func (_m *MockChannel) ExchangeDeclare(name string, kind string, durable bool, autoDelete bool, internal bool, noWait bool, args amqp.Table) error {
	ret := _m.Called(name, kind, durable, autoDelete, internal, noWait, args)

	var r0 error
	if rf, ok := ret.Get(0).(func(string, string, bool, bool, bool, bool, amqp.Table) error); ok {
		r0 = rf(name, kind, durable, autoDelete, internal, noWait, args)
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// Publish provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4
func (_m *MockChannel) Publish(_a0 string, _a1 string, _a2 bool, _a3 bool, _a4 amqp.Publishing) error {
	ret := _m.Called(_a0, _a1, _a2, _a3, _a4)

	var r0 error
	if rf, ok := ret.Get(0).(func(string, string, bool, bool, amqp.Publishing) error); ok {
		r0 = rf(_a0, _a1, _a2, _a3, _a4)
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// Qos provides a mock function with given fields: _a0, _a1, _a2
func (_m *MockChannel) Qos(_a0 int, _a1 int, _a2 bool) error {
	ret := _m.Called(_a0, _a1, _a2)

	var r0 error
	if rf, ok := ret.Get(0).(func(int, int, bool) error); ok {
		r0 = rf(_a0, _a1, _a2)
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// QueueBind provides a mock function with given fields: name, key, exchange, noWait, args
func (_m *MockChannel) QueueBind(name string, key string, exchange string, noWait bool, args amqp.Table) error {
	ret := _m.Called(name, key, exchange, noWait, args)

	var r0 error
	if rf, ok := ret.Get(0).(func(string, string, string, bool, amqp.Table) error); ok {
		r0 = rf(name, key, exchange, noWait, args)
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// QueueDeclare provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5
func (_m *MockChannel) QueueDeclare(_a0 string, _a1 bool, _a2 bool, _a3 bool, _a4 bool, _a5 amqp.Table) (amqp.Queue, error) {
	ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5)

	var r0 amqp.Queue
	if rf, ok := ret.Get(0).(func(string, bool, bool, bool, bool, amqp.Table) amqp.Queue); ok {
		r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5)
	} else {
		r0 = ret.Get(0).(amqp.Queue)
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string, bool, bool, bool, bool, amqp.Table) error); ok {
		r1 = rf(_a0, _a1, _a2, _a3, _a4, _a5)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// QueueInspect provides a mock function with given fields: _a0
func (_m *MockChannel) QueueInspect(_a0 string) (amqp.Queue, error) {
	ret := _m.Called(_a0)

	var r0 amqp.Queue
	if rf, ok := ret.Get(0).(func(string) amqp.Queue); ok {
		r0 = rf(_a0)
	} else {
		r0 = ret.Get(0).(amqp.Queue)
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string) error); ok {
		r1 = rf(_a0)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// MockPublisherClient is an autogenerated mock type for the PublisherClient type
type MockPublisherClient struct {
	mock.Mock
}

// Bind provides a mock function with given fields:
func (_m *MockPublisherClient) Bind() (chan *amqp.Error, error) {
	ret := _m.Called()

	var r0 chan *amqp.Error
	if rf, ok := ret.Get(0).(func() chan *amqp.Error); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(chan *amqp.Error)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func() error); ok {
		r1 = rf()
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// Channel provides a mock function with given fields:
func (_m *MockPublisherClient) Channel() (Channel, error) {
	ret := _m.Called()

	var r0 Channel
	if rf, ok := ret.Get(0).(func() Channel); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(Channel)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func() error); ok {
		r1 = rf()
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// Close provides a mock function with given fields:
func (_m *MockPublisherClient) Close() error {
	ret := _m.Called()

	var r0 error
	if rf, ok := ret.Get(0).(func() error); ok {
		r0 = rf()
	} else {
		r0 = ret.Error(0)
	}

	return r0
}

// QueueName provides a mock function with given fields:
func (_m *MockPublisherClient) QueueName() string {
	ret := _m.Called()

	var r0 string
	if rf, ok := ret.Get(0).(func() string); ok {
		r0 = rf()
	} else {
		r0 = ret.Get(0).(string)
	}

	return r0
}

// ReBind provides a mock function with given fields:
func (_m *MockPublisherClient) ReBind() (PublisherClient, chan *amqp.Error, error) {
	ret := _m.Called()

	var r0 PublisherClient
	if rf, ok := ret.Get(0).(func() PublisherClient); ok {
		r0 = rf()
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(PublisherClient)
		}
	}

	var r1 chan *amqp.Error
	if rf, ok := ret.Get(1).(func() chan *amqp.Error); ok {
		r1 = rf()
	} else {
		if ret.Get(1) != nil {
			r1 = ret.Get(1).(chan *amqp.Error)
		}
	}

	var r2 error
	if rf, ok := ret.Get(2).(func() error); ok {
		r2 = rf()
	} else {
		r2 = ret.Error(2)
	}

	return r0, r1, r2
}
<|endoftext|>


<commit_before>package reacher

import (
	"github.com/elisarver/reach/tag"
	"github.com/elisarver/reach/target"
)

// genReachTargets binds the appropriate function to generate a document
// to the ReachTargets func.
func ReachTargets(ls []target.Location, ds []*tag.Description) ([]string, error) {
	var output []string
	for _, l := range ls {
		resp, err := l.Retrieve()
		if err != nil {
			return []string{}, err
		}
		for _, d := range ds {
			output = append(output, tag.SelectMap(resp, d)...)
		}
	}
	return output, nil
}
<commit_msg>add a config option to reacher for reparenting<commit_after>package reacher

import (
	"github.com/elisarver/reach/tag"
	"github.com/elisarver/reach/target"
)

type config struct {
	Reparent bool
}

var Config = config{
	// Reparent represents the re-building of a URL by the originating host's URL.
	// When set, a relative URL becomes an absolute URL with the target.Location's
	// URL parts filling in missing values.
	Reparent: false,
}

func ReachTargets(ls []target.Location, ds []*tag.Description) ([]string, error) {
	var output []string
	for _, l := range ls {
		resp, err := l.Retrieve()
		if err != nil {
			return []string{}, err
		}
		for _, d := range ds {
			output = append(output, tag.SelectMap(resp, d)...)
		}
	}
	return output, nil
}
<|endoftext|>


<commit_before>package nfs

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/pborman/uuid"

	"github.com/portworx/kvdb"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"
)

const (
	Name         = "nfs"
	Type         = volume.File
	NfsDBKey     = "OpenStorageNFSKey"
	nfsMountPath = "/var/lib/openstorage/nfs/"
	nfsBlockFile = ".blockdevice"
)

// Implements the open storage volume interface.
type driver struct {
	*volume.DefaultEnumerator
	nfsServer string
	nfsPath   string
}

func copyFile(source string, dest string) (err error) {
	sourcefile, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourcefile.Close()

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}

	defer destfile.Close()

	_, err = io.Copy(destfile, sourcefile)
	if err == nil {
		sourceinfo, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, sourceinfo.Mode())
		}

	}

	return
}

func copyDir(source string, dest string) (err error) {
	// get properties of source dir
	sourceinfo, err := os.Stat(source)
	if err != nil {
		return err
	}

	// create dest dir

	err = os.MkdirAll(dest, sourceinfo.Mode())
	if err != nil {
		return err
	}

	directory, _ := os.Open(source)

	objects, err := directory.Readdir(-1)

	for _, obj := range objects {

		sourcefilepointer := source + "/" + obj.Name()

		destinationfilepointer := dest + "/" + obj.Name()

		if obj.IsDir() {
			// create sub-directories - recursively
			err = copyDir(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		} else {
			// perform copy
			err = copyFile(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		}

	}
	return
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
	path, ok := params["path"]
	if !ok {
		return nil, errors.New("No NFS path provided")
	}

	server, ok := params["server"]
	if !ok {
		log.Printf("No NFS server provided, will attempt to bind mount %s", path)
	} else {
		log.Printf("NFS driver initializing with %s:%s ", server, path)
	}

	inst := &driver{
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		nfsServer:         server,
		nfsPath:           path}

	err := os.MkdirAll(nfsMountPath, 0744)
	if err != nil {
		return nil, err
	}

	// Mount the nfs server locally on a unique path.
	syscall.Unmount(nfsMountPath, 0)
	if server != "" {
		err = syscall.Mount(":"+inst.nfsPath, nfsMountPath, "nfs", 0, "nolock,addr="+inst.nfsServer)
	} else {
		err = syscall.Mount(inst.nfsPath, nfsMountPath, "", syscall.MS_BIND, "")
	}
	if err != nil {
		log.Printf("Unable to mount %s:%s at %s (%+v)", inst.nfsServer, inst.nfsPath, nfsMountPath, err)
		return nil, err
	}

	log.Println("NFS initialized and driver mounted at: ", nfsMountPath)
	return inst, nil
}

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() volume.DriverType {
	return Type
}

// Status diagnostic information
func (d *driver) Status() [][2]string {
	return [][2]string{}
}

func (d *driver) Create(locator api.VolumeLocator, opt *api.CreateOptions, spec *api.VolumeSpec) (api.VolumeID, error) {
	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")

	// Create a directory on the NFS server with this UUID.
	err := os.MkdirAll(nfsMountPath+volumeID, 0744)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	f, err := os.Create(nfsMountPath + volumeID + nfsBlockFile)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	defer f.Close()

	err = f.Truncate(int64(spec.Size))
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	v := &api.Volume{
		ID:         api.VolumeID(volumeID),
		Locator:    locator,
		Ctime:      time.Now(),
		Spec:       spec,
		LastScan:   time.Now(),
		Format:     "nfs",
		State:      api.VolumeAvailable,
		DevicePath: path.Join(nfsMountPath, string(volumeID), nfsBlockFile),
	}

	err = d.CreateVol(v)
	if err != nil {
		return api.BadVolumeID, err
	}

	err = d.UpdateVol(v)

	return v.ID, err
}

func (d *driver) Delete(volumeID api.VolumeID) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	// Delete the simulated block volume
	os.Remove(v.DevicePath)

	// Delete the directory on the nfs server.
	os.RemoveAll(path.Join(nfsMountPath, string(volumeID)))

	err = d.DeleteVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	return nil
}

func (d *driver) Mount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}
	syscall.Unmount(mountpath, 0)
	err = syscall.Mount(path.Join(nfsMountPath, string(volumeID)), mountpath, string(v.Spec.Format), syscall.MS_BIND, "")
	if err != nil {
		log.Printf("Cannot mount %s at %s because %+v",
			path.Join(nfsMountPath, string(volumeID)), mountpath, err)
		return err
	}

	v.AttachPath = mountpath
	err = d.UpdateVol(v)

	return err
}

func (d *driver) Unmount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if v.AttachPath == "" {
		return fmt.Errorf("Device %v not mounted", volumeID)
	}
	err = syscall.Unmount(v.AttachPath, 0)
	if err != nil {
		return err
	}
	v.AttachPath = ""
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Snapshot(volumeID api.VolumeID, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	volIDs := make([]api.VolumeID, 1)
	volIDs[0] = volumeID
	vols, err := d.Inspect(volIDs)
	if err != nil {
		return api.BadVolumeID, nil
	}

	newVolumeID, err := d.Create(locator, nil, vols[0].Spec)
	if err != nil {
		return api.BadVolumeID, nil
	}

	// NFS does not support snapshots, so just copy the files.
	err = copyDir(nfsMountPath+string(volumeID), nfsMountPath+string(newVolumeID))
	if err != nil {
		d.Delete(newVolumeID)
		return api.BadVolumeID, nil
	}

	err = copyFile(nfsMountPath+string(volumeID)+nfsBlockFile, nfsMountPath+string(newVolumeID)+nfsBlockFile)
	if err != nil {
		d.Delete(newVolumeID)
		return api.BadVolumeID, nil
	}

	return newVolumeID, nil
}

func (d *driver) Attach(volumeID api.VolumeID) (string, error) {
	return path.Join(nfsMountPath, string(volumeID), nfsBlockFile), nil
}

func (d *driver) Format(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Detach(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Stats(volumeID api.VolumeID) (api.VolumeStats, error) {
	return api.VolumeStats{}, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID api.VolumeID) (api.VolumeAlerts, error) {
	return api.VolumeAlerts{}, volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	log.Printf("%s Shutting down", Name)
	syscall.Unmount(nfsMountPath, 0)
}

func init() {
	// Register ourselves as an openstorage volume driver.
	volume.Register(Name, Init)
}
<commit_msg>Disable block cloning over NFS<commit_after>package nfs

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/pborman/uuid"

	"github.com/portworx/kvdb"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"
)

const (
	Name         = "nfs"
	Type         = volume.File
	NfsDBKey     = "OpenStorageNFSKey"
	nfsMountPath = "/var/lib/openstorage/nfs/"
	nfsBlockFile = ".blockdevice"
)

// Implements the open storage volume interface.
type driver struct {
	*volume.DefaultEnumerator
	nfsServer string
	nfsPath   string
}

func copyFile(source string, dest string) (err error) {
	sourcefile, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourcefile.Close()

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}

	defer destfile.Close()

	_, err = io.Copy(destfile, sourcefile)
	if err == nil {
		sourceinfo, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, sourceinfo.Mode())
		}

	}

	return
}

func copyDir(source string, dest string) (err error) {
	// get properties of source dir
	sourceinfo, err := os.Stat(source)
	if err != nil {
		return err
	}

	// create dest dir

	err = os.MkdirAll(dest, sourceinfo.Mode())
	if err != nil {
		return err
	}

	directory, _ := os.Open(source)

	objects, err := directory.Readdir(-1)

	for _, obj := range objects {

		sourcefilepointer := source + "/" + obj.Name()

		destinationfilepointer := dest + "/" + obj.Name()

		if obj.IsDir() {
			// create sub-directories - recursively
			err = copyDir(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		} else {
			// perform copy
			err = copyFile(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		}

	}
	return
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
	path, ok := params["path"]
	if !ok {
		return nil, errors.New("No NFS path provided")
	}

	server, ok := params["server"]
	if !ok {
		log.Printf("No NFS server provided, will attempt to bind mount %s", path)
	} else {
		log.Printf("NFS driver initializing with %s:%s ", server, path)
	}

	inst := &driver{
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		nfsServer:         server,
		nfsPath:           path}

	err := os.MkdirAll(nfsMountPath, 0744)
	if err != nil {
		return nil, err
	}

	// Mount the nfs server locally on a unique path.
	syscall.Unmount(nfsMountPath, 0)
	if server != "" {
		err = syscall.Mount(":"+inst.nfsPath, nfsMountPath, "nfs", 0, "nolock,addr="+inst.nfsServer)
	} else {
		err = syscall.Mount(inst.nfsPath, nfsMountPath, "", syscall.MS_BIND, "")
	}
	if err != nil {
		log.Printf("Unable to mount %s:%s at %s (%+v)", inst.nfsServer, inst.nfsPath, nfsMountPath, err)
		return nil, err
	}

	log.Println("NFS initialized and driver mounted at: ", nfsMountPath)
	return inst, nil
}

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() volume.DriverType {
	return Type
}

// Status diagnostic information
func (d *driver) Status() [][2]string {
	return [][2]string{}
}

func (d *driver) Create(locator api.VolumeLocator, opt *api.CreateOptions, spec *api.VolumeSpec) (api.VolumeID, error) {
	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")

	// Create a directory on the NFS server with this UUID.
	err := os.MkdirAll(nfsMountPath+volumeID, 0744)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	f, err := os.Create(nfsMountPath + volumeID + nfsBlockFile)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	defer f.Close()

	err = f.Truncate(int64(spec.Size))
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	v := &api.Volume{
		ID:         api.VolumeID(volumeID),
		Locator:    locator,
		Ctime:      time.Now(),
		Spec:       spec,
		LastScan:   time.Now(),
		Format:     "nfs",
		State:      api.VolumeAvailable,
		DevicePath: path.Join(nfsMountPath, string(volumeID), nfsBlockFile),
	}

	err = d.CreateVol(v)
	if err != nil {
		return api.BadVolumeID, err
	}

	err = d.UpdateVol(v)

	return v.ID, err
}

func (d *driver) Delete(volumeID api.VolumeID) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	// Delete the simulated block volume
	os.Remove(v.DevicePath)

	// Delete the directory on the nfs server.
	os.RemoveAll(path.Join(nfsMountPath, string(volumeID)))

	err = d.DeleteVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	return nil
}

func (d *driver) Mount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}
	syscall.Unmount(mountpath, 0)
	err = syscall.Mount(path.Join(nfsMountPath, string(volumeID)), mountpath, string(v.Spec.Format), syscall.MS_BIND, "")
	if err != nil {
		log.Printf("Cannot mount %s at %s because %+v",
			path.Join(nfsMountPath, string(volumeID)), mountpath, err)
		return err
	}

	v.AttachPath = mountpath
	err = d.UpdateVol(v)

	return err
}

func (d *driver) Unmount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if v.AttachPath == "" {
		return fmt.Errorf("Device %v not mounted", volumeID)
	}
	err = syscall.Unmount(v.AttachPath, 0)
	if err != nil {
		return err
	}
	v.AttachPath = ""
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Snapshot(volumeID api.VolumeID, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	volIDs := make([]api.VolumeID, 1)
	volIDs[0] = volumeID
	vols, err := d.Inspect(volIDs)
	if err != nil {
		return api.BadVolumeID, nil
	}

	newVolumeID, err := d.Create(locator, nil, vols[0].Spec)
	if err != nil {
		return api.BadVolumeID, nil
	}

	// NFS does not support snapshots, so just copy the files.
	err = copyDir(nfsMountPath+string(volumeID), nfsMountPath+string(newVolumeID))
	if err != nil {
		d.Delete(newVolumeID)
		return api.BadVolumeID, nil
	}

	return newVolumeID, nil
}

func (d *driver) Attach(volumeID api.VolumeID) (string, error) {
	return path.Join(nfsMountPath, string(volumeID), nfsBlockFile), nil
}

func (d *driver) Format(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Detach(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Stats(volumeID api.VolumeID) (api.VolumeStats, error) {
	return api.VolumeStats{}, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID api.VolumeID) (api.VolumeAlerts, error) {
	return api.VolumeAlerts{}, volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	log.Printf("%s Shutting down", Name)
	syscall.Unmount(nfsMountPath, 0)
}

func init() {
	// Register ourselves as an openstorage volume driver.
	volume.Register(Name, Init)
}
<|endoftext|>