<commit_before>package logfilter

import (
	"testing"
)

func TestRepoMirrorLogExample(t *testing.T) {
	in := `172.31.30.229 - - [19/Jun/2015:09:24:24 +0000] "GET /v1/images/f467d023d63178a6686daab33049b7fec024f88e5b64898e9c89dafaaa4e1d8a/ancestry HTTP/1.1" 200 1836 "-" "docker/1.5.0 go/go1.3.3 git-commit/a8a31ef-dirty kernel/3.19.3 os/linux arch/amd64"`

	out, ok := extractAccEntry(in)

	if !ok {
		t.Fatal("failed to extract values")
	}

	if out.Status != 200 {
		t.Errorf("expected status %d but got %d\n", 200, out.Status)
	}
	// TODO:
}

func TestMethodeAPIExample(t *testing.T) {
	in := `127.0.0.1 - - [21/Apr/2015:12:15:34 +0000] "GET /eom-file/all/e09b49d6-e1fa-11e4-bb7f-00144feab7de HTTP/1.1" 200 53706 919 919`

	out, ok := extractAccEntry(in)
	if !ok {
		t.Fatal("failed to extract values")
	}
	if out.Status != 200 {
		t.Errorf("expected status %d but got %d\n", 200, out.Status)
	}
	// TODO:
}

func TestCmsNotifierPostExample(t *testing.T) {
	in := `172.17.42.1 -  -  [24/Jun/2015:11:09:36 +0000] "POST /notify HTTP/1.1" 500 - "-" "curl/7.42.0" 2197`
	out, ok := extractAccEntry(in)
	t.Logf("out status value %v", out.Status)
	if !ok {
		t.Fatal("failed to extract values")
	}
	if out.Status != 500 {
		t.Errorf("expected status %d but got %d\n", 500, out.Status)
	}
	// TODO:
}

func TestExtractAppEntry(t *testing.T) {
	var tests = []struct {
		message string
		level   string
	}{
		{`ERROR [2015-08-07 09:03:45,581] kafka.utils.Utils$: transaction_id=tid_lYpxZctRHb_kafka_bridge fetching topic metadata for topics [Set(NativeCmsPublicationEvents)] from broker [ArrayBuffer(id:0,host:172.23.219.136,port:9092)] failed|[dw-3910 - POST /notify]! kafka.common.KafkaException: fetching topic metadata for topics [Set(NativeCmsPublicationEvents)] from broker [ArrayBuffer(id:0,host:172.23.219.136,port:9092)] failed|! at kafka.client.ClientUtils$.fetchTopicMetadata(ClientUtils.scala:67) ~[app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.BrokerPartitionInfo.updateInfo(BrokerPartitionInfo.scala:82) ~[app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.async.DefaultEventHandler$$anonfun$handle$2.apply$mcV$sp(DefaultEventHandler.scala:78) ~[app.jar:0.0.1-SNAPSHOT]|! at kafka.utils.Utils$.swallow(Utils.scala:167) [app.jar:0.0.1-SNAPSHOT]|! at kafka.utils.Logging$class.swallowError(Logging.scala:106) [app.jar:0.0.1-SNAPSHOT]|! at kafka.utils.Utils$.swallowError(Utils.scala:46) [app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.async.DefaultEventHandler.handle(DefaultEventHandler.scala:78) [app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.Producer.send(Producer.scala:76) [app.jar:0.0.1-SNAPSHOT]|! at kafka.javaapi.producer.Producer.send(Producer.scala:33) [app.jar:0.0.1-SNAPSHOT]|! at com.ft.cmsnotifier.service.KafkaMessageProducer.produceNotifyEvent(KafkaMessageProducer.java:30) [app.jar:0.0.1-SNAPSHOT]|! at com.ft.cmsnotifier.resources.CmsNotifierResource.produceEventForContent(CmsNotifierResource.java:59) [app.jar:0.0.1-SNAPSHOT]|! at com.ft.cmsnotifier.resources.CmsNotifierResource.importContent(CmsNotifierResource.java:43) [app.jar:0.0.1-SNAPSHOT]|! at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) ~[na:na]|! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_51]|! at java.lang.reflect.Method.invoke(Method.java:497) ~[na:1.8.0_51]|! at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) [app.jar:0.0.1-SNAPSHOT]|! at com.sun.je`, `ERROR`},
		{`INFO  [2015-08-13 07:59:00,019] com.ft.platform.dropwizard.HealthCheckPageData:  event="advancedHealthCheck", action="detail", name="BinaryIngesterService", checkName="CanConnectToZooKeeper", ok="true", checkOutput="ZooKeeper instance responded, at least 1 are up."|[dw-19 - GET /__health]`, `INFO`},
		{`DEBUG [2015-08-12 14:14:34,421] io.dropwizard.setup.AdminEnvironment:  health checks = [CanConnectToKafka, CanConnectToZooKeeper, KafkaIsNotLagging, binary-writer ping, binary-writer version, deadlocks]|[main]`, `DEBUG`},
		{`WARN  [2015-08-12 14:14:35,004] kafka.consumer.ZookeeperConsumerConnector:  [content_d414a887c7ca-1439388871813-b0f5b10f], No broker partitions consumed by consumer thread content_d414a887c7ca-1439388871813-b0f5b10f-1 for topic CmsPublicationEvents|[pool-5-thread-1]`, `WARN`},
		{`INFO  [2015-08-13 08:01:28,393] com.ft.platform.dropwizard.HealthCheckPageData:  event="advancedHealthCheck", action="detail", name="CMSNotifierApplication", checkName="Can connect to Kafka on topic: NativeCmsPublicationEvents", ok="true", checkOutput="Kafka is connected and topic is present"|[dw-19 - GET /__health]`, `INFO`},
	}

	for _, test := range tests {
		appEntry, ok := extractAppEntry(test.message)
		if !ok {
			t.Fatalf("failed to extract values '%s'", test.message)
		}
		if appEntry.Level != test.level {
			t.Errorf("message: %s\nexpected level %s, actual level %s", test.message, test.level, appEntry.Level)
		}
	}
}

func TestExtractPamEntity(t *testing.T) {
	var tests = []struct {
		message       string
		UUID          string
		TransactionID string
		PublishDate   string
		PublishOk     string
		Duration      string
		Endpoint      string
	}{
		{`[splunkMetrics] 2015/12/21 10:01:37.336610 UUID=08d30fb4-a7b3-11e5-955c-1e1d6de94879 transaction_id=tid_28pbiavoqs publishDate=1450692093737000000 publishOk=true duration=6 endpoint=content`,
			"08d30fb4-a7b3-11e5-955c-1e1d6de94879",
			"tid_28pbiavoqs",
			"1450692093737000000",
			"true",
			"6",
			"content"},
		{`[splunkMetrics] 2015/12/21 10:01:37.336610 UUID=08d30fb4-a7b3-11e5-955c-1e1d6de94879 transaction_id=tid_28pbiavoqs publishDate=1450692093737000000 publishOk=true duration=6 endpoint=notifications-push`,
			"08d30fb4-a7b3-11e5-955c-1e1d6de94879",
			"tid_28pbiavoqs",
			"1450692093737000000",
			"true",
			"6",
			"notifications-push"},
	}

	for _, test := range tests {
		pamEntity, ok := extractPamEntity(test.message)
		if !ok {
			t.Fatalf("failed to extract values '%s'", test.message)
		}
		if pamEntity.UUID != test.UUID {
			t.Errorf("message: %s\nexpected UUID %s, actual UUID %s", test.message, test.UUID, pamEntity.UUID)
		}
		if pamEntity.TransactionID != test.TransactionID {
			t.Errorf("message: %s\nexpected transaction_id %s, actual transaction_id %s", test.message, test.TransactionID, pamEntity.TransactionID)
		}
		if pamEntity.PublishDate != test.PublishDate {
			t.Errorf("message: %s\nexpected publishDate %s, actual publishDate %s", test.message, test.PublishDate, pamEntity.PublishDate)
		}
		if pamEntity.PublishOk != test.PublishOk {
			t.Errorf("message: %s\nexpected publishOk %s, actual publishOk %s", test.message, test.PublishOk, pamEntity.PublishOk)
		}
		if pamEntity.Duration != test.Duration {
			t.Errorf("message: %s\nexpected duration %s, actual duration %s", test.message, test.Duration, pamEntity.Duration)
		}
		if pamEntity.Endpoint != test.Endpoint {
			t.Errorf("message: %s\nexpected endpoint %s, actual endpoint %s", test.message, test.Endpoint, pamEntity.Endpoint)
		}
	}
}

func TestExtractVarnishEntity(t *testing.T) {
	var tests = []struct {
		message   string
		AuthUser  string
		URI       string
		Status    string
		Resptime  string
		UserAgent string
	}{
		{`82.136.1.214, 172.24.88.199 next 14/Jun/2016:08:24:42 /__enriched-content-read-api/enrichedcontent/409ba29e-b7f8-417f-9847-f3332aa064a6 200 65526 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"`,
			"next",
			"/__enriched-content-read-api/enrichedcontent/409ba29e-b7f8-417f-9847-f3332aa064a6",
			"200",
			"65526",
			"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"},
	}

	for _, test := range tests {
		varnishEntity, ok := extractVarnishEntity(test.message)
		if !ok {
			t.Fatalf("failed to extract values '%s'", test.message)
		}
		if varnishEntity.URI != test.URI {
			t.Errorf("message: %s\nexpected uri %s, actual uri %s", test.message, test.URI, varnishEntity.URI)
		}
		if varnishEntity.Status != test.Status {
			t.Errorf("message: %s\nexpected status %s, actual status %s", test.message, test.Status, varnishEntity.Status)
		}
		if varnishEntity.Resptime != test.Resptime {
			t.Errorf("message: %s\nexpected respTime %s, actual respTime %s", test.message, test.Resptime, varnishEntity.Resptime)
		}
	}
}
<commit_msg>Use github.com/stretchr/testify/assert<commit_after>package logfilter

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestRepoMirrorLogExample(t *testing.T) {
	assert := assert.New(t)

	in := `172.31.30.229 - - [19/Jun/2015:09:24:24 +0000] "GET /v1/images/f467d023d63178a6686daab33049b7fec024f88e5b64898e9c89dafaaa4e1d8a/ancestry HTTP/1.1" 200 1836 "-" "docker/1.5.0 go/go1.3.3 git-commit/a8a31ef-dirty kernel/3.19.3 os/linux arch/amd64"`

	out, ok := extractAccEntry(in)

	if !ok {
		t.Fatal("failed to extract values")
	}

	assert.Equal(200, out.Status)
	// TODO:
}

func TestMethodeAPIExample(t *testing.T) {
	assert := assert.New(t)

	in := `127.0.0.1 - - [21/Apr/2015:12:15:34 +0000] "GET /eom-file/all/e09b49d6-e1fa-11e4-bb7f-00144feab7de HTTP/1.1" 200 53706 919 919`

	out, ok := extractAccEntry(in)
	if !ok {
		t.Fatal("failed to extract values")
	}

	assert.Equal(200, out.Status)
	// TODO:
}

func TestCmsNotifierPostExample(t *testing.T) {
	assert := assert.New(t)

	in := `172.17.42.1 -  -  [24/Jun/2015:11:09:36 +0000] "POST /notify HTTP/1.1" 500 - "-" "curl/7.42.0" 2197`
	out, ok := extractAccEntry(in)

	if !ok {
		t.Fatal("failed to extract values")
	}

	assert.Equal(500, out.Status)
	// TODO:
}

func TestExtractAppEntry(t *testing.T) {
	var tests = []struct {
		message string
		level   string
	}{
		{`ERROR [2015-08-07 09:03:45,581] kafka.utils.Utils$: transaction_id=tid_lYpxZctRHb_kafka_bridge fetching topic metadata for topics [Set(NativeCmsPublicationEvents)] from broker [ArrayBuffer(id:0,host:172.23.219.136,port:9092)] failed|[dw-3910 - POST /notify]! kafka.common.KafkaException: fetching topic metadata for topics [Set(NativeCmsPublicationEvents)] from broker [ArrayBuffer(id:0,host:172.23.219.136,port:9092)] failed|! at kafka.client.ClientUtils$.fetchTopicMetadata(ClientUtils.scala:67) ~[app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.BrokerPartitionInfo.updateInfo(BrokerPartitionInfo.scala:82) ~[app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.async.DefaultEventHandler$$anonfun$handle$2.apply$mcV$sp(DefaultEventHandler.scala:78) ~[app.jar:0.0.1-SNAPSHOT]|! at kafka.utils.Utils$.swallow(Utils.scala:167) [app.jar:0.0.1-SNAPSHOT]|! at kafka.utils.Logging$class.swallowError(Logging.scala:106) [app.jar:0.0.1-SNAPSHOT]|! at kafka.utils.Utils$.swallowError(Utils.scala:46) [app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.async.DefaultEventHandler.handle(DefaultEventHandler.scala:78) [app.jar:0.0.1-SNAPSHOT]|! at kafka.producer.Producer.send(Producer.scala:76) [app.jar:0.0.1-SNAPSHOT]|! at kafka.javaapi.producer.Producer.send(Producer.scala:33) [app.jar:0.0.1-SNAPSHOT]|! at com.ft.cmsnotifier.service.KafkaMessageProducer.produceNotifyEvent(KafkaMessageProducer.java:30) [app.jar:0.0.1-SNAPSHOT]|! at com.ft.cmsnotifier.resources.CmsNotifierResource.produceEventForContent(CmsNotifierResource.java:59) [app.jar:0.0.1-SNAPSHOT]|! at com.ft.cmsnotifier.resources.CmsNotifierResource.importContent(CmsNotifierResource.java:43) [app.jar:0.0.1-SNAPSHOT]|! at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) ~[na:na]|! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_51]|! at java.lang.reflect.Method.invoke(Method.java:497) ~[na:1.8.0_51]|! at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) [app.jar:0.0.1-SNAPSHOT]|! at com.sun.je`, `ERROR`},
		{`INFO  [2015-08-13 07:59:00,019] com.ft.platform.dropwizard.HealthCheckPageData:  event="advancedHealthCheck", action="detail", name="BinaryIngesterService", checkName="CanConnectToZooKeeper", ok="true", checkOutput="ZooKeeper instance responded, at least 1 are up."|[dw-19 - GET /__health]`, `INFO`},
		{`DEBUG [2015-08-12 14:14:34,421] io.dropwizard.setup.AdminEnvironment:  health checks = [CanConnectToKafka, CanConnectToZooKeeper, KafkaIsNotLagging, binary-writer ping, binary-writer version, deadlocks]|[main]`, `DEBUG`},
		{`WARN  [2015-08-12 14:14:35,004] kafka.consumer.ZookeeperConsumerConnector:  [content_d414a887c7ca-1439388871813-b0f5b10f], No broker partitions consumed by consumer thread content_d414a887c7ca-1439388871813-b0f5b10f-1 for topic CmsPublicationEvents|[pool-5-thread-1]`, `WARN`},
		{`INFO  [2015-08-13 08:01:28,393] com.ft.platform.dropwizard.HealthCheckPageData:  event="advancedHealthCheck", action="detail", name="CMSNotifierApplication", checkName="Can connect to Kafka on topic: NativeCmsPublicationEvents", ok="true", checkOutput="Kafka is connected and topic is present"|[dw-19 - GET /__health]`, `INFO`},
	}

	for _, test := range tests {
		appEntry, ok := extractAppEntry(test.message)
		if !ok {
			t.Fatalf("failed to extract values '%s'", test.message)
		}
		if appEntry.Level != test.level {
			t.Errorf("message: %s\nexpected level %s, actual level %s", test.message, test.level, appEntry.Level)
		}
	}
}

func TestExtractPamEntity(t *testing.T) {
	var tests = []struct {
		message       string
		UUID          string
		TransactionID string
		PublishDate   string
		PublishOk     string
		Duration      string
		Endpoint      string
	}{
		{`[splunkMetrics] 2015/12/21 10:01:37.336610 UUID=08d30fb4-a7b3-11e5-955c-1e1d6de94879 transaction_id=tid_28pbiavoqs publishDate=1450692093737000000 publishOk=true duration=6 endpoint=content`,
			"08d30fb4-a7b3-11e5-955c-1e1d6de94879",
			"tid_28pbiavoqs",
			"1450692093737000000",
			"true",
			"6",
			"content"},
		{`[splunkMetrics] 2015/12/21 10:01:37.336610 UUID=08d30fb4-a7b3-11e5-955c-1e1d6de94879 transaction_id=tid_28pbiavoqs publishDate=1450692093737000000 publishOk=true duration=6 endpoint=notifications-push`,
			"08d30fb4-a7b3-11e5-955c-1e1d6de94879",
			"tid_28pbiavoqs",
			"1450692093737000000",
			"true",
			"6",
			"notifications-push"},
	}

	for _, test := range tests {
		pamEntity, ok := extractPamEntity(test.message)
		if !ok {
			t.Fatalf("failed to extract values '%s'", test.message)
		}
		if pamEntity.UUID != test.UUID {
			t.Errorf("message: %s\nexpected UUID %s, actual UUID %s", test.message, test.UUID, pamEntity.UUID)
		}
		if pamEntity.TransactionID != test.TransactionID {
			t.Errorf("message: %s\nexpected transaction_id %s, actual transaction_id %s", test.message, test.TransactionID, pamEntity.TransactionID)
		}
		if pamEntity.PublishDate != test.PublishDate {
			t.Errorf("message: %s\nexpected publishDate %s, actual publishDate %s", test.message, test.PublishDate, pamEntity.PublishDate)
		}
		if pamEntity.PublishOk != test.PublishOk {
			t.Errorf("message: %s\nexpected publishOk %s, actual publishOk %s", test.message, test.PublishOk, pamEntity.PublishOk)
		}
		if pamEntity.Duration != test.Duration {
			t.Errorf("message: %s\nexpected duration %s, actual duration %s", test.message, test.Duration, pamEntity.Duration)
		}
		if pamEntity.Endpoint != test.Endpoint {
			t.Errorf("message: %s\nexpected endpoint %s, actual endpoint %s", test.message, test.Endpoint, pamEntity.Endpoint)
		}
	}
}

func TestExtractVarnishEntity(t *testing.T) {
	var tests = []struct {
		message   string
		AuthUser  string
		URI       string
		Status    string
		Resptime  string
		UserAgent string
	}{
		{`82.136.1.214, 172.24.88.199 next 14/Jun/2016:08:24:42 /__enriched-content-read-api/enrichedcontent/409ba29e-b7f8-417f-9847-f3332aa064a6 200 65526 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"`,
			"next",
			"/__enriched-content-read-api/enrichedcontent/409ba29e-b7f8-417f-9847-f3332aa064a6",
			"200",
			"65526",
			"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"},
	}

	for _, test := range tests {
		varnishEntity, ok := extractVarnishEntity(test.message)
		if !ok {
			t.Fatalf("failed to extract values '%s'", test.message)
		}
		if varnishEntity.URI != test.URI {
			t.Errorf("message: %s\nexpected uri %s, actual uri %s", test.message, test.URI, varnishEntity.URI)
		}
		if varnishEntity.Status != test.Status {
			t.Errorf("message: %s\nexpected status %s, actual status %s", test.message, test.Status, varnishEntity.Status)
		}
		if varnishEntity.Resptime != test.Resptime {
			t.Errorf("message: %s\nexpected respTime %s, actual respTime %s", test.message, test.Resptime, varnishEntity.Resptime)
		}
	}
}
<|endoftext|>


<commit_before>package facade

import (
	"fmt"
	"path"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"

	"github.com/nyaxt/otaru"
	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/blobstore/cachedblobstore"
	"github.com/nyaxt/otaru/btncrypt"
	"github.com/nyaxt/otaru/chunkstore"
	oflags "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/gc"
	"github.com/nyaxt/otaru/gcloud/auth"
	"github.com/nyaxt/otaru/gcloud/datastore"
	"github.com/nyaxt/otaru/gcloud/gcs"
	"github.com/nyaxt/otaru/inodedb"
	"github.com/nyaxt/otaru/inodedb/blobstoredbstatesnapshotio"
	"github.com/nyaxt/otaru/inodedb/inodedbsyncer"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/metadata"
	"github.com/nyaxt/otaru/mgmt"
	"github.com/nyaxt/otaru/scheduler"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("facade")

type Otaru struct {
	C btncrypt.Cipher

	S *scheduler.Scheduler
	R *scheduler.RepetitiveJobRunner

	Tsrc  oauth2.TokenSource
	DSCfg *datastore.Config
	GL    *datastore.GlobalLocker

	MetadataBS blobstore.BlobStore
	DefaultBS  blobstore.BlobStore

	BackendBS blobstore.BlobStore

	CacheTgtBS         *blobstore.FileBlobStore
	CBS                *cachedblobstore.CachedBlobStore
	CacheSyncJob       scheduler.ID
	AutoReduceCacheJob scheduler.ID

	SSLoc blobstoredbstatesnapshotio.SSLocator
	SIO   *blobstoredbstatesnapshotio.DBStateSnapshotIO

	TxIO        inodedb.DBTransactionLogIO
	CTxIO       inodedb.DBTransactionLogIO
	TxIOSyncJob scheduler.ID

	IDBBE      *inodedb.DB
	IDBS       *inodedb.DBService
	IDBSyncJob scheduler.ID

	FS *otaru.FileSystem

	AutoGCJob scheduler.ID

	MGMT *mgmt.Server
}

func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
	o := &Otaru{}

	var err error

	key := btncrypt.KeyFromPassword(cfg.Password)
	o.C, err = btncrypt.NewCipher(key)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init Cipher: %v", err)
	}

	o.S = scheduler.NewScheduler()
	o.R = scheduler.NewRepetitiveJobRunner(o.S)

	if !cfg.LocalDebug {
		o.Tsrc, err = auth.GetGCloudTokenSource(context.TODO(), cfg.CredentialsFilePath, cfg.TokenCacheFilePath, false)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
		}
		o.DSCfg = datastore.NewConfig(cfg.ProjectName, cfg.BucketName, o.C, o.Tsrc)
		o.GL = datastore.NewGlobalLocker(o.DSCfg, GenHostName(), "FIXME: fill info")
		if err := o.GL.Lock(); err != nil {
			return nil, err
		}
	}

	o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
	}

	if !cfg.LocalDebug {
		o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Tsrc, oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
		}
		if !cfg.UseSeparateBucketForMetadata {
			o.BackendBS = o.DefaultBS
		} else {
			metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
			o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Tsrc, oflags.O_RDWRCREATE)
			if err != nil {
				o.Close()
				return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
			}

			o.BackendBS = blobstore.Mux{
				blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
				blobstore.MuxEntry{nil, o.DefaultBS},
			}
		}
	} else {
		o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(DefaultConfigDir(), "bbs"), oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init FileBlobStore (backend for local debugging): %v", err)
		}
	}

	queryFn := chunkstore.NewQueryChunkVersion(o.C)
	o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, o.S, oflags.O_RDWRCREATE /* FIXME */, queryFn)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
	}
	if err := o.CBS.RestoreState(o.C); err != nil {
		logger.Warningf(mylog, "Attempted to restore cachedblobstore state but failed: %v", err)
	}
	o.CacheSyncJob = cachedblobstore.SetupCacheSync(o.CBS, o.R)
	o.AutoReduceCacheJob = cachedblobstore.SetupAutoReduceCache(o.CBS, o.R, cfg.CacheHighWatermarkInBytes, cfg.CacheLowWatermarkInBytes)

	if !cfg.LocalDebug {
		o.SSLoc = datastore.NewINodeDBSSLocator(o.DSCfg)
	} else {
		o.SSLoc = blobstoredbstatesnapshotio.SimpleSSLocator{}
	}
	o.SIO = blobstoredbstatesnapshotio.New(o.CBS, o.C, o.SSLoc)

	if !cfg.LocalDebug {
		txio := datastore.NewDBTransactionLogIO(o.DSCfg)
		o.TxIO = txio
		o.TxIOSyncJob = o.R.SyncEveryPeriod(txio, 300*time.Millisecond)
	} else {
		o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
	}
	o.CTxIO = inodedb.NewCachedDBTransactionLogIO(o.TxIO)

	if oneshotcfg.Mkfs {
		o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
		}
	} else {
		o.IDBBE, err = inodedb.NewDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewDB failed: %v", err)
		}
	}

	o.IDBS = inodedb.NewDBService(o.IDBBE)
	o.IDBSyncJob = o.R.RunEveryPeriod(inodedbsyncer.NewSyncTask(o.IDBS), 30*time.Second)

	o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)

	o.AutoGCJob = o.R.RunEveryPeriod(&gc.GCTask{o.CBS, o.IDBS, false}, time.Duration(cfg.GCPeriod)*time.Second)

	o.MGMT = mgmt.NewServer(cfg.HttpApiAddr)
	if err := o.runMgmtServer(cfg); err != nil {
		o.Close()
		return nil, fmt.Errorf("Mgmt server run failed: %v", err)
	}

	return o, nil
}

func (o *Otaru) Close() error {
	errs := []error{}

	if o.R != nil {
		o.R.Stop()
	}

	if o.S != nil {
		o.S.AbortAllAndStop()
	}

	if o.FS != nil {
		if err := o.FS.Sync(); err != nil {
			errs = append(errs, err)
		}
	}

	if o.IDBS != nil {
		o.IDBS.Quit()
	}

	if o.IDBBE != nil {
		if err := o.IDBBE.Sync(); err != nil {
			errs = append(errs, err)
		}
	}

	if o.CBS != nil {
		if err := o.CBS.Sync(); err != nil {
			errs = append(errs, err)
		}
		if err := o.CBS.SaveState(o.C); err != nil {
			errs = append(errs, err)
		}
		o.CBS.Quit()
	}

	if o.GL != nil {
		if err := o.GL.Unlock(); err != nil {
			errs = append(errs, err)
		}
	}

	return util.ToErrors(errs)
}
<commit_msg>update facade to match new cachedblobstore api<commit_after>package facade

import (
	"fmt"
	"path"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"

	"github.com/nyaxt/otaru"
	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/blobstore/cachedblobstore"
	"github.com/nyaxt/otaru/btncrypt"
	"github.com/nyaxt/otaru/chunkstore"
	oflags "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/gc"
	"github.com/nyaxt/otaru/gcloud/auth"
	"github.com/nyaxt/otaru/gcloud/datastore"
	"github.com/nyaxt/otaru/gcloud/gcs"
	"github.com/nyaxt/otaru/inodedb"
	"github.com/nyaxt/otaru/inodedb/blobstoredbstatesnapshotio"
	"github.com/nyaxt/otaru/inodedb/inodedbsyncer"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/metadata"
	"github.com/nyaxt/otaru/mgmt"
	"github.com/nyaxt/otaru/scheduler"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("facade")

type Otaru struct {
	C btncrypt.Cipher

	S *scheduler.Scheduler
	R *scheduler.RepetitiveJobRunner

	Tsrc  oauth2.TokenSource
	DSCfg *datastore.Config
	GL    *datastore.GlobalLocker

	MetadataBS blobstore.BlobStore
	DefaultBS  blobstore.BlobStore

	BackendBS blobstore.BlobStore

	CacheTgtBS         *blobstore.FileBlobStore
	CBS                *cachedblobstore.CachedBlobStore
	AutoReduceCacheJob scheduler.ID

	SSLoc blobstoredbstatesnapshotio.SSLocator
	SIO   *blobstoredbstatesnapshotio.DBStateSnapshotIO

	TxIO        inodedb.DBTransactionLogIO
	CTxIO       inodedb.DBTransactionLogIO
	TxIOSyncJob scheduler.ID

	IDBBE      *inodedb.DB
	IDBS       *inodedb.DBService
	IDBSyncJob scheduler.ID

	FS *otaru.FileSystem

	AutoGCJob scheduler.ID

	MGMT *mgmt.Server
}

func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
	o := &Otaru{}

	var err error

	key := btncrypt.KeyFromPassword(cfg.Password)
	o.C, err = btncrypt.NewCipher(key)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init Cipher: %v", err)
	}

	o.S = scheduler.NewScheduler()
	o.R = scheduler.NewRepetitiveJobRunner(o.S)

	if !cfg.LocalDebug {
		o.Tsrc, err = auth.GetGCloudTokenSource(context.TODO(), cfg.CredentialsFilePath, cfg.TokenCacheFilePath, false)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
		}
		o.DSCfg = datastore.NewConfig(cfg.ProjectName, cfg.BucketName, o.C, o.Tsrc)
		o.GL = datastore.NewGlobalLocker(o.DSCfg, GenHostName(), "FIXME: fill info")
		if err := o.GL.Lock(); err != nil {
			return nil, err
		}
	}

	o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
	}

	if !cfg.LocalDebug {
		o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Tsrc, oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
		}
		if !cfg.UseSeparateBucketForMetadata {
			o.BackendBS = o.DefaultBS
		} else {
			metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
			o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Tsrc, oflags.O_RDWRCREATE)
			if err != nil {
				o.Close()
				return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
			}

			o.BackendBS = blobstore.Mux{
				blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
				blobstore.MuxEntry{nil, o.DefaultBS},
			}
		}
	} else {
		o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(DefaultConfigDir(), "bbs"), oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init FileBlobStore (backend for local debugging): %v", err)
		}
	}

	queryFn := chunkstore.NewQueryChunkVersion(o.C)
	o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, o.S, oflags.O_RDWRCREATE /* FIXME */, queryFn)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
	}
	if err := o.CBS.RestoreState(o.C); err != nil {
		logger.Warningf(mylog, "Attempted to restore cachedblobstore state but failed: %v", err)
	}
	o.AutoReduceCacheJob = cachedblobstore.SetupAutoReduceCache(o.CBS, o.R, cfg.CacheHighWatermarkInBytes, cfg.CacheLowWatermarkInBytes)

	if !cfg.LocalDebug {
		o.SSLoc = datastore.NewINodeDBSSLocator(o.DSCfg)
	} else {
		o.SSLoc = blobstoredbstatesnapshotio.SimpleSSLocator{}
	}
	o.SIO = blobstoredbstatesnapshotio.New(o.CBS, o.C, o.SSLoc)

	if !cfg.LocalDebug {
		txio := datastore.NewDBTransactionLogIO(o.DSCfg)
		o.TxIO = txio
		o.TxIOSyncJob = o.R.SyncEveryPeriod(txio, 300*time.Millisecond)
	} else {
		o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
	}
	o.CTxIO = inodedb.NewCachedDBTransactionLogIO(o.TxIO)

	if oneshotcfg.Mkfs {
		o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
		}
	} else {
		o.IDBBE, err = inodedb.NewDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewDB failed: %v", err)
		}
	}

	o.IDBS = inodedb.NewDBService(o.IDBBE)
	o.IDBSyncJob = o.R.RunEveryPeriod(inodedbsyncer.NewSyncTask(o.IDBS), 30*time.Second)

	o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)

	o.AutoGCJob = o.R.RunEveryPeriod(&gc.GCTask{o.CBS, o.IDBS, false}, time.Duration(cfg.GCPeriod)*time.Second)

	o.MGMT = mgmt.NewServer(cfg.HttpApiAddr)
	if err := o.runMgmtServer(cfg); err != nil {
		o.Close()
		return nil, fmt.Errorf("Mgmt server run failed: %v", err)
	}

	return o, nil
}

func (o *Otaru) Close() error {
	errs := []error{}

	if o.R != nil {
		o.R.Stop()
	}

	if o.S != nil {
		o.S.AbortAllAndStop()
	}

	if o.FS != nil {
		if err := o.FS.Sync(); err != nil {
			errs = append(errs, err)
		}
	}

	if o.IDBS != nil {
		o.IDBS.Quit()
	}

	if o.IDBBE != nil {
		if err := o.IDBBE.Sync(); err != nil {
			errs = append(errs, err)
		}
	}

	if o.CBS != nil {
		if err := o.CBS.Sync(); err != nil {
			errs = append(errs, err)
		}
		if err := o.CBS.SaveState(o.C); err != nil {
			errs = append(errs, err)
		}
		o.CBS.Quit()
	}

	if o.GL != nil {
		if err := o.GL.Unlock(); err != nil {
			errs = append(errs, err)
		}
	}

	return util.ToErrors(errs)
}
<|endoftext|>


<commit_before>package form

import (
	"math"
	"regexp"
	"time"
)

// Input Name
func (fns FieldFuncs) Name(name string) {
	fns.Call("set_name", map[string]interface{}{
		"set_name": name,
	})

	fns["set_name"] = nil

	fns["name"] = func(m map[string]interface{}) {
		*(m["name"].(*string)) = name
	}
}

// Make Field Mandatory
func (fns FieldFuncs) Mandatory() *string {
	err := ""

	fns["mandatory"] = func(m map[string]interface{}) {
		*(m["mandatory"].(*bool)) = true
		*(m["err"].(*string)) = err
	}

	return &err
}

// Size of Field
type Size struct {
	Min, Max       int
	MinErr, MaxErr string
}

// Size of Field
func (fns FieldFuncs) Size() *Size {
	size := &Size{-1, -1, "", ""}

	fns["size"] = func(m map[string]interface{}) {
		*(m["min"].(*int)) = size.Min
		*(m["max"].(*int)) = size.Max
		*(m["minErr"].(*string)) = size.MinErr
		*(m["maxErr"].(*string)) = size.MaxErr
	}

	return size
}

// Range of Integer Field
type RangeInt struct {
	Min, Max       int64
	MinErr, MaxErr string
}

// Range of Integer Field
func (fns FieldFuncs) RangeInt() *RangeInt {
	rangeInt := &RangeInt{-9223372036854775808, 9223372036854775807, "", ""}

	fns["range"] = func(m map[string]interface{}) {
		*(m["min"].(*int64)) = rangeInt.Min
		*(m["max"].(*int64)) = rangeInt.Max
		*(m["minErr"].(*string)) = rangeInt.MinErr
		*(m["maxErr"].(*string)) = rangeInt.MaxErr
	}

	return rangeInt
}

// Range of Float Field
type RangeFloat struct {
	Min, Max       float64
	MinErr, MaxErr string
}

// Range of Float Field
func (fns FieldFuncs) RangeFloat() *RangeFloat {
	rangeInt := &RangeFloat{math.NaN(), math.NaN(), "", ""}

	fns["range"] = func(m map[string]interface{}) {
		*(m["min"].(*float64)) = rangeInt.Min
		*(m["max"].(*float64)) = rangeInt.Max
		*(m["minErr"].(*string)) = rangeInt.MinErr
		*(m["maxErr"].(*string)) = rangeInt.MaxErr
	}

	return rangeInt
}

// Range of Time Field
type RangeTime struct {
	Min, Max       time.Time
	MinErr, MaxErr string
}

// Range of Time Field
func (fns FieldFuncs) RangeTime() *RangeTime {
	rangeInt := &RangeTime{time.Time{}, time.Time{}, "", ""}

	fns["range"] = func(m map[string]interface{}) {
		*(m["min"].(*time.Time)) = rangeInt.Min
		*(m["max"].(*time.Time)) = rangeInt.Max
		*(m["minErr"].(*string)) = rangeInt.MinErr
		*(m["maxErr"].(*string)) = rangeInt.MaxErr
	}

	return rangeInt
}

// Number of Step (int64)
func (fns FieldFuncs) StepInt(i int64) *string {
	err := ""

	fns["step"] = func(m map[string]interface{}) {
		*(m["step"].(*int64)) = i
		*(m["err"].(*string)) = err
	}

	return &err
}

// Number of Step Float64
func (fns FieldFuncs) StepFloat(f float64) *string {
	err := ""

	fns["step"] = func(m map[string]interface{}) {
		*(m["step"].(*float64)) = f
		*(m["err"].(*string)) = err
	}

	return &err
}

// Must Match Field
type MustMatch struct {
	Name  string
	Value *string
	Err   string
}

// Must Match Field
func (fns FieldFuncs) MustMatch() *MustMatch {
	mustMatch := &MustMatch{}

	fns["mustmatch"] = func(m map[string]interface{}) {
		*(m["name"].(*string)) = mustMatch.Name
		*(m["value"].(*string)) = *mustMatch.Value
		*(m["err"].(*string)) = mustMatch.Err
	}

	return mustMatch
}

// Regular Expression
func (fns FieldFuncs) Pattern(re *regexp.Regexp) *string {
	err := ""

	fns["pattern"] = func(m map[string]interface{}) {
		*(m["pattern"].(**regexp.Regexp)) = re
		*(m["err"].(*string)) = err
	}

	return &err
}

// Rows and Columns
func (fns FieldFuncs) Textarea(rows, cols int) {
	fns["textarea"] = func(m map[string]interface{}) {
		*(m["rows"].(*int)) = rows
		*(m["cols"].(*int)) = cols
	}
}

// Custom Email Error
func (fns FieldFuncs) EmailError(err string) {
	fns["email"] = func(m map[string]interface{}) {
		*(m["email"].(*string)) = err
	}
}

// Attribute
func (fns FieldFuncs) Attr(attr map[string]string) {
	fns["attr"] = func(m map[string]interface{}) {
		*(m["attr"].(*map[string]string)) = attr
	}
}

// Append Options, accept []Option, []OptionInt, []OptionFloat
func (fns FieldFuncs) Options(v interface{}) {
	fns["option"] = func(m map[string]interface{}) {
		switch v := v.(type) {
		case []Option:
			*(m["option"].(*[]Option)) = v
		case []OptionInt:
			*(m["option"].(*[]OptionInt)) = v
		case []OptionFloat:
			*(m["option"].(*[]OptionFloat)) = v
		}
	}
}

// Append Radios, accept []Radio, []RadioInt, []RadioFloat
func (fns FieldFuncs) Radios(v interface{}) {
	fns["radio"] = func(m map[string]interface{}) {
		switch v := v.(type) {
		case []Radio:
			*(m["radio"].(*[]Radio)) = v
		case []RadioInt:
			*(m["radio"].(*[]RadioInt)) = v
		case []RadioFloat:
			*(m["radio"].(*[]RadioFloat)) = v
		}
	}
}

// File
type File struct {
	Size      int64
	SizeErr   string
	Accept    []string
	AcceptErr string
}

// File
func (fns FieldFuncs) File() *File {
	file := &File{Size: -1}

	fns["file"] = func(m map[string]interface{}) {
		*(m["size"].(*int64)) = file.Size
		*(m["sizeErr"].(*string)) = file.SizeErr
		*(m["accept"].(*[]string)) = file.Accept
		*(m["acceptErr"].(*string)) = file.AcceptErr
	}

	return file
}

// HTML
type HTML struct {
	Before string
	After  string
}

// HTML
func (fns FieldFuncs) HTML() *HTML {
	html := &HTML{}

	fns["html"] = func(m map[string]interface{}) {
		*(m["before"].(*string)) = html.Before
		*(m["after"].(*string)) = html.After
	}

	return html
}

// Label
type Label struct {
	Content, For string
	Attr         map[string]string
}

// Label
func (fns FieldFuncs) Label() *Label {
	label := &Label{}

	fns["html"] = func(m map[string]interface{}) {
		*(m["content"].(*string)) = label.Content
		*(m["for"].(*string)) = label.For
		*(m["attr"].(*map[string]string)) = label.Attr
	}

	return label
}

// Custom Rules
func (fns FieldFuncs) Custom(fn func(*error, *string)) {
	fns["ext"] = func(m map[string]interface{}) {
		fn(m["error"].(*error), m["warning"].(*string))
	}
}
<commit_msg>Fixed bug with label<commit_after>package form

import (
	"math"
	"regexp"
	"time"
)

// Input Name
func (fns FieldFuncs) Name(name string) {
	fns.Call("set_name", map[string]interface{}{
		"set_name": name,
	})

	fns["set_name"] = nil

	fns["name"] = func(m map[string]interface{}) {
		*(m["name"].(*string)) = name
	}
}

// Make Field Mandatory
func (fns FieldFuncs) Mandatory() *string {
	err := ""

	fns["mandatory"] = func(m map[string]interface{}) {
		*(m["mandatory"].(*bool)) = true
		*(m["err"].(*string)) = err
	}

	return &err
}

// Size of Field
type Size struct {
	Min, Max       int
	MinErr, MaxErr string
}

// Size of Field
func (fns FieldFuncs) Size() *Size {
	size := &Size{-1, -1, "", ""}

	fns["size"] = func(m map[string]interface{}) {
		*(m["min"].(*int)) = size.Min
		*(m["max"].(*int)) = size.Max
		*(m["minErr"].(*string)) = size.MinErr
		*(m["maxErr"].(*string)) = size.MaxErr
	}

	return size
}

// Range of Integer Field
type RangeInt struct {
	Min, Max       int64
	MinErr, MaxErr string
}

// Range of Integer Field
func (fns FieldFuncs) RangeInt() *RangeInt {
	rangeInt := &RangeInt{-9223372036854775808, 9223372036854775807, "", ""}

	fns["range"] = func(m map[string]interface{}) {
		*(m["min"].(*int64)) = rangeInt.Min
		*(m["max"].(*int64)) = rangeInt.Max
		*(m["minErr"].(*string)) = rangeInt.MinErr
		*(m["maxErr"].(*string)) = rangeInt.MaxErr
	}

	return rangeInt
}

// Range of Float Field
type RangeFloat struct {
	Min, Max       float64
	MinErr, MaxErr string
}

// Range of Float Field
func (fns FieldFuncs) RangeFloat() *RangeFloat {
	rangeInt := &RangeFloat{math.NaN(), math.NaN(), "", ""}

	fns["range"] = func(m map[string]interface{}) {
		*(m["min"].(*float64)) = rangeInt.Min
		*(m["max"].(*float64)) = rangeInt.Max
		*(m["minErr"].(*string)) = rangeInt.MinErr
		*(m["maxErr"].(*string)) = rangeInt.MaxErr
	}

	return rangeInt
}

// Range of Time Field
type RangeTime struct {
	Min, Max       time.Time
	MinErr, MaxErr string
}

// Range of Time Field
func (fns FieldFuncs) RangeTime() *RangeTime {
	rangeInt := &RangeTime{time.Time{}, time.Time{}, "", ""}

	fns["range"] = func(m map[string]interface{}) {
		*(m["min"].(*time.Time)) = rangeInt.Min
		*(m["max"].(*time.Time)) = rangeInt.Max
		*(m["minErr"].(*string)) = rangeInt.MinErr
		*(m["maxErr"].(*string)) = rangeInt.MaxErr
	}

	return rangeInt
}

// Number of Step (int64)
func (fns FieldFuncs) StepInt(i int64) *string {
	err := ""

	fns["step"] = func(m map[string]interface{}) {
		*(m["step"].(*int64)) = i
		*(m["err"].(*string)) = err
	}

	return &err
}

// Number of Step Float64
func (fns FieldFuncs) StepFloat(f float64) *string {
	err := ""

	fns["step"] = func(m map[string]interface{}) {
		*(m["step"].(*float64)) = f
		*(m["err"].(*string)) = err
	}

	return &err
}

// Must Match Field
type MustMatch struct {
	Name  string
	Value *string
	Err   string
}

// Must Match Field
func (fns FieldFuncs) MustMatch() *MustMatch {
	mustMatch := &MustMatch{}

	fns["mustmatch"] = func(m map[string]interface{}) {
		*(m["name"].(*string)) = mustMatch.Name
		*(m["value"].(*string)) = *mustMatch.Value
		*(m["err"].(*string)) = mustMatch.Err
	}

	return mustMatch
}

// Regular Expression
func (fns FieldFuncs) Pattern(re *regexp.Regexp) *string {
	err := ""

	fns["pattern"] = func(m map[string]interface{}) {
		*(m["pattern"].(**regexp.Regexp)) = re
		*(m["err"].(*string)) = err
	}

	return &err
}

// Rows and Columns
func (fns FieldFuncs) Textarea(rows, cols int) {
	fns["textarea"] = func(m map[string]interface{}) {
		*(m["rows"].(*int)) = rows
		*(m["cols"].(*int)) = cols
	}
}

// Custom Email Error
func (fns FieldFuncs) EmailError(err string) {
	fns["email"] = func(m map[string]interface{}) {
		*(m["email"].(*string)) = err
	}
}

// Attribute
func (fns FieldFuncs) Attr(attr map[string]string) {
	fns["attr"] = func(m map[string]interface{}) {
		*(m["attr"].(*map[string]string)) = attr
	}
}

// Append Options, accept []Option, []OptionInt, []OptionFloat
func (fns FieldFuncs) Options(v interface{}) {
	fns["option"] = func(m map[string]interface{}) {
		switch v := v.(type) {
		case []Option:
			*(m["option"].(*[]Option)) = v
		case []OptionInt:
			*(m["option"].(*[]OptionInt)) = v
		case []OptionFloat:
			*(m["option"].(*[]OptionFloat)) = v
		}
	}
}

// Append Radios, accept []Radio, []RadioInt, []RadioFloat
func (fns FieldFuncs) Radios(v interface{}) {
	fns["radio"] = func(m map[string]interface{}) {
		switch v := v.(type) {
		case []Radio:
			*(m["radio"].(*[]Radio)) = v
		case []RadioInt:
			*(m["radio"].(*[]RadioInt)) = v
		case []RadioFloat:
			*(m["radio"].(*[]RadioFloat)) = v
		}
	}
}

// File
type File struct {
	Size      int64
	SizeErr   string
	Accept    []string
	AcceptErr string
}

// File
func (fns FieldFuncs) File() *File {
	file := &File{Size: -1}

	fns["file"] = func(m map[string]interface{}) {
		*(m["size"].(*int64)) = file.Size
		*(m["sizeErr"].(*string)) = file.SizeErr
		*(m["accept"].(*[]string)) = file.Accept
		*(m["acceptErr"].(*string)) = file.AcceptErr
	}

	return file
}

// HTML
type HTML struct {
	Before string
	After  string
}

// HTML
func (fns FieldFuncs) HTML() *HTML {
	html := &HTML{}

	fns["html"] = func(m map[string]interface{}) {
		*(m["before"].(*string)) = html.Before
		*(m["after"].(*string)) = html.After
	}

	return html
}

// Label
type Label struct {
	Content, For string
	Attr         map[string]string
}

// Label
func (fns FieldFuncs) Label() *Label {
	label := &Label{}

	fns["label"] = func(m map[string]interface{}) {
		*(m["content"].(*string)) = label.Content
		*(m["for"].(*string)) = label.For
		*(m["attr"].(*map[string]string)) = label.Attr
	}

	return label
}

// Custom Rules
func (fns FieldFuncs) Custom(fn func(*error, *string)) {
	fns["ext"] = func(m map[string]interface{}) {
		fn(m["error"].(*error), m["warning"].(*string))
	}
}
<|endoftext|>


<commit_before>package files

import (
	"github.com/smira/aptly/aptly"
	"github.com/smira/aptly/utils"
	"os"
	"path/filepath"
)

// PublishedStorage abstract file system with public dirs (published repos)
type PublishedStorage struct {
	rootPath string
}

// Check interface
var (
	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
)

// NewPublishedStorage creates new instance of PublishedStorage which specified root
func NewPublishedStorage(root string) *PublishedStorage {
	return &PublishedStorage{rootPath: filepath.Join(root, "public")}
}

// PublicPath returns root of public part
func (storage *PublishedStorage) PublicPath() string {
	return storage.rootPath
}

// MkDir creates directory recursively under public path
func (storage *PublishedStorage) MkDir(path string) error {
	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
}

// CreateFile creates file for writing under public path
func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
	return os.Create(filepath.Join(storage.rootPath, path))
}

// Remove removes single file under public path
func (storage *PublishedStorage) Remove(path string) error {
	filepath := filepath.Join(storage.rootPath, path)
	return os.Remove(filepath)
}

// RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
	filepath := filepath.Join(storage.rootPath, path)
	if progress != nil {
		progress.Printf("Removing %s...\n", filepath)
	}
	return os.RemoveAll(filepath)
}

// LinkFromPool links package file from pool to dist's pool location
//
// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is filepath to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool, sourcePath string) error {
	// verify that package pool is local pool is filesystem pool
	_ = sourcePool.(*PackagePool)

	baseName := filepath.Base(sourcePath)
	poolPath := filepath.Join(storage.rootPath, publishedDirectory)

	err := os.MkdirAll(poolPath, 0755)
	if err != nil {
		return err
	}

	_, err = os.Stat(filepath.Join(poolPath, baseName))
	if err == nil { // already exists, skip
		return nil
	}

	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
}

// Filelist returns list of files under prefix
func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
	root := filepath.Join(storage.rootPath, prefix)
	result := []string{}

	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			result = append(result, path[len(root)+1:])
		}
		return nil
	})

	return result, err
}

// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
	return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
}

// RenameFile renames (moves) file
func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
}
<commit_msg>When linking, check that inode file matches if linking to same file. #65<commit_after>package files

import (
	"fmt"
	"github.com/smira/aptly/aptly"
	"github.com/smira/aptly/utils"
	"os"
	"path/filepath"
	"syscall"
)

// PublishedStorage abstract file system with public dirs (published repos)
type PublishedStorage struct {
	rootPath string
}

// Check interface
var (
	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
)

// NewPublishedStorage creates new instance of PublishedStorage which specified root
func NewPublishedStorage(root string) *PublishedStorage {
	return &PublishedStorage{rootPath: filepath.Join(root, "public")}
}

// PublicPath returns root of public part
func (storage *PublishedStorage) PublicPath() string {
	return storage.rootPath
}

// MkDir creates directory recursively under public path
func (storage *PublishedStorage) MkDir(path string) error {
	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
}

// CreateFile creates file for writing under public path
func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
	return os.Create(filepath.Join(storage.rootPath, path))
}

// Remove removes single file under public path
func (storage *PublishedStorage) Remove(path string) error {
	filepath := filepath.Join(storage.rootPath, path)
	return os.Remove(filepath)
}

// RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
	filepath := filepath.Join(storage.rootPath, path)
	if progress != nil {
		progress.Printf("Removing %s...\n", filepath)
	}
	return os.RemoveAll(filepath)
}

// LinkFromPool links package file from pool to dist's pool location
//
// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is filepath to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool, sourcePath string) error {
	// verify that package pool is local pool is filesystem pool
	_ = sourcePool.(*PackagePool)

	baseName := filepath.Base(sourcePath)
	poolPath := filepath.Join(storage.rootPath, publishedDirectory)

	err := os.MkdirAll(poolPath, 0755)
	if err != nil {
		return err
	}

	var dstStat, srcStat os.FileInfo

	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
	if err == nil {
		// already exists, check source file
		srcStat, err = os.Stat(sourcePath)
		if err != nil {
			// source file doesn't exist? problem!
			return err
		}

		srcSys := srcStat.Sys().(*syscall.Stat_t)
		dstSys := dstStat.Sys().(*syscall.Stat_t)

		if srcSys.Ino != dstSys.Ino {
			return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName))
		}
		return nil
	}

	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
}

// Filelist returns list of files under prefix
func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
	root := filepath.Join(storage.rootPath, prefix)
	result := []string{}

	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			result = append(result, path[len(root)+1:])
		}
		return nil
	})

	return result, err
}

// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
	return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
}

// RenameFile renames (moves) file
func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"sort"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/mitchellh/go-homedir"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/aws/aws-sdk-go/service/s3"
)

func resourceAwsS3BucketObject() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsS3BucketObjectPut,
		Read:   resourceAwsS3BucketObjectRead,
		Update: resourceAwsS3BucketObjectPut,
		Delete: resourceAwsS3BucketObjectDelete,

		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"acl": {
				Type:         schema.TypeString,
				Default:      "private",
				Optional:     true,
				ValidateFunc: validateS3BucketObjectAclType,
			},

			"cache_control": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_disposition": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_encoding": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_language": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_type": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"key": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"source": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"content"},
			},

			"content": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"source"},
			},

			"storage_class": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				ValidateFunc: validateS3BucketObjectStorageClassType,
			},

			"server_side_encryption": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateS3BucketObjectServerSideEncryption,
				Computed:     true,
			},

			"kms_key_id": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateArn,
			},

			"etag": {
				Type: schema.TypeString,
				// This will conflict with SSE-C and SSE-KMS encryption and multi-part upload
				// if/when it's actually implemented. The Etag then won't match raw-file MD5.
				// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
				Optional:      true,
				Computed:      true,
				ConflictsWith: []string{"kms_key_id", "server_side_encryption"},
			},

			"version_id": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceAwsS3BucketObjectPut(d *schema.ResourceData, meta interface{}) error {
	s3conn := meta.(*AWSClient).s3conn

	var body io.ReadSeeker

	if v, ok := d.GetOk("source"); ok {
		source := v.(string)
		path, err := homedir.Expand(source)
		if err != nil {
			return fmt.Errorf("Error expanding homedir in source (%s): %s", source, err)
		}
		file, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("Error opening S3 bucket object source (%s): %s", source, err)
		}

		body = file
	} else if v, ok := d.GetOk("content"); ok {
		content := v.(string)
		body = bytes.NewReader([]byte(content))
	} else {
		return fmt.Errorf("Must specify \"source\" or \"content\" field")
	}

	bucket := d.Get("bucket").(string)
	key := d.Get("key").(string)

	putInput := &s3.PutObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
		ACL:    aws.String(d.Get("acl").(string)),
		Body:   body,
	}

	if v, ok := d.GetOk("storage_class"); ok {
		putInput.StorageClass = aws.String(v.(string))
	}

	if v, ok := d.GetOk("cache_control"); ok {
		putInput.CacheControl = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_type"); ok {
		putInput.ContentType = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_encoding"); ok {
		putInput.ContentEncoding = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_language"); ok {
		putInput.ContentLanguage = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_disposition"); ok {
		putInput.ContentDisposition = aws.String(v.(string))
	}

	if v, ok := d.GetOk("server_side_encryption"); ok {
		putInput.ServerSideEncryption = aws.String(v.(string))
	}

	if v, ok := d.GetOk("kms_key_id"); ok {
		putInput.SSEKMSKeyId = aws.String(v.(string))
		putInput.ServerSideEncryption = aws.String(s3.ServerSideEncryptionAwsKms)
	}

	if v, ok := d.GetOk("tags"); ok {
		// The tag-set must be encoded as URL Query parameters.
		values := url.Values{}
		for k, v := range v.(map[string]interface{}) {
			values.Add(k, v.(string))
		}
		putInput.Tagging = aws.String(values.Encode())
	}

	resp, err := s3conn.PutObject(putInput)
	if err != nil {
		return fmt.Errorf("Error putting object in S3 bucket (%s): %s", bucket, err)
	}

	// See https://forums.aws.amazon.com/thread.jspa?threadID=44003
	d.Set("etag", strings.Trim(*resp.ETag, `"`))

	d.Set("version_id", resp.VersionId)
	d.SetId(key)
	return resourceAwsS3BucketObjectRead(d, meta)
}

func resourceAwsS3BucketObjectRead(d *schema.ResourceData, meta interface{}) error {
	s3conn := meta.(*AWSClient).s3conn

	bucket := d.Get("bucket").(string)
	key := d.Get("key").(string)

	resp, err := s3conn.HeadObject(
		&s3.HeadObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		})

	if err != nil {
		// If S3 returns a 404 Request Failure, mark the object as destroyed
		if awsErr, ok := err.(awserr.RequestFailure); ok && awsErr.StatusCode() == 404 {
			d.SetId("")
			log.Printf("[WARN] Error Reading Object (%s), object not found (HTTP status 404)", key)
			return nil
		}
		return err
	}
	log.Printf("[DEBUG] Reading S3 Bucket Object meta: %s", resp)

	d.Set("cache_control", resp.CacheControl)
	d.Set("content_disposition", resp.ContentDisposition)
	d.Set("content_encoding", resp.ContentEncoding)
	d.Set("content_language", resp.ContentLanguage)
	d.Set("content_type", resp.ContentType)
	d.Set("version_id", resp.VersionId)
	d.Set("server_side_encryption", resp.ServerSideEncryption)

	// Only set non-default KMS key ID (one that doesn't match default)
	if resp.SSEKMSKeyId != nil {
		// retrieve S3 KMS Default Master Key
		kmsconn := meta.(*AWSClient).kmsconn
		kmsresp, err := kmsconn.DescribeKey(&kms.DescribeKeyInput{
			KeyId: aws.String("alias/aws/s3"),
		})
		if err != nil {
			return fmt.Errorf("Failed to describe default S3 KMS key (alias/aws/s3): %s", err)
		}

		if *resp.SSEKMSKeyId != *kmsresp.KeyMetadata.Arn {
			log.Printf("[DEBUG] S3 object is encrypted using a non-default KMS Key ID: %s", *resp.SSEKMSKeyId)
			d.Set("kms_key_id", resp.SSEKMSKeyId)
		}
	}
	d.Set("etag", strings.Trim(*resp.ETag, `"`))

	// The "STANDARD" (which is also the default) storage
	// class when set would not be included in the results.
	d.Set("storage_class", s3.StorageClassStandard)
	if resp.StorageClass != nil {
		d.Set("storage_class", resp.StorageClass)
	}

	tagResp, err := s3conn.GetObjectTagging(
		&s3.GetObjectTaggingInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		})
	if err != nil {
		return err
	}
	d.Set("tags", tagsToMapS3(tagResp.TagSet))

	return nil
}

func resourceAwsS3BucketObjectDelete(d *schema.ResourceData, meta interface{}) error {
	s3conn := meta.(*AWSClient).s3conn

	bucket := d.Get("bucket").(string)
	key := d.Get("key").(string)

	if _, ok := d.GetOk("version_id"); ok {
		// Bucket is versioned, we need to delete all versions
		vInput := s3.ListObjectVersionsInput{
			Bucket: aws.String(bucket),
			Prefix: aws.String(key),
		}
		out, err := s3conn.ListObjectVersions(&vInput)
		if err != nil {
			return fmt.Errorf("Failed listing S3 object versions: %s", err)
		}

		for _, v := range out.Versions {
			input := s3.DeleteObjectInput{
				Bucket:    aws.String(bucket),
				Key:       aws.String(key),
				VersionId: v.VersionId,
			}
			_, err := s3conn.DeleteObject(&input)
			if err != nil {
				return fmt.Errorf("Error deleting S3 object version of %s:\n %s:\n %s",
					key, v, err)
			}
		}
	} else {
		// Just delete the object
		input := s3.DeleteObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		}
		_, err := s3conn.DeleteObject(&input)
		if err != nil {
			return fmt.Errorf("Error deleting S3 bucket object: %s", err)
		}
	}

	return nil
}

func validateS3BucketObjectAclType(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)

	cannedAcls := map[string]bool{
		s3.ObjectCannedACLPrivate:                true,
		s3.ObjectCannedACLPublicRead:             true,
		s3.ObjectCannedACLPublicReadWrite:        true,
		s3.ObjectCannedACLAuthenticatedRead:      true,
		s3.ObjectCannedACLAwsExecRead:            true,
		s3.ObjectCannedACLBucketOwnerRead:        true,
		s3.ObjectCannedACLBucketOwnerFullControl: true,
	}

	sentenceJoin := func(m map[string]bool) string {
		keys := make([]string, 0, len(m))
		for k := range m {
			keys = append(keys, fmt.Sprintf("%q", k))
		}
		sort.Strings(keys)

		length := len(keys)
		words := make([]string, length)
		copy(words, keys)

		words[length-1] = fmt.Sprintf("or %s", words[length-1])
		return strings.Join(words, ", ")
	}

	if _, ok := cannedAcls[value]; !ok {
		errors = append(errors, fmt.Errorf(
			"%q contains an invalid canned ACL type %q. Valid types are either %s",
			k, value, sentenceJoin(cannedAcls)))
	}
	return
}

func validateS3BucketObjectStorageClassType(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)

	storageClass := map[string]bool{
		s3.StorageClassStandard:          true,
		s3.StorageClassReducedRedundancy: true,
		s3.StorageClassStandardIa:        true,
	}

	if _, ok := storageClass[value]; !ok {
		errors = append(errors, fmt.Errorf(
			"%q contains an invalid Storage Class type %q. Valid types are either %q, %q, or %q",
			k, value, s3.StorageClassStandard, s3.StorageClassReducedRedundancy,
			s3.StorageClassStandardIa))
	}
	return
}

func validateS3BucketObjectServerSideEncryption(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)

	serverSideEncryption := map[string]bool{
		s3.ServerSideEncryptionAes256: true,
		s3.ServerSideEncryptionAwsKms: true,
	}

	if _, ok := serverSideEncryption[value]; !ok {
		errors = append(errors, fmt.Errorf(
			"%q contains an invalid Server Side Encryption value %q. Valid values are %q and %q",
			k, value, s3.ServerSideEncryptionAes256, s3.ServerSideEncryptionAwsKms))
	}
	return
}
<commit_msg>provider aws: add more details to s3 error message<commit_after>package aws

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"sort"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/mitchellh/go-homedir"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/aws/aws-sdk-go/service/s3"
)

func resourceAwsS3BucketObject() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsS3BucketObjectPut,
		Read:   resourceAwsS3BucketObjectRead,
		Update: resourceAwsS3BucketObjectPut,
		Delete: resourceAwsS3BucketObjectDelete,

		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"acl": {
				Type:         schema.TypeString,
				Default:      "private",
				Optional:     true,
				ValidateFunc: validateS3BucketObjectAclType,
			},

			"cache_control": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_disposition": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_encoding": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_language": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"content_type": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"key": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"source": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"content"},
			},

			"content": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"source"},
			},

			"storage_class": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				ValidateFunc: validateS3BucketObjectStorageClassType,
			},

			"server_side_encryption": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateS3BucketObjectServerSideEncryption,
				Computed:     true,
			},

			"kms_key_id": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateArn,
			},

			"etag": {
				Type: schema.TypeString,
				// This will conflict with SSE-C and SSE-KMS encryption and multi-part upload
				// if/when it's actually implemented. The Etag then won't match raw-file MD5.
				// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
				Optional:      true,
				Computed:      true,
				ConflictsWith: []string{"kms_key_id", "server_side_encryption"},
			},

			"version_id": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceAwsS3BucketObjectPut(d *schema.ResourceData, meta interface{}) error {
	s3conn := meta.(*AWSClient).s3conn

	var body io.ReadSeeker

	if v, ok := d.GetOk("source"); ok {
		source := v.(string)
		path, err := homedir.Expand(source)
		if err != nil {
			return fmt.Errorf("Error expanding homedir in source (%s): %s", source, err)
		}
		file, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("Error opening S3 bucket object source (%s): %s", source, err)
		}

		body = file
	} else if v, ok := d.GetOk("content"); ok {
		content := v.(string)
		body = bytes.NewReader([]byte(content))
	} else {
		return fmt.Errorf("Must specify \"source\" or \"content\" field")
	}

	bucket := d.Get("bucket").(string)
	key := d.Get("key").(string)

	putInput := &s3.PutObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
		ACL:    aws.String(d.Get("acl").(string)),
		Body:   body,
	}

	if v, ok := d.GetOk("storage_class"); ok {
		putInput.StorageClass = aws.String(v.(string))
	}

	if v, ok := d.GetOk("cache_control"); ok {
		putInput.CacheControl = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_type"); ok {
		putInput.ContentType = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_encoding"); ok {
		putInput.ContentEncoding = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_language"); ok {
		putInput.ContentLanguage = aws.String(v.(string))
	}

	if v, ok := d.GetOk("content_disposition"); ok {
		putInput.ContentDisposition = aws.String(v.(string))
	}

	if v, ok := d.GetOk("server_side_encryption"); ok {
		putInput.ServerSideEncryption = aws.String(v.(string))
	}

	if v, ok := d.GetOk("kms_key_id"); ok {
		putInput.SSEKMSKeyId = aws.String(v.(string))
		putInput.ServerSideEncryption = aws.String(s3.ServerSideEncryptionAwsKms)
	}

	if v, ok := d.GetOk("tags"); ok {
		// The tag-set must be encoded as URL Query parameters.
		values := url.Values{}
		for k, v := range v.(map[string]interface{}) {
			values.Add(k, v.(string))
		}
		putInput.Tagging = aws.String(values.Encode())
	}

	resp, err := s3conn.PutObject(putInput)
	if err != nil {
		return fmt.Errorf("Error putting object in S3 bucket (%s): %s", bucket, err)
	}

	// See https://forums.aws.amazon.com/thread.jspa?threadID=44003
	d.Set("etag", strings.Trim(*resp.ETag, `"`))

	d.Set("version_id", resp.VersionId)
	d.SetId(key)
	return resourceAwsS3BucketObjectRead(d, meta)
}

func resourceAwsS3BucketObjectRead(d *schema.ResourceData, meta interface{}) error {
	s3conn := meta.(*AWSClient).s3conn

	bucket := d.Get("bucket").(string)
	key := d.Get("key").(string)

	resp, err := s3conn.HeadObject(
		&s3.HeadObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		})

	if err != nil {
		// If S3 returns a 404 Request Failure, mark the object as destroyed
		if awsErr, ok := err.(awserr.RequestFailure); ok && awsErr.StatusCode() == 404 {
			d.SetId("")
			log.Printf("[WARN] Error Reading Object (%s), object not found (HTTP status 404)", key)
			return nil
		}
		return err
	}
	log.Printf("[DEBUG] Reading S3 Bucket Object meta: %s", resp)

	d.Set("cache_control", resp.CacheControl)
	d.Set("content_disposition", resp.ContentDisposition)
	d.Set("content_encoding", resp.ContentEncoding)
	d.Set("content_language", resp.ContentLanguage)
	d.Set("content_type", resp.ContentType)
	d.Set("version_id", resp.VersionId)
	d.Set("server_side_encryption", resp.ServerSideEncryption)

	// Only set non-default KMS key ID (one that doesn't match default)
	if resp.SSEKMSKeyId != nil {
		// retrieve S3 KMS Default Master Key
		kmsconn := meta.(*AWSClient).kmsconn
		kmsresp, err := kmsconn.DescribeKey(&kms.DescribeKeyInput{
			KeyId: aws.String("alias/aws/s3"),
		})
		if err != nil {
			return fmt.Errorf("Failed to describe default S3 KMS key (alias/aws/s3): %s", err)
		}

		if *resp.SSEKMSKeyId != *kmsresp.KeyMetadata.Arn {
			log.Printf("[DEBUG] S3 object is encrypted using a non-default KMS Key ID: %s", *resp.SSEKMSKeyId)
			d.Set("kms_key_id", resp.SSEKMSKeyId)
		}
	}
	d.Set("etag", strings.Trim(*resp.ETag, `"`))

	// The "STANDARD" (which is also the default) storage
	// class when set would not be included in the results.
	d.Set("storage_class", s3.StorageClassStandard)
	if resp.StorageClass != nil {
		d.Set("storage_class", resp.StorageClass)
	}

	tagResp, err := s3conn.GetObjectTagging(
		&s3.GetObjectTaggingInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		})
	if err != nil {
		return err
	}
	d.Set("tags", tagsToMapS3(tagResp.TagSet))

	return nil
}

func resourceAwsS3BucketObjectDelete(d *schema.ResourceData, meta interface{}) error {
	s3conn := meta.(*AWSClient).s3conn

	bucket := d.Get("bucket").(string)
	key := d.Get("key").(string)

	if _, ok := d.GetOk("version_id"); ok {
		// Bucket is versioned, we need to delete all versions
		vInput := s3.ListObjectVersionsInput{
			Bucket: aws.String(bucket),
			Prefix: aws.String(key),
		}
		out, err := s3conn.ListObjectVersions(&vInput)
		if err != nil {
			return fmt.Errorf("Failed listing S3 object versions: %s", err)
		}

		for _, v := range out.Versions {
			input := s3.DeleteObjectInput{
				Bucket:    aws.String(bucket),
				Key:       aws.String(key),
				VersionId: v.VersionId,
			}
			_, err := s3conn.DeleteObject(&input)
			if err != nil {
				return fmt.Errorf("Error deleting S3 object version of %s:\n %s:\n %s",
					key, v, err)
			}
		}
	} else {
		// Just delete the object
		input := s3.DeleteObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		}
		_, err := s3conn.DeleteObject(&input)
		if err != nil {
			return fmt.Errorf("Error deleting S3 bucket object: %s  Bucket: %q Object: %q", err, bucket, key)
		}
	}

	return nil
}

func validateS3BucketObjectAclType(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)

	cannedAcls := map[string]bool{
		s3.ObjectCannedACLPrivate:                true,
		s3.ObjectCannedACLPublicRead:             true,
		s3.ObjectCannedACLPublicReadWrite:        true,
		s3.ObjectCannedACLAuthenticatedRead:      true,
		s3.ObjectCannedACLAwsExecRead:            true,
		s3.ObjectCannedACLBucketOwnerRead:        true,
		s3.ObjectCannedACLBucketOwnerFullControl: true,
	}

	sentenceJoin := func(m map[string]bool) string {
		keys := make([]string, 0, len(m))
		for k := range m {
			keys = append(keys, fmt.Sprintf("%q", k))
		}
		sort.Strings(keys)

		length := len(keys)
		words := make([]string, length)
		copy(words, keys)

		words[length-1] = fmt.Sprintf("or %s", words[length-1])
		return strings.Join(words, ", ")
	}

	if _, ok := cannedAcls[value]; !ok {
		errors = append(errors, fmt.Errorf(
			"%q contains an invalid canned ACL type %q. Valid types are either %s",
			k, value, sentenceJoin(cannedAcls)))
	}
	return
}

func validateS3BucketObjectStorageClassType(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)

	storageClass := map[string]bool{
		s3.StorageClassStandard:          true,
		s3.StorageClassReducedRedundancy: true,
		s3.StorageClassStandardIa:        true,
	}

	if _, ok := storageClass[value]; !ok {
		errors = append(errors, fmt.Errorf(
			"%q contains an invalid Storage Class type %q. Valid types are either %q, %q, or %q",
			k, value, s3.StorageClassStandard, s3.StorageClassReducedRedundancy,
			s3.StorageClassStandardIa))
	}
	return
}

func validateS3BucketObjectServerSideEncryption(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)

	serverSideEncryption := map[string]bool{
		s3.ServerSideEncryptionAes256: true,
		s3.ServerSideEncryptionAwsKms: true,
	}

	if _, ok := serverSideEncryption[value]; !ok {
		errors = append(errors, fmt.Errorf(
			"%q contains an invalid Server Side Encryption value %q. Valid values are %q and %q",
			k, value, s3.ServerSideEncryptionAes256, s3.ServerSideEncryptionAwsKms))
	}
	return
}
<|endoftext|>


<commit_before>/* log_test.go - test for log.go */
package log

import (
	"testing"
)

func TestNewLogger(t *testing.T) {
	// var conf = Conf{Name: "test", Dir: "./log/", Level: "DEBUG", Console: false, Daily: true, BackupNum: 2}
	var conf = Conf{Name: "test", Dir: "./log/", Level: "Info", Console: false, Daily: true, BackupNum: 2}

	var (
		err    error
		logger Logger
	)
	if logger, err = NewLogger(conf); err != nil {
		t.Errorf("NewLogger(conf{%#v}) = error{%#v}", conf, err)
	}

	logger.Debug("Debug")
	logger.Info("Info")
	logger.Warn("Warn")
	logger.Error("Error")
	logger.Critic("Critic")

	logger.Close()
}

func TestNewLoggerWithConfFile(t *testing.T) {

	var (
		err    error
		dir    string
		conf   string
		logger Logger
	)
	dir = "./log"
	conf = "log_test.xml"
	if logger, err = NewLoggerWithConfFile(dir, conf); err != nil {
		t.Errorf("NewLoggerWithConfFile(dir{%s}, conf{%#v}) = error{%#v}", dir, conf, err)
	}

	// And now we're ready!
	logger.Finest("This will only go to those of you really cool UDP kids!  If you change enabled=true.")
	logger.Debug("Oh no!  %d + %d = %d!", 2, 2, 2+2)
	logger.Info("About that time, eh chaps?")
	logger.Warn("Warn")
	logger.Error("Error")
	logger.Critic("Critic")

	logger.Close()
}
<commit_msg>add multiple loggers test<commit_after>/* log_test.go - test for log.go */
package log

import (
	"testing"
)

func TestNewLogger(t *testing.T) {
	// var conf = Conf{Name: "test", Dir: "./log/", Level: "DEBUG", Console: false, Daily: true, BackupNum: 2}
	var conf = Conf{Name: "test", Dir: "./log/", Level: "Info", Console: false, Daily: true, BackupNum: 2}

	var (
		err    error
		logger Logger
	)
	if logger, err = NewLogger(conf); err != nil {
		t.Errorf("NewLogger(conf{%#v}) = error{%#v}", conf, err)
	}

	logger.Debug("Debug")
	logger.Info("Info")
	logger.Warn("Warn")
	logger.Error("Error")
	logger.Critic("Critic")

	logger.Close()
}

func TestNewLoggerWithConfFile(t *testing.T) {
	var (
		err    error
		dir    string
		conf   string
		logger Logger
	)
	dir = "./log"
	conf = "log_test.xml"
	if logger, err = NewLoggerWithConfFile(dir, conf); err != nil {
		t.Errorf("NewLoggerWithConfFile(dir{%s}, conf{%#v}) = error{%#v}", dir, conf, err)
	}

	// And now we're ready!
	logger.Finest("This will only go to those of you really cool UDP kids!  If you change enabled=true.")
	logger.Debug("Oh no!  %d + %d = %d!", 2, 2, 2+2)
	logger.Info("About that time, eh chaps?")
	logger.Warn("Warn")
	logger.Error("Error")
	logger.Critic("Critic")

	logger.Close()
}

func TestMultiLoggers(t *testing.T) {
	var conf = Conf{Name: "test1", Dir: "./log/", Level: "Info", Console: false, Daily: true, BackupNum: 2}

	var (
		err     error
		logger1 Logger
		logger2 Logger
	)
	if logger1, err = NewLogger(conf); err != nil {
		t.Errorf("NewLogger(conf{%#v}) = error{%#v}", conf, err)
	}
	conf = Conf{Name: "test2", Dir: "./log/", Level: "Info", Console: false, Daily: true, BackupNum: 2}
	if logger2, err = NewLogger(conf); err != nil {
		t.Errorf("NewLogger(conf{%#v}) = error{%#v}", conf, err)
	}

	logger1.Debug("Debug")
	logger1.Info("Info")
	logger1.Warn("Warn")
	logger1.Error("Error")
	logger1.Critic("Critic")

	logger2.Debug("Debug")
	logger2.Info("Info")
	logger2.Warn("Warn")
	logger2.Error("Error")
	logger2.Critic("Critic")

	logger2.Close()
	logger1.Close()
}
<|endoftext|>


<commit_before>package client

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"

	"github.com/flynn/go-discoverd"
	"github.com/flynn/lorne/types"
	"github.com/flynn/rpcplus"
)

var ErrNoServers = errors.New("lorne: no servers found")

func New(id string) (*Client, error) {
	services, err := discoverd.NewServiceSet("flynn-lorne")
	services.Filter(map[string]string{"id": id})
	if err != nil {
		return nil, err
	}
	addrs := services.Addrs()
	if len(addrs) == 0 {
		return nil, ErrNoServers
	}
	c, err := rpcplus.DialHTTP("tcp", addrs[0])
	return &Client{c, services}, err
}

type Client struct {
	c *rpcplus.Client

	service *discoverd.ServiceSet
}

func (c *Client) JobList() (map[string]lorne.Job, error) {
	var jobs map[string]lorne.Job
	err := c.c.Call("Host.JobList", struct{}{}, &jobs)
	return jobs, err
}

func (c *Client) GetJob(id string) (*lorne.Job, error) {
	var res lorne.Job
	err := c.c.Call("Host.GetJob", id, &res)
	return &res, err
}

func (c *Client) StopJob(id string) error {
	return c.c.Call("Host.StopJob", id, &struct{}{})
}

var ErrWouldWait = errors.New("lorne: attach would wait")

func (c *Client) Attach(req *lorne.AttachReq, wait bool) (ReadWriteCloser, func() error, error) {
	data, err := json.Marshal(req)
	if err != nil {
		return nil, nil, err
	}
	httpReq, err := http.NewRequest("POST", "/attach", bytes.NewBuffer(data))
	if err != nil {
		return nil, nil, err
	}
	addrs := c.service.Addrs()
	if len(addrs) == 0 {
		return nil, nil, ErrNoServers
	}
	conn, err := net.Dial("tcp", addrs[0])
	if err != nil {
		return nil, nil, err
	}
	clientconn := httputil.NewClientConn(conn, nil)
	res, err := clientconn.Do(httpReq)
	if err != nil && err != httputil.ErrPersistEOF {
		return nil, nil, err
	}
	if res.StatusCode != 200 {
		return nil, nil, fmt.Errorf("lorne: unexpected status %d", res.StatusCode)
	}
	var rwc io.ReadWriteCloser
	var buf *bufio.Reader
	rwc, buf = clientconn.Hijack()
	if buf.Buffered() > 0 {
		rwc = struct {
			io.Reader
			writeCloser
		}{
			io.MultiReader(io.LimitReader(buf, int64(buf.Buffered())), rwc),
			rwc.(writeCloser),
		}
	}

	attachState := make([]byte, 1)
	if _, err := rwc.Read(attachState); err != nil {
		rwc.Close()
		return nil, nil, err
	}

	handleState := func() error {
		switch attachState[0] {
		case lorne.AttachSuccess:
			return nil
		case lorne.AttachError:
			errBytes, err := ioutil.ReadAll(rwc)
			rwc.Close()
			if err != nil {
				return err
			}
			return errors.New(string(errBytes))
		default:
			rwc.Close()
			return fmt.Errorf("lorne: unknown attach state: %d", attachState)
		}
	}

	if attachState[0] == lorne.AttachWaiting {
		if !wait {
			rwc.Close()
			return nil, nil, ErrWouldWait
		}
		return rwc.(ReadWriteCloser), func() error {
			if _, err := rwc.Read(attachState); err != nil {
				rwc.Close()
				return err
			}
			return handleState()
		}, nil
	}

	return rwc.(ReadWriteCloser), nil, handleState()
}

type ReadWriteCloser interface {
	io.ReadWriteCloser
	CloseWrite() error
}

type writeCloser interface {
	io.WriteCloser
	CloseWrite() error
}
<commit_msg>pkg/lorne: Update discoverd usage<commit_after>package client

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"

	"github.com/flynn/go-discoverd"
	"github.com/flynn/lorne/types"
	"github.com/flynn/rpcplus"
)

var ErrNoServers = errors.New("lorne: no servers found")

func New(id string) (*Client, error) {
	services, err := discoverd.NewServiceSet("flynn-lorne")
	services.Filter(map[string]string{"id": id})
	if err != nil {
		return nil, err
	}
	addrs := services.Addrs()
	if len(addrs) == 0 {
		return nil, ErrNoServers
	}
	c, err := rpcplus.DialHTTP("tcp", addrs[0])
	return &Client{c, services}, err
}

type Client struct {
	c *rpcplus.Client

	service discoverd.ServiceSet
}

func (c *Client) JobList() (map[string]lorne.Job, error) {
	var jobs map[string]lorne.Job
	err := c.c.Call("Host.JobList", struct{}{}, &jobs)
	return jobs, err
}

func (c *Client) GetJob(id string) (*lorne.Job, error) {
	var res lorne.Job
	err := c.c.Call("Host.GetJob", id, &res)
	return &res, err
}

func (c *Client) StopJob(id string) error {
	return c.c.Call("Host.StopJob", id, &struct{}{})
}

var ErrWouldWait = errors.New("lorne: attach would wait")

func (c *Client) Attach(req *lorne.AttachReq, wait bool) (ReadWriteCloser, func() error, error) {
	data, err := json.Marshal(req)
	if err != nil {
		return nil, nil, err
	}
	httpReq, err := http.NewRequest("POST", "/attach", bytes.NewBuffer(data))
	if err != nil {
		return nil, nil, err
	}
	addrs := c.service.Addrs()
	if len(addrs) == 0 {
		return nil, nil, ErrNoServers
	}
	conn, err := net.Dial("tcp", addrs[0])
	if err != nil {
		return nil, nil, err
	}
	clientconn := httputil.NewClientConn(conn, nil)
	res, err := clientconn.Do(httpReq)
	if err != nil && err != httputil.ErrPersistEOF {
		return nil, nil, err
	}
	if res.StatusCode != 200 {
		return nil, nil, fmt.Errorf("lorne: unexpected status %d", res.StatusCode)
	}
	var rwc io.ReadWriteCloser
	var buf *bufio.Reader
	rwc, buf = clientconn.Hijack()
	if buf.Buffered() > 0 {
		rwc = struct {
			io.Reader
			writeCloser
		}{
			io.MultiReader(io.LimitReader(buf, int64(buf.Buffered())), rwc),
			rwc.(writeCloser),
		}
	}

	attachState := make([]byte, 1)
	if _, err := rwc.Read(attachState); err != nil {
		rwc.Close()
		return nil, nil, err
	}

	handleState := func() error {
		switch attachState[0] {
		case lorne.AttachSuccess:
			return nil
		case lorne.AttachError:
			errBytes, err := ioutil.ReadAll(rwc)
			rwc.Close()
			if err != nil {
				return err
			}
			return errors.New(string(errBytes))
		default:
			rwc.Close()
			return fmt.Errorf("lorne: unknown attach state: %d", attachState)
		}
	}

	if attachState[0] == lorne.AttachWaiting {
		if !wait {
			rwc.Close()
			return nil, nil, ErrWouldWait
		}
		return rwc.(ReadWriteCloser), func() error {
			if _, err := rwc.Read(attachState); err != nil {
				rwc.Close()
				return err
			}
			return handleState()
		}, nil
	}

	return rwc.(ReadWriteCloser), nil, handleState()
}

type ReadWriteCloser interface {
	io.ReadWriteCloser
	CloseWrite() error
}

type writeCloser interface {
	io.WriteCloser
	CloseWrite() error
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os/exec"
	"strconv"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/shared"
)

// Helper functions
func networkGetInterfaces(d *Daemon) ([]string, error) {
	networks, err := dbNetworks(d.db)
	if err != nil {
		return nil, err
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		if !shared.StringInSlice(iface.Name, networks) {
			networks = append(networks, iface.Name)
		}
	}

	return networks, nil
}

func networkIsInUse(c container, name string) bool {
	for _, d := range c.ExpandedDevices() {
		if d["type"] != "nic" {
			continue
		}

		if !shared.StringInSlice(d["nictype"], []string{"bridged", "macvlan"}) {
			continue
		}

		if d["parent"] == "" {
			continue
		}

		if d["parent"] == name {
			return true
		}
	}

	return false
}

// API endpoints
func networksGet(d *Daemon, r *http.Request) Response {
	recursionStr := r.FormValue("recursion")
	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	ifs, err := networkGetInterfaces(d)
	if err != nil {
		return InternalError(err)
	}

	resultString := []string{}
	resultMap := []shared.NetworkConfig{}
	for _, iface := range ifs {
		if recursion == 0 {
			resultString = append(resultString, fmt.Sprintf("/%s/networks/%s", shared.APIVersion, iface))
		} else {
			net, err := doNetworkGet(d, iface)
			if err != nil {
				continue
			}
			resultMap = append(resultMap, net)
		}
	}

	if recursion == 0 {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

func networksPost(d *Daemon, r *http.Request) Response {
	req := shared.NetworkConfig{}

	// Parse the request
	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	if req.Type != "" && req.Type != "bridge" {
		return BadRequest(fmt.Errorf("Only 'bridge' type networks can be created"))
	}

	networks, err := networkGetInterfaces(d)
	if err != nil {
		return InternalError(err)
	}

	if shared.StringInSlice(req.Name, networks) {
		return BadRequest(fmt.Errorf("The network already exists"))
	}

	// Create the database entry
	_, err = dbNetworkCreate(d.db, req.Name, req.Config)
	if err != nil {
		return InternalError(
			fmt.Errorf("Error inserting %s into database: %s", req.Name, err))
	}

	return SyncResponseLocation(true, nil, fmt.Sprintf("/%s/networks/%s", shared.APIVersion, req.Name))
}

var networksCmd = Command{name: "networks", get: networksGet, post: networksPost}

func networkGet(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	n, err := doNetworkGet(d, name)
	if err != nil {
		return InternalError(err)
	}

	return SyncResponse(true, &n)
}

func doNetworkGet(d *Daemon, name string) (shared.NetworkConfig, error) {
	iface, err := net.InterfaceByName(name)
	if err != nil {
		return shared.NetworkConfig{}, err
	}

	// Prepare the response
	n := shared.NetworkConfig{}
	n.Name = iface.Name
	n.UsedBy = []string{}
	n.Config = map[string]string{}

	// Look for containers using the interface
	cts, err := dbContainersList(d.db, cTypeRegular)
	if err != nil {
		return shared.NetworkConfig{}, err
	}

	for _, ct := range cts {
		c, err := containerLoadByName(d, ct)
		if err != nil {
			return shared.NetworkConfig{}, err
		}

		if networkIsInUse(c, n.Name) {
			n.UsedBy = append(n.UsedBy, fmt.Sprintf("/%s/containers/%s", shared.APIVersion, ct))
		}
	}

	// Set the device type as needed
	if shared.IsLoopback(iface) {
		n.Type = "loopback"
	} else if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", n.Name)) {
		// Look for a DB entry
		_, dbNetwork, err := dbNetworkGet(d.db, name)
		if err == nil {
			n.Managed = true
			n.Config = dbNetwork.Config
		}

		n.Type = "bridge"
	} else if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/device", n.Name)) {
		n.Type = "physical"
	} else if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/bonding", n.Name)) {
		n.Type = "bond"
	} else {
		_, err := exec.Command("ovs-vsctl", "br-exists", n.Name).CombinedOutput()
		if err == nil {
			n.Type = "bridge"
		} else {
			n.Type = "unknown"
		}
	}

	return n, nil
}

var networkCmd = Command{name: "networks/{name}", get: networkGet}
<commit_msg>network: Fix doNetworkGet for DB-only entries<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/shared"
)

// Helper functions
func networkGetInterfaces(d *Daemon) ([]string, error) {
	networks, err := dbNetworks(d.db)
	if err != nil {
		return nil, err
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		if !shared.StringInSlice(iface.Name, networks) {
			networks = append(networks, iface.Name)
		}
	}

	return networks, nil
}

func networkIsInUse(c container, name string) bool {
	for _, d := range c.ExpandedDevices() {
		if d["type"] != "nic" {
			continue
		}

		if !shared.StringInSlice(d["nictype"], []string{"bridged", "macvlan"}) {
			continue
		}

		if d["parent"] == "" {
			continue
		}

		if d["parent"] == name {
			return true
		}
	}

	return false
}

// API endpoints
func networksGet(d *Daemon, r *http.Request) Response {
	recursionStr := r.FormValue("recursion")
	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	ifs, err := networkGetInterfaces(d)
	if err != nil {
		return InternalError(err)
	}

	resultString := []string{}
	resultMap := []shared.NetworkConfig{}
	for _, iface := range ifs {
		if recursion == 0 {
			resultString = append(resultString, fmt.Sprintf("/%s/networks/%s", shared.APIVersion, iface))
		} else {
			net, err := doNetworkGet(d, iface)
			if err != nil {
				continue
			}
			resultMap = append(resultMap, net)
		}
	}

	if recursion == 0 {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

func networksPost(d *Daemon, r *http.Request) Response {
	req := shared.NetworkConfig{}

	// Parse the request
	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	if req.Type != "" && req.Type != "bridge" {
		return BadRequest(fmt.Errorf("Only 'bridge' type networks can be created"))
	}

	networks, err := networkGetInterfaces(d)
	if err != nil {
		return InternalError(err)
	}

	if shared.StringInSlice(req.Name, networks) {
		return BadRequest(fmt.Errorf("The network already exists"))
	}

	// Create the database entry
	_, err = dbNetworkCreate(d.db, req.Name, req.Config)
	if err != nil {
		return InternalError(
			fmt.Errorf("Error inserting %s into database: %s", req.Name, err))
	}

	return SyncResponseLocation(true, nil, fmt.Sprintf("/%s/networks/%s", shared.APIVersion, req.Name))
}

var networksCmd = Command{name: "networks", get: networksGet, post: networksPost}

func networkGet(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	n, err := doNetworkGet(d, name)
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, &n)
}

func doNetworkGet(d *Daemon, name string) (shared.NetworkConfig, error) {
	// Get some information
	osInfo, _ := net.InterfaceByName(name)
	_, dbInfo, _ := dbNetworkGet(d.db, name)

	// Sanity check
	if osInfo == nil && dbInfo == nil {
		return shared.NetworkConfig{}, os.ErrNotExist
	}

	// Prepare the response
	n := shared.NetworkConfig{}
	n.Name = name
	n.UsedBy = []string{}
	n.Config = map[string]string{}

	// Look for containers using the interface
	cts, err := dbContainersList(d.db, cTypeRegular)
	if err != nil {
		return shared.NetworkConfig{}, err
	}

	for _, ct := range cts {
		c, err := containerLoadByName(d, ct)
		if err != nil {
			return shared.NetworkConfig{}, err
		}

		if networkIsInUse(c, n.Name) {
			n.UsedBy = append(n.UsedBy, fmt.Sprintf("/%s/containers/%s", shared.APIVersion, ct))
		}
	}

	// Set the device type as needed
	if osInfo != nil && shared.IsLoopback(osInfo) {
		n.Type = "loopback"
	} else if dbInfo != nil || shared.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", n.Name)) {
		if dbInfo != nil {
			n.Managed = true
			n.Config = dbInfo.Config
		}

		n.Type = "bridge"
	} else if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/device", n.Name)) {
		n.Type = "physical"
	} else if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/bonding", n.Name)) {
		n.Type = "bond"
	} else {
		_, err := exec.Command("ovs-vsctl", "br-exists", n.Name).CombinedOutput()
		if err == nil {
			n.Type = "bridge"
		} else {
			n.Type = "unknown"
		}
	}

	return n, nil
}

var networkCmd = Command{name: "networks/{name}", get: networkGet}
<|endoftext|>