<commit_before>package main

/*
// todo @@@ testme
func TestMakePod(t *testing.T) {
	builder := DefaultBuildManager{buildScripts: BuildScripts{URL: "repo", Branch: "repobranch"}}

	buildEvent := v1.UserBuildEvent{Team: "ae6rt", Project: "somelib", Ref: "master", ID: "uuid"}

	projectMap := map[string]v1.Project{
		"ae6rt/somelib": v1.Project{
			Team:        "ae6rt",
			ProjectName: "somelib",
			Descriptor:  v1.ProjectDescriptor{Image: "magic-image"},
			Sidecars: []string{`
{
    "env": [
        {
            "name": "MYSQL_ROOT_PASSWORD",
            "value": "r00t"
        }
    ],
    "image": "mysql:5.6",
    "name": "mysql",
    "ports": [
        {
            "containerPort": 3306
        }
    ]
}`, `
{
    "image": "rabbitmq:3.5.4",
    "name": "rabbitmq",
    "ports": [
        {
            "containerPort": 5672
        }
    ]
}`,
			},
		},
	}

	baseContainer := builder.makeBaseContainer(buildEvent, projectMap)
	sidecars := builder.makeSidecarContainers(buildEvent, projectMap)

	var arr []k8sapi.Container
	arr = append(arr, baseContainer)
	arr = append(arr, sidecars...)

	pod := builder.makePod(buildEvent, arr)

	if pod.ObjectMeta.Name != "uuid" {
		t.Fatalf("Want uuid but got %v\n", pod.ObjectMeta.Name)
	}
	if pod.ObjectMeta.Namespace != "decap" {
		t.Fatalf("Want decap but got %v\n", pod.ObjectMeta.Namespace)
	}

	labels := pod.ObjectMeta.Labels
	if labels["type"] != "decap-build" {
		t.Fatalf("Want decap-build but got %v\n", labels["type"])
	}
	if labels["team"] != projectMap["ae6rt/somelib"].Team {
		t.Fatalf("Want ae6rt but got %v\n", labels["team"])
	}
	if labels["project"] != projectMap["ae6rt/somelib"].ProjectName {
		t.Fatalf("Want somelib but got %v\n", labels["project"])
	}

	if len(pod.Spec.Volumes) != 2 {
		t.Fatalf("Want 2 but got %v\n", len(pod.Spec.Volumes))
	}

	volume := pod.Spec.Volumes[0]
	if volume.Name != "build-scripts" {
		t.Fatalf("Want build-scripts but got %v\n", volume.Name)
	}
	if volume.VolumeSource.GitRepo.Repository != "repo" {
		t.Fatalf("Want repo but got %v\n", volume.VolumeSource.GitRepo.Repository)
	}
	if volume.VolumeSource.GitRepo.Revision != "repobranch" {
		t.Fatalf("Want repobranch but got %v\n", volume.VolumeSource.GitRepo.Revision)
	}

	volume = pod.Spec.Volumes[1]
	if volume.Name != "decap-credentials" {
		t.Fatalf("Want decap-credentials but got %v\n", volume.Name)
	}
	if volume.VolumeSource.Secret.SecretName != "decap-credentials" {
		t.Fatalf("Want repo but got %v\n", volume.VolumeSource.Secret.SecretName)
	}

	if len(pod.Spec.Containers) != 1+len(sidecars) {
		t.Fatalf("Want %d but got %v\n", 1+len(sidecars), len(pod.Spec.Containers))
	}

	if pod.Spec.RestartPolicy != "Never" {
		t.Fatalf("Want Never but got %v\n", pod.Spec.RestartPolicy)
	}

}
*/
<commit_msg>[issue/39]  fix makepod_test.go<commit_after>package main

import (
	"testing"

	"github.com/ae6rt/decap/web/api/v1"
	k8sapi "k8s.io/client-go/pkg/api/v1"
)

type MakePodProjectManagerMock struct {
	ProjectManagerBaseMock
	project v1.Project
	url     string
	branch  string
}

func (t *MakePodProjectManagerMock) Get(key string) *v1.Project {
	return &t.project
}

func (t *MakePodProjectManagerMock) RepositoryURL() string {
	return t.url
}

func (t *MakePodProjectManagerMock) RepositoryBranch() string {
	return t.branch
}

func TestMakePod(t *testing.T) {
	var tests = []struct {
		team        string
		projectName string
		ref         string
		ID          string
		project     v1.Project
	}{
		{
			team:        "ae6rt",
			projectName: "somelib",
			ref:         "master",
			ID:          "uuid",
			project: v1.Project{
				Team:        "ae6rt",
				ProjectName: "somelib",
				Descriptor:  v1.ProjectDescriptor{Image: "magic-image"},
				Sidecars: []string{
					`{"env":[{"name":"MYSQL_ROOT_PASSWORD","value":"r00t"}],"image":"mysql:5.6","name":"mysql","ports":[{"containerPort":3306}]}`,
					`{"image":"rabbitmq:3.5.4","name":"rabbitmq","ports":[{"containerPort":5672}]}`,
				},
			},
		},
	}
	for testNumber, test := range tests {
		projectManager := &MakePodProjectManagerMock{project: test.project, url: "repo", branch: "repobranch"}
		builder := DefaultBuildManager{projectManager: projectManager}
		buildEvent := v1.UserBuildEvent{Team: test.team, Project: test.projectName, Ref: test.ref, ID: test.ID}

		baseContainer := builder.makeBaseContainer(buildEvent)
		sidecars := builder.makeSidecarContainers(buildEvent)

		var arr []k8sapi.Container
		arr = append(arr, baseContainer)
		arr = append(arr, sidecars...)

		pod := builder.makePod(buildEvent, arr)

		if pod.ObjectMeta.Name != test.ID {
			t.Errorf("Test %d: Want uuid but got %v\n", testNumber, pod.ObjectMeta.Name)
		}

		if pod.ObjectMeta.Namespace != "decap" {
			t.Errorf("Test %d: Want decap but got %v\n", testNumber, pod.ObjectMeta.Namespace)
		}

		labels := pod.ObjectMeta.Labels
		if labels["type"] != "decap-build" {
			t.Errorf("Test %d: want decap-build but got %v\n", testNumber, labels["type"])
		}
		if labels["team"] != projectManager.Get(buildEvent.ProjectKey()).Team {
			t.Errorf("Test %d: want %s but got %v\n", testNumber, buildEvent.Team, labels["team"])
		}
		if labels["project"] != projectManager.Get(buildEvent.ProjectKey()).ProjectName {
			t.Errorf("Test %d: want somelib but got %v\n", testNumber, labels["project"])
		}

		if len(pod.Spec.Volumes) != 2 {
			t.Errorf("Test %d: want 2 but got %v\n", testNumber, len(pod.Spec.Volumes))
		}

		volume := pod.Spec.Volumes[0]
		if volume.Name != "build-scripts" {
			t.Errorf("Test %d: want build-scripts but got %v\n", testNumber, volume.Name)
		}
		if volume.VolumeSource.GitRepo.Repository != "repo" {
			t.Errorf("Test %d: want repo but got %v\n", testNumber, volume.VolumeSource.GitRepo.Repository)
		}
		if volume.VolumeSource.GitRepo.Revision != "repobranch" {
			t.Errorf("Test %d: want repobranch but got %v\n", testNumber, volume.VolumeSource.GitRepo.Revision)
		}

		volume = pod.Spec.Volumes[1]
		if volume.Name != "decap-credentials" {
			t.Errorf("Test %d: want decap-credentials but got %v\n", testNumber, volume.Name)
		}
		if volume.VolumeSource.Secret.SecretName != "decap-credentials" {
			t.Errorf("Test %d: want repo but got %v\n", testNumber, volume.VolumeSource.Secret.SecretName)
		}

		if len(pod.Spec.Containers) != 1+len(sidecars) {
			t.Errorf("Test %d: want %d but got %v\n", testNumber, 1+len(sidecars), len(pod.Spec.Containers))
		}

		if pod.Spec.RestartPolicy != "Never" {
			t.Errorf("Test %d: want Never but got %v\n", testNumber, pod.Spec.RestartPolicy)
		}
	}
}
<|endoftext|>


<commit_before>package vagrant_test

import (
	"os"
	"testing"

	"github.com/mlafeldt/chef-runner/driver"
	. "github.com/mlafeldt/chef-runner/driver/vagrant"
	"github.com/mlafeldt/chef-runner/log"
	"github.com/mlafeldt/chef-runner/util"
	"github.com/stretchr/testify/assert"
)

func init() {
	// Be quiet during testing
	log.SetLevel(log.LevelWarn)
}

func TestDriverInterface(t *testing.T) {
	assert.Implements(t, (*driver.Driver)(nil), new(Driver))
}

func TestNewDriver(t *testing.T) {
	util.InDir("../../testdata", func() {
		oldPath := os.Getenv("PATH")
		os.Setenv("PATH", "./bin:/usr/bin:/bin")
		defer os.Setenv("PATH", oldPath)

		drv, err := NewDriver("some-machine")
		if assert.NoError(t, err) {
			defer os.RemoveAll(".chef-runner")
			assert.Equal(t, "default", drv.SSHClient.Host)
			assert.Equal(t, ".chef-runner/vagrant/machines/some-machine/ssh_config",
				drv.SSHClient.ConfigFile)
			assert.Equal(t, "default", drv.RsyncClient.RemoteHost)
		}
	})
}

func TestString(t *testing.T) {
	expect := "Vagrant driver (machine: some-machine)"
	actual := Driver{Machine: "some-machine"}.String()
	assert.Equal(t, expect, actual)
}
<commit_msg>Use os.PathListSeparator<commit_after>package vagrant_test

import (
	"os"
	"strings"
	"testing"

	"github.com/mlafeldt/chef-runner/driver"
	. "github.com/mlafeldt/chef-runner/driver/vagrant"
	"github.com/mlafeldt/chef-runner/log"
	"github.com/mlafeldt/chef-runner/util"
	"github.com/stretchr/testify/assert"
)

func init() {
	// Be quiet during testing
	log.SetLevel(log.LevelWarn)
}

func TestDriverInterface(t *testing.T) {
	assert.Implements(t, (*driver.Driver)(nil), new(Driver))
}

func TestNewDriver(t *testing.T) {
	util.InDir("../../testdata", func() {
		oldPath := os.Getenv("PATH")
		os.Setenv("PATH", strings.Join([]string{"bin", oldPath},
			string(os.PathListSeparator)))
		defer os.Setenv("PATH", oldPath)

		drv, err := NewDriver("some-machine")
		if assert.NoError(t, err) {
			defer os.RemoveAll(".chef-runner")
			assert.Equal(t, "default", drv.SSHClient.Host)
			assert.Equal(t, ".chef-runner/vagrant/machines/some-machine/ssh_config",
				drv.SSHClient.ConfigFile)
			assert.Equal(t, "default", drv.RsyncClient.RemoteHost)
		}
	})
}

func TestString(t *testing.T) {
	expect := "Vagrant driver (machine: some-machine)"
	actual := Driver{Machine: "some-machine"}.String()
	assert.Equal(t, expect, actual)
}
<|endoftext|>


<commit_before>package service

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"runtime"
	"time"

	"math"
	"os"
	"strconv"
	"sync"

	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

func (app *goActiveLearningApp) UpdateOrCreateExample(e *model.Example) error {
	return app.repo.UpdateOrCreateExample(e)
}

func (app *goActiveLearningApp) UpdateScore(e *model.Example) error {
	return app.repo.UpdateScore(e)
}

func (app *goActiveLearningApp) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	return app.repo.InsertExampleFromScanner(scanner)
}

func (app *goActiveLearningApp) InsertExamplesFromReader(reader io.Reader) error {
	return app.repo.InsertExamplesFromReader(reader)
}

func (app *goActiveLearningApp) SearchExamples() (model.Examples, error) {
	return app.repo.SearchExamples()
}

func (app *goActiveLearningApp) SearchRecentExamples(from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamples(from, limit)
}

func (app *goActiveLearningApp) SearchRecentExamplesByHost(host string, from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamplesByHost(host, from, limit)
}

func (app *goActiveLearningApp) SearchExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByLabel(label, limit)
}

func (app *goActiveLearningApp) SearchLabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchLabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveExamples(limit)
}

func (app *goActiveLearningApp) SearchNegativeExamples(limit int) (model.Examples, error) {
	return app.repo.SearchNegativeExamples(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchUnlabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveScoredExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveScoredExamples(limit)
}

func (app *goActiveLearningApp) FindExampleByUlr(url string) (*model.Example, error) {
	return app.repo.FindExampleByUlr(url)
}

func (app *goActiveLearningApp) FindExampleById(id int) (*model.Example, error) {
	return app.repo.FindExampleById(id)
}

func (app *goActiveLearningApp) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	return app.repo.SearchExamplesByUlrs(urls)
}

func (app *goActiveLearningApp) SearchExamplesByIds(ids []int) (model.Examples, error) {
	return app.repo.SearchExamplesByIds(ids)
}

func (app *goActiveLearningApp) SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByKeywords(keywords, aggregator, limit)
}

func (app *goActiveLearningApp) DeleteAllExamples() error {
	return app.repo.DeleteAllExamples()
}

func (app *goActiveLearningApp) CountPositiveExamples() (int, error) {
	return app.repo.CountPositiveExamples()
}

func (app *goActiveLearningApp) CountNegativeExamples() (int, error) {
	return app.repo.CountNegativeExamples()
}

func (app *goActiveLearningApp) CountUnlabeledExamples() (int, error) {
	return app.repo.CountUnlabeledExamples()
}

func (app *goActiveLearningApp) UpdateFeatureVector(e *model.Example) error {
	return app.repo.UpdateFeatureVector(e)
}

func (app *goActiveLearningApp) UpdateHatenaBookmark(e *model.Example) error {
	return app.repo.UpdateHatenaBookmark(e)
}

func (app *goActiveLearningApp) UpdateOrCreateReferringTweets(e *model.Example) error {
	return app.repo.UpdateOrCreateReferringTweets(e)
}

func (app *goActiveLearningApp) UpdateTweetLabel(exampleId int, idStr string, label model.LabelType) error {
	return app.repo.UpdateTweetLabel(exampleId, idStr, label)
}

func (app *goActiveLearningApp) SearchReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchPositiveReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchPositiveReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchNegativeReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchNegativeReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchUnlabeledReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchRecentReferringTweetsWithHighScore(from time.Time, scoreThreshold float64, limit int) (model.ReferringTweets, error) {
	return app.repo.SearchRecentReferringTweetsWithHighScore(from, scoreThreshold, limit)
}

func hatenaBookmarkByExampleId(hatenaBookmarks []*model.HatenaBookmark) map[int]*model.HatenaBookmark {
	result := make(map[int]*model.HatenaBookmark)
	for _, hb := range hatenaBookmarks {
		result[hb.ExampleId] = hb
	}
	return result
}

func (app *goActiveLearningApp) AttachMetadataIncludingFeatureVector(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	// make sure that example id must be filled
	for _, e := range examples {
		if e.Id == 0 {
			tmp, err := app.FindExampleByUlr(e.Url)
			if err != nil {
				return err
			}
			e.Id = tmp.Id
		}
	}

	fvList, err := app.repo.SearchFeatureVector(examples)
	if err != nil {
		return err
	}

	for _, e := range examples {
		if fv, ok := fvList[e.Id]; ok {
			e.Fv = fv
		}
	}

	return app.AttachMetadata(examples, bookmarkLimit, tweetLimit)
}

func (app *goActiveLearningApp) AttachMetadata(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples, bookmarkLimit)
	if err != nil {
		return err
	}
	hbByid := hatenaBookmarkByExampleId(hatenaBookmarks)
	for _, e := range examples {
		if b, ok := hbByid[e.Id]; ok {
			e.HatenaBookmark = b
		} else {
			e.HatenaBookmark = &model.HatenaBookmark{Bookmarks: []*model.Bookmark{}}
		}
	}

	referringTweetsById, err := app.repo.SearchReferringTweetsList(examples, tweetLimit)
	if err != nil {
		return err
	}
	for _, e := range examples {
		if t, ok := referringTweetsById[e.Id]; ok {
			e.ReferringTweets = &t
		} else {
			e.ReferringTweets = &model.ReferringTweets{}
		}
	}
	return nil
}

func (app *goActiveLearningApp) UpdateRecommendation(listName string, examples model.Examples) error {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return err
	}

	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	rec := model.Recommendation{RecommendationListType: listType, ExampleIds: exampleIds}
	return app.repo.UpdateRecommendation(rec)
}

func (app *goActiveLearningApp) GetRecommendation(listName string) (model.Examples, error) {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return nil, err
	}
	rec, err := app.repo.FindRecommendation(listType)
	return app.repo.SearchExamplesByIds(rec.ExampleIds)
}

func (app *goActiveLearningApp) splitExamplesByStatusOK(examples model.Examples) (model.Examples, model.Examples, error) {
	urls := make([]string, 0)
	exampleByurl := make(map[string]*model.Example)
	for _, e := range examples {
		exampleByurl[e.Url] = e
		urls = append(urls, e.Url)
	}
	tmpExamples, err := app.SearchExamplesByUlrs(urls)
	if err != nil {
		return nil, nil, err
	}

	examplesWithMetaData := model.Examples{}
	examplesWithEmptyMetaData := model.Examples{}
	for _, e := range tmpExamples {
		if e.StatusCode == http.StatusOK {
			examplesWithMetaData = append(examplesWithMetaData, exampleByurl[e.Url])
			delete(exampleByurl, e.Url)
		} else {
			examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, exampleByurl[e.Url])
			delete(exampleByurl, e.Url)
		}
	}
	for _, e := range exampleByurl {
		examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
	}
	return examplesWithMetaData, examplesWithEmptyMetaData, nil
}

func fetchMetaData(e *model.Example) error {
	article, err := fetcher.GetArticle(e.Url)
	if err != nil {
		return err
	}

	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.OgType = article.OgType
	e.OgImage = article.OgImage
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Favicon = article.Favicon

	now := time.Now()
	if article.PublishDate != nil && now.After(*article.PublishDate) {
		e.CreatedAt = *article.PublishDate
		e.UpdatedAt = *article.PublishDate
	}

	fv := util.RemoveDuplicate(example.ExtractFeatures(*e))
	if len(fv) > 100000 {
		return fmt.Errorf("too large features (N = %d) for %s", len(fv), e.FinalUrl)
	}
	e.Fv = fv

	return nil
}

func (app *goActiveLearningApp) Fetch(examples model.Examples) {
	batchSize := 100
	examplesList := make([]model.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		examplesWithMetaData, examplesWithEmptyMetaData, err := app.splitExamplesByStatusOK(l)
		if err != nil {
			log.Println(err.Error())
		}
		// ToDo: 本当に必要か考える
		app.AttachMetadataIncludingFeatureVector(examplesWithMetaData, 0, 0)

		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *model.Example, idx int) {
				defer wg.Done()
				cnt, err := app.repo.GetErrorCount(e)
				if err != nil {
					log.Println(err.Error())
				}
				if cnt < 5 {
					fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
					if err := fetchMetaData(e); err != nil {
						app.repo.IncErrorCount(e)
						log.Println(err.Error())
					}
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}
<commit_msg>異常に古いtimestampはおかしいので現在時刻を差し込む<commit_after>package service

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"runtime"
	"time"

	"math"
	"os"
	"strconv"
	"sync"

	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

func (app *goActiveLearningApp) UpdateOrCreateExample(e *model.Example) error {
	return app.repo.UpdateOrCreateExample(e)
}

func (app *goActiveLearningApp) UpdateScore(e *model.Example) error {
	return app.repo.UpdateScore(e)
}

func (app *goActiveLearningApp) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	return app.repo.InsertExampleFromScanner(scanner)
}

func (app *goActiveLearningApp) InsertExamplesFromReader(reader io.Reader) error {
	return app.repo.InsertExamplesFromReader(reader)
}

func (app *goActiveLearningApp) SearchExamples() (model.Examples, error) {
	return app.repo.SearchExamples()
}

func (app *goActiveLearningApp) SearchRecentExamples(from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamples(from, limit)
}

func (app *goActiveLearningApp) SearchRecentExamplesByHost(host string, from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamplesByHost(host, from, limit)
}

func (app *goActiveLearningApp) SearchExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByLabel(label, limit)
}

func (app *goActiveLearningApp) SearchLabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchLabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveExamples(limit)
}

func (app *goActiveLearningApp) SearchNegativeExamples(limit int) (model.Examples, error) {
	return app.repo.SearchNegativeExamples(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchUnlabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveScoredExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveScoredExamples(limit)
}

func (app *goActiveLearningApp) FindExampleByUlr(url string) (*model.Example, error) {
	return app.repo.FindExampleByUlr(url)
}

func (app *goActiveLearningApp) FindExampleById(id int) (*model.Example, error) {
	return app.repo.FindExampleById(id)
}

func (app *goActiveLearningApp) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	return app.repo.SearchExamplesByUlrs(urls)
}

func (app *goActiveLearningApp) SearchExamplesByIds(ids []int) (model.Examples, error) {
	return app.repo.SearchExamplesByIds(ids)
}

func (app *goActiveLearningApp) SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByKeywords(keywords, aggregator, limit)
}

func (app *goActiveLearningApp) DeleteAllExamples() error {
	return app.repo.DeleteAllExamples()
}

func (app *goActiveLearningApp) CountPositiveExamples() (int, error) {
	return app.repo.CountPositiveExamples()
}

func (app *goActiveLearningApp) CountNegativeExamples() (int, error) {
	return app.repo.CountNegativeExamples()
}

func (app *goActiveLearningApp) CountUnlabeledExamples() (int, error) {
	return app.repo.CountUnlabeledExamples()
}

func (app *goActiveLearningApp) UpdateFeatureVector(e *model.Example) error {
	return app.repo.UpdateFeatureVector(e)
}

func (app *goActiveLearningApp) UpdateHatenaBookmark(e *model.Example) error {
	return app.repo.UpdateHatenaBookmark(e)
}

func (app *goActiveLearningApp) UpdateOrCreateReferringTweets(e *model.Example) error {
	return app.repo.UpdateOrCreateReferringTweets(e)
}

func (app *goActiveLearningApp) UpdateTweetLabel(exampleId int, idStr string, label model.LabelType) error {
	return app.repo.UpdateTweetLabel(exampleId, idStr, label)
}

func (app *goActiveLearningApp) SearchReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchPositiveReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchPositiveReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchNegativeReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchNegativeReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchUnlabeledReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchRecentReferringTweetsWithHighScore(from time.Time, scoreThreshold float64, limit int) (model.ReferringTweets, error) {
	return app.repo.SearchRecentReferringTweetsWithHighScore(from, scoreThreshold, limit)
}

func hatenaBookmarkByExampleId(hatenaBookmarks []*model.HatenaBookmark) map[int]*model.HatenaBookmark {
	result := make(map[int]*model.HatenaBookmark)
	for _, hb := range hatenaBookmarks {
		result[hb.ExampleId] = hb
	}
	return result
}

func (app *goActiveLearningApp) AttachMetadataIncludingFeatureVector(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	// make sure that example id must be filled
	for _, e := range examples {
		if e.Id == 0 {
			tmp, err := app.FindExampleByUlr(e.Url)
			if err != nil {
				return err
			}
			e.Id = tmp.Id
		}
	}

	fvList, err := app.repo.SearchFeatureVector(examples)
	if err != nil {
		return err
	}

	for _, e := range examples {
		if fv, ok := fvList[e.Id]; ok {
			e.Fv = fv
		}
	}

	return app.AttachMetadata(examples, bookmarkLimit, tweetLimit)
}

func (app *goActiveLearningApp) AttachMetadata(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples, bookmarkLimit)
	if err != nil {
		return err
	}
	hbByid := hatenaBookmarkByExampleId(hatenaBookmarks)
	for _, e := range examples {
		if b, ok := hbByid[e.Id]; ok {
			e.HatenaBookmark = b
		} else {
			e.HatenaBookmark = &model.HatenaBookmark{Bookmarks: []*model.Bookmark{}}
		}
	}

	referringTweetsById, err := app.repo.SearchReferringTweetsList(examples, tweetLimit)
	if err != nil {
		return err
	}
	for _, e := range examples {
		if t, ok := referringTweetsById[e.Id]; ok {
			e.ReferringTweets = &t
		} else {
			e.ReferringTweets = &model.ReferringTweets{}
		}
	}
	return nil
}

func (app *goActiveLearningApp) UpdateRecommendation(listName string, examples model.Examples) error {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return err
	}

	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	rec := model.Recommendation{RecommendationListType: listType, ExampleIds: exampleIds}
	return app.repo.UpdateRecommendation(rec)
}

func (app *goActiveLearningApp) GetRecommendation(listName string) (model.Examples, error) {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return nil, err
	}
	rec, err := app.repo.FindRecommendation(listType)
	return app.repo.SearchExamplesByIds(rec.ExampleIds)
}

func (app *goActiveLearningApp) splitExamplesByStatusOK(examples model.Examples) (model.Examples, model.Examples, error) {
	urls := make([]string, 0)
	exampleByurl := make(map[string]*model.Example)
	for _, e := range examples {
		exampleByurl[e.Url] = e
		urls = append(urls, e.Url)
	}
	tmpExamples, err := app.SearchExamplesByUlrs(urls)
	if err != nil {
		return nil, nil, err
	}

	examplesWithMetaData := model.Examples{}
	examplesWithEmptyMetaData := model.Examples{}
	for _, e := range tmpExamples {
		if e.StatusCode == http.StatusOK {
			examplesWithMetaData = append(examplesWithMetaData, exampleByurl[e.Url])
			delete(exampleByurl, e.Url)
		} else {
			examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, exampleByurl[e.Url])
			delete(exampleByurl, e.Url)
		}
	}
	for _, e := range exampleByurl {
		examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
	}
	return examplesWithMetaData, examplesWithEmptyMetaData, nil
}

func fetchMetaData(e *model.Example) error {
	article, err := fetcher.GetArticle(e.Url)
	if err != nil {
		return err
	}

	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.OgType = article.OgType
	e.OgImage = article.OgImage
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Favicon = article.Favicon

	now := time.Now()
	tooOldDate := time.Date(2000, time.January, 1, 1, 1, 0, 0, time.UTC)
	if article.PublishDate != nil && (now.After(*article.PublishDate) || tooOldDate.Before(*article.PublishDate)) {
		e.CreatedAt = *article.PublishDate
		e.UpdatedAt = *article.PublishDate
	}

	fv := util.RemoveDuplicate(example.ExtractFeatures(*e))
	if len(fv) > 100000 {
		return fmt.Errorf("too large features (N = %d) for %s", len(fv), e.FinalUrl)
	}
	e.Fv = fv

	return nil
}

func (app *goActiveLearningApp) Fetch(examples model.Examples) {
	batchSize := 100
	examplesList := make([]model.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		examplesWithMetaData, examplesWithEmptyMetaData, err := app.splitExamplesByStatusOK(l)
		if err != nil {
			log.Println(err.Error())
		}
		// ToDo: 本当に必要か考える
		app.AttachMetadataIncludingFeatureVector(examplesWithMetaData, 0, 0)

		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *model.Example, idx int) {
				defer wg.Done()
				cnt, err := app.repo.GetErrorCount(e)
				if err != nil {
					log.Println(err.Error())
				}
				if cnt < 5 {
					fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
					if err := fetchMetaData(e); err != nil {
						app.repo.IncErrorCount(e)
						log.Println(err.Error())
					}
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}
<|endoftext|>


<commit_before>package pkcs11uri

import (
	"fmt"
	"testing"

	"github.com/cloudflare/cfssl/crypto/pkcs11key"
)

type pkcs11UriTest struct {
	URI    string
	Config *pkcs11key.Config
}

func cmpConfigs(a, b *pkcs11key.Config) bool {
	if a == nil {
		if b == nil {
			return true
		}
		return false
	}

	if b == nil {
		return false
	}

	return (a.Module == b.Module) &&
		(a.TokenLabel == b.TokenLabel) &&
		(a.PIN == b.PIN) &&
		(a.PrivateKeyLabel == b.PrivateKeyLabel)
}

func diffConfigs(want, have *pkcs11key.Config) {
	if have == nil && want != nil {
		fmt.Printf("Expected config, have nil.")
		return
	} else if have == nil && want == nil {
		return
	}

	diff := func(kind, v1, v2 string) {
		if v1 != v2 {
			fmt.Printf("%s: want '%s', have '%s'\n", kind, v1, v2)
		}
	}

	diff("Module", want.Module, have.Module)
	diff("TokenLabel", want.TokenLabel, have.TokenLabel)
	diff("PIN", want.PIN, have.PIN)
	diff("PrivateKeyLabel", want.PrivateKeyLabel, have.PrivateKeyLabel)
}

/* Config from PKCS #11 signer
type Config struct {
	Module string
	Token  string
	PIN    string
	Label  string
}
*/

var pkcs11UriCases = []pkcs11UriTest{
	{"pkcs11:token=Software%20PKCS%2311%20softtoken;manufacturer=Snake%20Oil,%20Inc.?pin-value=the-pin",
		&pkcs11key.Config{
			TokenLabel: "Software PKCS#11 softtoken",
			PIN:        "the-pin",
		}},
	{"pkcs11:token=Sun%20Token",
		&pkcs11key.Config{
			TokenLabel: "Sun Token",
		}},
	{"pkcs11:object=test-privkey;token=test-token?pin-source=file:testdata/pin&module-name=test-module",
		&pkcs11key.Config{
			PrivateKeyLabel: "test-privkey",
			TokenLabel:      "test-token",
			PIN:             "123456",
			Module:          "test-module",
		}},
}

func TestParse(t *testing.T) {
	for _, c := range pkcs11UriCases {
		cfg, err := Parse(c.URI)
		if err != nil {
			t.Fatalf("Failed on URI '%s'", c.URI)
		}
		if !cmpConfigs(c.Config, cfg) {
			diffConfigs(c.Config, cfg)
			t.Fatal("Configs don't match.")
		}
	}
}

var pkcs11UriFails = []string{
	"https://github.com/cloudflare/cfssl",
	"pkcs11:?pin-source=http://foo",
	"pkcs11:?pin-source=file:testdata/nosuchfile",
}

func TestParse(t *testing.T) {
	for _, c := range pkcs11UriFails {
		_, err := Parse(c)
		if err == nil {
			t.Fatalf("Expected URI '%s' to fail to parse.", c)
		}
	}
}
<commit_msg>Fix conflicting test names.<commit_after>package pkcs11uri

import (
	"fmt"
	"testing"

	"github.com/cloudflare/cfssl/crypto/pkcs11key"
)

type pkcs11UriTest struct {
	URI    string
	Config *pkcs11key.Config
}

func cmpConfigs(a, b *pkcs11key.Config) bool {
	if a == nil {
		if b == nil {
			return true
		}
		return false
	}

	if b == nil {
		return false
	}

	return (a.Module == b.Module) &&
		(a.TokenLabel == b.TokenLabel) &&
		(a.PIN == b.PIN) &&
		(a.PrivateKeyLabel == b.PrivateKeyLabel)
}

func diffConfigs(want, have *pkcs11key.Config) {
	if have == nil && want != nil {
		fmt.Printf("Expected config, have nil.")
		return
	} else if have == nil && want == nil {
		return
	}

	diff := func(kind, v1, v2 string) {
		if v1 != v2 {
			fmt.Printf("%s: want '%s', have '%s'\n", kind, v1, v2)
		}
	}

	diff("Module", want.Module, have.Module)
	diff("TokenLabel", want.TokenLabel, have.TokenLabel)
	diff("PIN", want.PIN, have.PIN)
	diff("PrivateKeyLabel", want.PrivateKeyLabel, have.PrivateKeyLabel)
}

/* Config from PKCS #11 signer
type Config struct {
	Module string
	Token  string
	PIN    string
	Label  string
}
*/

var pkcs11UriCases = []pkcs11UriTest{
	{"pkcs11:token=Software%20PKCS%2311%20softtoken;manufacturer=Snake%20Oil,%20Inc.?pin-value=the-pin",
		&pkcs11key.Config{
			TokenLabel: "Software PKCS#11 softtoken",
			PIN:        "the-pin",
		}},
	{"pkcs11:token=Sun%20Token",
		&pkcs11key.Config{
			TokenLabel: "Sun Token",
		}},
	{"pkcs11:object=test-privkey;token=test-token?pin-source=file:testdata/pin&module-name=test-module",
		&pkcs11key.Config{
			PrivateKeyLabel: "test-privkey",
			TokenLabel:      "test-token",
			PIN:             "123456",
			Module:          "test-module",
		}},
}

func TestParseSuccess(t *testing.T) {
	for _, c := range pkcs11UriCases {
		cfg, err := Parse(c.URI)
		if err != nil {
			t.Fatalf("Failed on URI '%s'", c.URI)
		}
		if !cmpConfigs(c.Config, cfg) {
			diffConfigs(c.Config, cfg)
			t.Fatal("Configs don't match.")
		}
	}
}

var pkcs11UriFails = []string{
	"https://github.com/cloudflare/cfssl",
	"pkcs11:?pin-source=http://foo",
	"pkcs11:?pin-source=file:testdata/nosuchfile",
}

func TestParseFail(t *testing.T) {
	for _, c := range pkcs11UriFails {
		_, err := Parse(c)
		if err == nil {
			t.Fatalf("Expected URI '%s' to fail to parse.", c)
		}
	}
}
<|endoftext|>


<commit_before>package hb

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	defaultBaseURL = "https://hummingbird.me/"
)

// Client manages communication with the Hummingbird API.
type Client struct {
	client *http.Client

	BaseURL *url.URL

	User    *UserService
	Anime   *AnimeService
	Library *LibraryService
}

// NewClient returns a new Hummingbird API client.
func NewClient(httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: httpClient, BaseURL: baseURL}

	c.User = &UserService{client: c}
	c.Anime = &AnimeService{client: c}
	c.Library = &LibraryService{client: c}
	return c
}

// NewRequest creates an API request. A relative URL can be provided in urlStr,
// in which case it is resolved relative to the BaseURL of the Client.
// Relative URLs should always be specified without a preceding slash. If body
// is passed as an argument, then it will be encoded to JSON and used as the
// request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	url, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(url)

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, fmt.Errorf("cannot encode body: %v", err)
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/json")

	return req, nil
}

// Do sends an API request and returns the API response. If an API error has
// occurred both the response and the error will be returned in case the caller
// wishes to further inspect the response. If v is passed as an argument, then
// the API response is JSON decoded and stored to v.
func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	err = checkResponse(resp)
	if err != nil {
		return resp, err
	}

	if v != nil {
		err = json.NewDecoder(resp.Body).Decode(v)
	}
	return resp, err
}

// checkResponse checks the API response for errors. A response is considered an
// error if it has status code outside the 200 range. API error responses are
// expected to have a JSON response body that maps to ErrorResponse. Other
// response bodies are ignored.
func checkResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	body, err := ioutil.ReadAll(r.Body)
	if err == nil && body != nil {
		json.Unmarshal(body, errorResponse)
	}
	return errorResponse
}

// ErrorResponse represents a Hummingbird API error response.
type ErrorResponse struct {
	Response *http.Response
	Message  string `json:"error"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %v %v", r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Message)
}
<commit_msg>Fix shadowing of err<commit_after>package hb

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	defaultBaseURL = "https://hummingbird.me/"
)

// Client manages communication with the Hummingbird API.
type Client struct {
	client *http.Client

	BaseURL *url.URL

	User    *UserService
	Anime   *AnimeService
	Library *LibraryService
}

// NewClient returns a new Hummingbird API client.
func NewClient(httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: httpClient, BaseURL: baseURL}

	c.User = &UserService{client: c}
	c.Anime = &AnimeService{client: c}
	c.Library = &LibraryService{client: c}
	return c
}

// NewRequest creates an API request. A relative URL can be provided in urlStr,
// in which case it is resolved relative to the BaseURL of the Client.
// Relative URLs should always be specified without a preceding slash. If body
// is passed as an argument, then it will be encoded to JSON and used as the
// request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	url, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(url)

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err = json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, fmt.Errorf("cannot encode body: %v", err)
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/json")

	return req, nil
}

// Do sends an API request and returns the API response. If an API error has
// occurred both the response and the error will be returned in case the caller
// wishes to further inspect the response. If v is passed as an argument, then
// the API response is JSON decoded and stored to v.
func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	err = checkResponse(resp)
	if err != nil {
		return resp, err
	}

	if v != nil {
		err = json.NewDecoder(resp.Body).Decode(v)
	}
	return resp, err
}

// checkResponse checks the API response for errors. A response is considered an
// error if it has status code outside the 200 range. API error responses are
// expected to have a JSON response body that maps to ErrorResponse. Other
// response bodies are ignored.
func checkResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	body, err := ioutil.ReadAll(r.Body)
	if err == nil && body != nil {
		json.Unmarshal(body, errorResponse)
	}
	return errorResponse
}

// ErrorResponse represents a Hummingbird API error response.
type ErrorResponse struct {
	Response *http.Response
	Message  string `json:"error"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %v %v", r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Message)
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {

    fmt.Println("Hello")

}
<commit_msg>change: hello.go not needed<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

// ClusterAutoscalerVersion contains version of CA.
const ClusterAutoscalerVersion = "0.3.0"
<commit_msg>Cluster-autoscaler: bump version to 0.4.0-alpha<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

// ClusterAutoscalerVersion contains version of CA.
const ClusterAutoscalerVersion = "0.4.0-alpha1"
<|endoftext|>


<commit_before>package etcd

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/AsynkronIT/protoactor-go/cluster"
	"github.com/AsynkronIT/protoactor-go/log"
	"github.com/coreos/etcd/clientv3"
)

var (
	plog = log.New(log.InfoLevel, "[CLUSTER/ETCD]")
)

type Provider struct {
	cluster       *cluster.Cluster
	baseKey       string
	clusterName   string
	deregistered  bool
	shutdown      bool
	self          *Node
	members       map[string]*Node // all, contains self.
	clusterError  error
	client        *clientv3.Client
	leaseID       clientv3.LeaseID
	cancelWatch   func()
	cancelWatchCh chan bool
	keepAliveTTL  time.Duration
	retryInterval time.Duration
	revision      uint64
	// deregisterCritical time.Duration
}

func New() (*Provider, error) {
	return NewWithConfig("/protoactor", clientv3.Config{
		Endpoints:   []string{"127.0.0.1:2379"},
		DialTimeout: time.Second * 5,
	})
}

func NewWithConfig(baseKey string, cfg clientv3.Config) (*Provider, error) {
	client, err := clientv3.New(cfg)
	if err != nil {
		return nil, err
	}
	p := &Provider{
		client:        client,
		keepAliveTTL:  3 * time.Second,
		retryInterval: 1 * time.Second,
		baseKey:       baseKey,
		members:       map[string]*Node{},
		cancelWatchCh: make(chan bool),
	}
	return p, nil
}

func (p *Provider) init(c *cluster.Cluster) error {
	p.cluster = c
	addr := p.cluster.ActorSystem.Address()
	host, port, err := splitHostPort(addr)
	if err != nil {
		return err
	}

	p.cluster = c
	p.clusterName = p.cluster.Config.Name
	knownKinds := c.GetClusterKinds()
	nodeName := fmt.Sprintf("%v@%v:%v", p.clusterName, host, port)
	p.self = NewNode(nodeName, host, port, knownKinds)
	p.self.SetMeta("id", p.getID())
	return nil
}

func (p *Provider) StartMember(c *cluster.Cluster) error {
	if err := p.init(c); err != nil {
		return err
	}

	// fetch memberlist
	nodes, err := p.fetchNodes()
	if err != nil {
		return err
	}
	// initialize members
	p.updateNodesWithSelf(nodes)
	p.publishClusterTopologyEvent()
	p.startWatching()

	// register self
	if err := p.registerService(); err != nil {
		return err
	}
	ctx := context.TODO()
	p.startKeepAlive(ctx)
	return nil
}

func (p *Provider) StartClient(c *cluster.Cluster) error {
	if err := p.init(c); err != nil {
		return err
	}
	nodes, err := p.fetchNodes()
	if err != nil {
		return err
	}
	// initialize members
	p.updateNodes(nodes)
	p.publishClusterTopologyEvent()
	p.startWatching()
	return nil
}

func (p *Provider) Shutdown(graceful bool) error {
	p.shutdown = true
	if !p.deregistered {
		err := p.deregisterService()
		if err != nil {
			plog.Error("deregisterMember", log.Error(err))
			return err
		}
		p.deregistered = true
	}
	if p.cancelWatch != nil {
		p.cancelWatch()
		p.cancelWatch = nil
	}
	return nil
}

func (p *Provider) UpdateClusterState(state cluster.ClusterState) error {
	if p.shutdown {
		return fmt.Errorf("shutdowned")
	}
	data, err := json.Marshal(state)
	if err != nil {
		return err
	}
	value := base64.StdEncoding.EncodeToString(data)
	p.self.SetMeta("state", value)
	return p.registerService()
}

func (p *Provider) keepAliveForever(ctx context.Context) error {
	if p.self == nil {
		return fmt.Errorf("keepalive must be after initialize.")
	}

	data, err := p.self.Serialize()
	if err != nil {
		return err
	}
	fullKey := p.getEtcdKey()
	if err != nil {
		return err
	}

	leaseId := p.getLeaseID()
	if leaseId <= 0 {
		_leaseId, err := p.newLeaseID()
		if err != nil {
			return err
		}
		leaseId = _leaseId
	}

	if leaseId <= 0 {
		return fmt.Errorf("grant lease failed. leaseId=%d", leaseId)
	}
	_, err = p.client.Put(context.TODO(), fullKey, string(data), clientv3.WithLease(leaseId))
	if err != nil {
		return err
	}
	kaRespCh, err := p.client.KeepAlive(context.TODO(), leaseId)
	if err != nil {
		return err
	}

	for resp := range kaRespCh {
		if resp == nil {
			return fmt.Errorf("keep alive failed. resp=%s", resp.String())
		}
		// plog.Infof("keep alive %s ttl=%d", p.getID(), resp.TTL)
		if p.shutdown {
			return nil
		}
	}
	return nil
}

func (p *Provider) startKeepAlive(ctx context.Context) {
	go func() {
		for !p.shutdown {
			if err := ctx.Err(); err != nil {
				plog.Info("Keepalive was stopped.", log.Error(err))
				return
			}

			if err := p.keepAliveForever(ctx); err != nil {
				plog.Info("Failure refreshing service TTL. ReTrying...", log.Duration("after", p.retryInterval))
				time.Sleep(p.retryInterval)
			}
		}
	}()
}

func (p *Provider) getID() string {
	return p.self.ID
}

func (p *Provider) getEtcdKey() string {
	return p.buildKey(p.clusterName, p.getID())
}

func (p *Provider) registerService() error {
	data, err := p.self.Serialize()
	if err != nil {
		return err
	}
	fullKey := p.getEtcdKey()
	if err != nil {
		return err
	}
	leaseId := p.getLeaseID()
	if leaseId <= 0 {
		_leaseId, err := p.newLeaseID()
		if err != nil {
			return err
		}
		leaseId = _leaseId
	}
	_, err = p.client.Put(context.TODO(), fullKey, string(data), clientv3.WithLease(leaseId))
	if err != nil {
		return err
	}
	return nil
}

func (p *Provider) deregisterService() error {
	fullKey := p.getEtcdKey()
	_, err := p.client.Delete(context.TODO(), fullKey)
	return err
}

func (p *Provider) handleWatchResponse(resp clientv3.WatchResponse) map[string]*Node {
	changes := map[string]*Node{}
	for _, ev := range resp.Events {
		key := string(ev.Kv.Key)
		nodeId, err := getNodeID(key, "/")
		if err != nil {
			plog.Error("Invalid member.", log.String("key", key))
			continue
		}

		switch ev.Type {
		case clientv3.EventTypePut:
			node, err := NewNodeFromBytes(ev.Kv.Value)
			if err != nil {
				plog.Error("Invalid member.", log.String("key", key))
				continue
			}
			if p.self.Equal(node) {
				plog.Debug("Skip self.", log.String("key", key))
				continue
			}
			if _, ok := p.members[nodeId]; ok {
				plog.Debug("Update member.", log.String("key", key))
			} else {
				plog.Debug("New member.", log.String("key", key))
			}
			changes[nodeId] = node
		case clientv3.EventTypeDelete:
			node, ok := p.members[nodeId]
			if !ok {
				continue
			}
			plog.Debug("Delete member.", log.String("key", key))
			cloned := (*node)
			cloned.SetAlive(false)
			changes[nodeId] = &cloned
		default:
			plog.Error("Invalid etcd event.type.", log.String("key", key),
				log.String("type", ev.Type.String()))
		}
	}
	return changes
}

func (p *Provider) keepWatching(ctx context.Context) error {
	clusterKey := p.buildKey(p.clusterName)
	stream := p.client.Watch(ctx, clusterKey, clientv3.WithPrefix())
	return p._keepWatching(stream)
}

func (p *Provider) _keepWatching(stream clientv3.WatchChan) error {
	for resp := range stream {
		if err := resp.Err(); err != nil {
			plog.Error("Failure watching service.")
			return err
		}
		if len(resp.Events) <= 0 {
			plog.Error("Empty etcd.events.", log.Int("events", len(resp.Events)))
			continue
		}
		nodesChanges := p.handleWatchResponse(resp)
		p.updateNodesWithChanges(nodesChanges)
		p.publishClusterTopologyEvent()
	}
	return nil
}

func (p *Provider) startWatching() {
	ctx := context.TODO()
	ctx, cancel := context.WithCancel(ctx)
	p.cancelWatch = cancel
	go func() {
		for !p.shutdown {
			if err := p.keepWatching(ctx); err != nil {
				plog.Error("Failed to keepWatching.", log.Error(err))
				p.clusterError = err
			}
		}
	}()
}

// GetHealthStatus returns an error if the cluster health status has problems
func (p *Provider) GetHealthStatus() error {
	return p.clusterError
}

func newContext(timeout time.Duration) (context.Context, context.CancelFunc) {
	return context.WithTimeout(context.TODO(), timeout)
}

func (p *Provider) buildKey(names ...string) string {
	return strings.Join(append([]string{p.baseKey}, names...), "/")
}

func (p *Provider) fetchNodes() ([]*Node, error) {
	key := p.buildKey(p.clusterName)
	resp, err := p.client.Get(context.TODO(), key, clientv3.WithPrefix())
	if err != nil {
		return nil, err
	}
	nodes := []*Node{}
	for _, v := range resp.Kvs {
		n := Node{}
		if err := n.Deserialize(v.Value); err != nil {
			return nil, err
		}
		nodes = append(nodes, &n)
	}
	p.revision = uint64(resp.Header.Revision)
	plog.Debug("fetched members", log.Uint64("revision", p.revision))
	return nodes, nil
}

func (p *Provider) updateNodes(members []*Node) {
	for _, n := range members {
		p.members[n.ID] = n
	}
}

func (p *Provider) updateNodesWithSelf(members []*Node) {
	p.updateNodes(members)
	p.members[p.self.ID] = p.self
}

func (p *Provider) updateNodesWithChanges(changes map[string]*Node) {
	for memberId, member := range changes {
		p.members[memberId] = member
		if !member.IsAlive() {
			delete(p.members, memberId)
		}
	}
}

func (p *Provider) createClusterTopologyEvent() cluster.TopologyEvent {
	res := make(cluster.TopologyEvent, len(p.members))
	i := 0
	for _, m := range p.members {
		res[i] = m.MemberStatus()
		i++
	}
	return res
}

func (p *Provider) publishClusterTopologyEvent() {
	res := p.createClusterTopologyEvent()
	plog.Info("Update cluster.", log.Int("members", len(res)))
	// for _, m := range res {
	// 	plog.Info("\t", log.Object("member", m))
	// }
	p.cluster.MemberList.UpdateClusterTopology(res, p.revision)
	// p.cluster.ActorSystem.EventStream.Publish(res)
}

func (p *Provider) getLeaseID() clientv3.LeaseID {
	val := (int64)(p.leaseID)
	return (clientv3.LeaseID)(atomic.LoadInt64(&val))
}

func (p *Provider) setLeaseID(leaseID clientv3.LeaseID) {
	val := (int64)(p.leaseID)
	atomic.StoreInt64(&val, (int64)(leaseID))
}

func (p *Provider) newLeaseID() (clientv3.LeaseID, error) {
	lease := clientv3.NewLease(p.client)
	ttlSecs := int64(p.keepAliveTTL / time.Second)
	resp, err := lease.Grant(context.TODO(), ttlSecs)
	if err != nil {
		return 0, err
	}
	return resp.ID, nil
}

func splitHostPort(addr string) (host string, port int, err error) {
	if h, p, e := net.SplitHostPort(addr); e != nil {
		if addr != "nonhost" {
			err = e
		}
		host = "nonhost"
		port = -1
	} else {
		host = h
		port, err = strconv.Atoi(p)
	}
	return
}
<commit_msg>fix cluster provider/etcd<commit_after>package etcd

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/AsynkronIT/protoactor-go/cluster"
	"github.com/AsynkronIT/protoactor-go/log"
	"github.com/coreos/etcd/clientv3"
)

var (
	plog = log.New(log.InfoLevel, "[CLUSTER/ETCD]")
)

type Provider struct {
	cluster       *cluster.Cluster
	baseKey       string
	clusterName   string
	deregistered  bool
	shutdown      bool
	self          *Node
	members       map[string]*Node // all, contains self.
	clusterError  error
	client        *clientv3.Client
	leaseID       clientv3.LeaseID
	cancelWatch   func()
	cancelWatchCh chan bool
	keepAliveTTL  time.Duration
	retryInterval time.Duration
	revision      uint64
	// deregisterCritical time.Duration
}

func New() (*Provider, error) {
	return NewWithConfig("/protoactor", clientv3.Config{
		Endpoints:   []string{"127.0.0.1:2379"},
		DialTimeout: time.Second * 5,
	})
}

func NewWithConfig(baseKey string, cfg clientv3.Config) (*Provider, error) {
	client, err := clientv3.New(cfg)
	if err != nil {
		return nil, err
	}
	p := &Provider{
		client:        client,
		keepAliveTTL:  3 * time.Second,
		retryInterval: 1 * time.Second,
		baseKey:       baseKey,
		members:       map[string]*Node{},
		cancelWatchCh: make(chan bool),
	}
	return p, nil
}

func (p *Provider) init(c *cluster.Cluster) error {
	p.cluster = c
	addr := p.cluster.ActorSystem.Address()
	host, port, err := splitHostPort(addr)
	if err != nil {
		return err
	}

	p.cluster = c
	p.clusterName = p.cluster.Config.Name
	knownKinds := c.GetClusterKinds()
	nodeName := fmt.Sprintf("%v@%v:%v", p.clusterName, host, port)
	p.self = NewNode(nodeName, host, port, knownKinds)
	p.self.SetMeta("id", p.getID())
	return nil
}

func (p *Provider) StartMember(c *cluster.Cluster) error {
	if err := p.init(c); err != nil {
		return err
	}

	// fetch memberlist
	nodes, err := p.fetchNodes()
	if err != nil {
		return err
	}
	// initialize members
	p.updateNodesWithSelf(nodes)
	p.publishClusterTopologyEvent()
	p.startWatching()

	// register self
	if err := p.registerService(); err != nil {
		return err
	}
	ctx := context.TODO()
	p.startKeepAlive(ctx)
	return nil
}

func (p *Provider) StartClient(c *cluster.Cluster) error {
	if err := p.init(c); err != nil {
		return err
	}
	nodes, err := p.fetchNodes()
	if err != nil {
		return err
	}
	// initialize members
	p.updateNodes(nodes)
	p.publishClusterTopologyEvent()
	p.startWatching()
	return nil
}

func (p *Provider) Shutdown(graceful bool) error {
	p.shutdown = true
	if !p.deregistered {
		err := p.deregisterService()
		if err != nil {
			plog.Error("deregisterMember", log.Error(err))
			return err
		}
		p.deregistered = true
	}
	if p.cancelWatch != nil {
		p.cancelWatch()
		p.cancelWatch = nil
	}
	return nil
}

func (p *Provider) UpdateClusterState(state cluster.ClusterState) error {
	if p.shutdown {
		return fmt.Errorf("shutdowned")
	}
	data, err := json.Marshal(state)
	if err != nil {
		return err
	}
	value := base64.StdEncoding.EncodeToString(data)
	p.self.SetMeta("state", value)
	return p.registerService()
}

func (p *Provider) keepAliveForever(ctx context.Context) error {
	if p.self == nil {
		return fmt.Errorf("keepalive must be after initialize.")
	}

	data, err := p.self.Serialize()
	if err != nil {
		return err
	}
	fullKey := p.getEtcdKey()
	if err != nil {
		return err
	}

	leaseId := p.getLeaseID()
	if leaseId <= 0 {
		_leaseId, err := p.newLeaseID()
		if err != nil {
			return err
		}
		leaseId = _leaseId
	}

	if leaseId <= 0 {
		return fmt.Errorf("grant lease failed. leaseId=%d", leaseId)
	}
	_, err = p.client.Put(context.TODO(), fullKey, string(data), clientv3.WithLease(leaseId))
	if err != nil {
		return err
	}
	kaRespCh, err := p.client.KeepAlive(context.TODO(), leaseId)
	if err != nil {
		return err
	}

	for resp := range kaRespCh {
		if resp == nil {
			return fmt.Errorf("keep alive failed. resp=%s", resp.String())
		}
		// plog.Infof("keep alive %s ttl=%d", p.getID(), resp.TTL)
		if p.shutdown {
			return nil
		}
	}
	return nil
}

func (p *Provider) startKeepAlive(ctx context.Context) {
	go func() {
		for !p.shutdown {
			if err := ctx.Err(); err != nil {
				plog.Info("Keepalive was stopped.", log.Error(err))
				return
			}

			if err := p.keepAliveForever(ctx); err != nil {
				plog.Info("Failure refreshing service TTL. ReTrying...", log.Duration("after", p.retryInterval))
				time.Sleep(p.retryInterval)
			}
		}
	}()
}

func (p *Provider) getID() string {
	return p.self.ID
}

func (p *Provider) getEtcdKey() string {
	return p.buildKey(p.clusterName, p.getID())
}

func (p *Provider) registerService() error {
	data, err := p.self.Serialize()
	if err != nil {
		return err
	}
	fullKey := p.getEtcdKey()
	if err != nil {
		return err
	}
	leaseId := p.getLeaseID()
	if leaseId <= 0 {
		_leaseId, err := p.newLeaseID()
		if err != nil {
			return err
		}
		leaseId = _leaseId
	}
	_, err = p.client.Put(context.TODO(), fullKey, string(data), clientv3.WithLease(leaseId))
	if err != nil {
		return err
	}
	return nil
}

func (p *Provider) deregisterService() error {
	fullKey := p.getEtcdKey()
	_, err := p.client.Delete(context.TODO(), fullKey)
	return err
}

func (p *Provider) handleWatchResponse(resp clientv3.WatchResponse) map[string]*Node {
	changes := map[string]*Node{}
	for _, ev := range resp.Events {
		key := string(ev.Kv.Key)
		nodeId, err := getNodeID(key, "/")
		if err != nil {
			plog.Error("Invalid member.", log.String("key", key))
			continue
		}

		switch ev.Type {
		case clientv3.EventTypePut:
			node, err := NewNodeFromBytes(ev.Kv.Value)
			if err != nil {
				plog.Error("Invalid member.", log.String("key", key))
				continue
			}
			if p.self.Equal(node) {
				plog.Debug("Skip self.", log.String("key", key))
				continue
			}
			if _, ok := p.members[nodeId]; ok {
				plog.Debug("Update member.", log.String("key", key))
			} else {
				plog.Debug("New member.", log.String("key", key))
			}
			changes[nodeId] = node
		case clientv3.EventTypeDelete:
			node, ok := p.members[nodeId]
			if !ok {
				continue
			}
			plog.Debug("Delete member.", log.String("key", key))
			cloned := (*node)
			cloned.SetAlive(false)
			changes[nodeId] = &cloned
		default:
			plog.Error("Invalid etcd event.type.", log.String("key", key),
				log.String("type", ev.Type.String()))
		}
	}
	p.revision = uint64(resp.Header.GetRevision())
	return changes
}

func (p *Provider) keepWatching(ctx context.Context) error {
	clusterKey := p.buildKey(p.clusterName)
	stream := p.client.Watch(ctx, clusterKey, clientv3.WithPrefix())
	return p._keepWatching(stream)
}

func (p *Provider) _keepWatching(stream clientv3.WatchChan) error {
	for resp := range stream {
		if err := resp.Err(); err != nil {
			plog.Error("Failure watching service.")
			return err
		}
		if len(resp.Events) <= 0 {
			plog.Error("Empty etcd.events.", log.Int("events", len(resp.Events)))
			continue
		}
		nodesChanges := p.handleWatchResponse(resp)
		p.updateNodesWithChanges(nodesChanges)
		p.publishClusterTopologyEvent()
	}
	return nil
}

func (p *Provider) startWatching() {
	ctx := context.TODO()
	ctx, cancel := context.WithCancel(ctx)
	p.cancelWatch = cancel
	go func() {
		for !p.shutdown {
			if err := p.keepWatching(ctx); err != nil {
				plog.Error("Failed to keepWatching.", log.Error(err))
				p.clusterError = err
			}
		}
	}()
}

// GetHealthStatus returns an error if the cluster health status has problems
func (p *Provider) GetHealthStatus() error {
	return p.clusterError
}

func newContext(timeout time.Duration) (context.Context, context.CancelFunc) {
	return context.WithTimeout(context.TODO(), timeout)
}

func (p *Provider) buildKey(names ...string) string {
	return strings.Join(append([]string{p.baseKey}, names...), "/")
}

func (p *Provider) fetchNodes() ([]*Node, error) {
	key := p.buildKey(p.clusterName)
	resp, err := p.client.Get(context.TODO(), key, clientv3.WithPrefix())
	if err != nil {
		return nil, err
	}
	nodes := []*Node{}
	for _, v := range resp.Kvs {
		n := Node{}
		if err := n.Deserialize(v.Value); err != nil {
			return nil, err
		}
		nodes = append(nodes, &n)
	}
	p.revision = uint64(resp.Header.GetRevision())
	// plog.Debug("fetch nodes",
	// 	log.Uint64("raft term", resp.Header.GetRaftTerm()),
	// 	log.Int64("revision", resp.Header.GetRevision()))
	return nodes, nil
}

func (p *Provider) updateNodes(members []*Node) {
	for _, n := range members {
		p.members[n.ID] = n
	}
}

func (p *Provider) updateNodesWithSelf(members []*Node) {
	p.updateNodes(members)
	p.members[p.self.ID] = p.self
}

func (p *Provider) updateNodesWithChanges(changes map[string]*Node) {
	for memberId, member := range changes {
		p.members[memberId] = member
		if !member.IsAlive() {
			delete(p.members, memberId)
		}
	}
}

func (p *Provider) createClusterTopologyEvent() cluster.TopologyEvent {
	res := make(cluster.TopologyEvent, len(p.members))
	i := 0
	for _, m := range p.members {
		res[i] = m.MemberStatus()
		i++
	}
	return res
}

func (p *Provider) publishClusterTopologyEvent() {
	res := p.createClusterTopologyEvent()
	plog.Info("Update cluster.", log.Int("members", len(res)))
	// for _, m := range res {
	// 	plog.Info("\t", log.Object("member", m))
	// }
	p.cluster.MemberList.UpdateClusterTopology(res, p.revision)
	// p.cluster.ActorSystem.EventStream.Publish(res)
}

func (p *Provider) getLeaseID() clientv3.LeaseID {
	val := (int64)(p.leaseID)
	return (clientv3.LeaseID)(atomic.LoadInt64(&val))
}

func (p *Provider) setLeaseID(leaseID clientv3.LeaseID) {
	val := (int64)(p.leaseID)
	atomic.StoreInt64(&val, (int64)(leaseID))
}

func (p *Provider) newLeaseID() (clientv3.LeaseID, error) {
	lease := clientv3.NewLease(p.client)
	ttlSecs := int64(p.keepAliveTTL / time.Second)
	resp, err := lease.Grant(context.TODO(), ttlSecs)
	if err != nil {
		return 0, err
	}
	return resp.ID, nil
}

func splitHostPort(addr string) (host string, port int, err error) {
	if h, p, e := net.SplitHostPort(addr); e != nil {
		if addr != "nonhost" {
			err = e
		}
		host = "nonhost"
		port = -1
	} else {
		host = h
		port, err = strconv.Atoi(p)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {
	fmt.Println("Hello World!")
}
<commit_msg>Add CNCF copyright header.<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
)

func main() {
	fmt.Println("Hello World!")
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/drud/ddev/pkg/ddevapp"
	"github.com/drud/ddev/pkg/exec"
	"github.com/drud/ddev/pkg/fileutil"
	"github.com/drud/ddev/pkg/nodeps"
	"github.com/drud/ddev/pkg/testcommon"
	asrt "github.com/stretchr/testify/assert"
	"os"
	"path/filepath"
	"testing"
)

func TestComposerCmd(t *testing.T) {
	assert := asrt.New(t)

	oldDir, err := os.Getwd()
	assert.NoError(err)
	// nolint: errcheck
	defer os.Chdir(oldDir)

	tmpDir := testcommon.CreateTmpDir(t.Name())
	err = os.Chdir(tmpDir)
	assert.NoError(err)

	// Basic config
	args := []string{"config", "--project-type", "php"}
	_, err = exec.RunCommand(DdevBin, args)
	assert.NoError(err)

	// Test trivial command
	args = []string{"composer"}
	out, err := exec.RunCommand(DdevBin, args)
	assert.NoError(err)
	assert.Contains(out, "Available commands:")

	// Get an app just so we can do waits and check webcacheenabled etc.
	app, err := ddevapp.NewApp(tmpDir, true, "")
	assert.NoError(err)
	//nolint: errcheck
	defer app.Stop(true, false)

	// Test create-project
	// ddev composer create --prefer-dist --no-interaction --no-dev psr/log:1.1.0
	args = []string{"composer", "create", "--prefer-dist", "--no-interaction", "--no-dev", "psr/log:1.1.0"}
	out, err = exec.RunCommand(DdevBin, args)
	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
	assert.Contains(out, "Created project in ")
	ddevapp.WaitForSync(app, 2)
	assert.FileExists(filepath.Join(tmpDir, "Psr/Log/LogLevel.php"))

	// This particlar --no-install does not seem to work on Docker Toolbox
	// with NFS so skip there.
	if !(nodeps.IsDockerToolbox() && app.NFSMountEnabled) {
		err = app.StartAndWaitForSync(5)
		assert.NoError(err)
		// ddev composer create --prefer-dist --no-interaction --no-dev psr/log:1.1.0 --no-install
		args = []string{"composer", "create", "--prefer-dist", "--no-interaction", "--no-dev", "psr/log:1.1.0", "--no-install"}
		out, err = exec.RunCommand(DdevBin, args)
		assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
		assert.Contains(out, "Created project in ")
		ddevapp.WaitForSync(app, 2)
		assert.FileExists(filepath.Join(tmpDir, "Psr/Log/LogLevel.php"))
	}

	// Test a composer require, with passthrough args
	args = []string{"composer", "require", "sebastian/version", "--no-plugins", "--ansi"}
	out, err = exec.RunCommand(DdevBin, args)
	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
	assert.Contains(out, "Generating autoload files")
	ddevapp.WaitForSync(app, 2)
	assert.FileExists(filepath.Join(tmpDir, "vendor/sebastian/version/composer.json"))
	// Test a composer remove
	if nodeps.IsDockerToolbox() {
		// On docker toolbox, git objects are read-only, causing the composer remove to fail.
		_, err = exec.RunCommand(DdevBin, []string{"exec", "chmod", "-R", "u+w", "//var/www/html/"})
		assert.NoError(err)
	}
	args = []string{"composer", "remove", "sebastian/version"}
	out, err = exec.RunCommand(DdevBin, args)
	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
	assert.Contains(out, "Generating autoload files")
	ddevapp.WaitForSync(app, 2)
	assert.False(fileutil.FileExists(filepath.Join(tmpDir, "vendor/sebastian")))
}
<commit_msg>Skip composer --no-install on Windows NFS (tests only) (#2012)<commit_after>package cmd

import (
	"github.com/drud/ddev/pkg/ddevapp"
	"github.com/drud/ddev/pkg/exec"
	"github.com/drud/ddev/pkg/fileutil"
	"github.com/drud/ddev/pkg/nodeps"
	"github.com/drud/ddev/pkg/testcommon"
	asrt "github.com/stretchr/testify/assert"
	"os"
	"path/filepath"
	"runtime"
	"testing"
)

func TestComposerCmd(t *testing.T) {
	assert := asrt.New(t)

	oldDir, err := os.Getwd()
	assert.NoError(err)
	// nolint: errcheck
	defer os.Chdir(oldDir)

	tmpDir := testcommon.CreateTmpDir(t.Name())
	err = os.Chdir(tmpDir)
	assert.NoError(err)

	// Basic config
	args := []string{"config", "--project-type", "php"}
	_, err = exec.RunCommand(DdevBin, args)
	assert.NoError(err)

	// Test trivial command
	args = []string{"composer"}
	out, err := exec.RunCommand(DdevBin, args)
	assert.NoError(err)
	assert.Contains(out, "Available commands:")

	// Get an app just so we can do waits and check webcacheenabled etc.
	app, err := ddevapp.NewApp(tmpDir, true, "")
	assert.NoError(err)
	//nolint: errcheck
	defer app.Stop(true, false)

	// Test create-project
	// These two often fail on Windows with NFS
	// It appears to be something about composer itself?
	if !(runtime.GOOS == "windows" && app.NFSMountEnabled) {
		// ddev composer create --prefer-dist --no-interaction --no-dev psr/log:1.1.0
		args = []string{"composer", "create", "--prefer-dist", "--no-interaction", "--no-dev", "psr/log:1.1.0"}
		out, err = exec.RunCommand(DdevBin, args)
		assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
		assert.Contains(out, "Created project in ")
		ddevapp.WaitForSync(app, 2)
		assert.FileExists(filepath.Join(tmpDir, "Psr/Log/LogLevel.php"))

		err = app.StartAndWaitForSync(5)
		assert.NoError(err)
		// ddev composer create --prefer-dist--no-dev --no-install psr/log:1.1.0
		args = []string{"composer", "create", "--prefer-dist", "--no-dev", "--no-install", "psr/log:1.1.0"}
		out, err = exec.RunCommand(DdevBin, args)
		assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
		assert.Contains(out, "Created project in ")
		ddevapp.WaitForSync(app, 2)
		assert.FileExists(filepath.Join(tmpDir, "Psr/Log/LogLevel.php"))
	}

	// Test a composer require, with passthrough args
	args = []string{"composer", "require", "sebastian/version", "--no-plugins", "--ansi"}
	out, err = exec.RunCommand(DdevBin, args)
	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
	assert.Contains(out, "Generating autoload files")
	ddevapp.WaitForSync(app, 2)
	assert.FileExists(filepath.Join(tmpDir, "vendor/sebastian/version/composer.json"))
	// Test a composer remove
	if nodeps.IsDockerToolbox() {
		// On docker toolbox, git objects are read-only, causing the composer remove to fail.
		_, err = exec.RunCommand(DdevBin, []string{"exec", "chmod", "-R", "u+w", "//var/www/html/"})
		assert.NoError(err)
	}
	args = []string{"composer", "remove", "sebastian/version"}
	out, err = exec.RunCommand(DdevBin, args)
	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
	assert.Contains(out, "Generating autoload files")
	ddevapp.WaitForSync(app, 2)
	assert.False(fileutil.FileExists(filepath.Join(tmpDir, "vendor/sebastian")))
}
<|endoftext|>


<commit_before>// Copyright 2015 iquota Authors. All rights reserved.
// Use of this source code is governed by a BSD style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/gorilla/context"
	"github.com/spf13/viper"
	"github.com/ubccr/iquota"
)

func errorHandler(app *Application, w http.ResponseWriter, status int, err *iquota.IsiError) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	if err != nil {
		out, err := json.Marshal(err)
		if err != nil {
			logrus.Printf("Error encoding error message as json: %s", err)
			return
		}
		w.Write(out)
	}
}

func IndexHandler(app *Application) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user := context.Get(r, "user").(*User)
		if user == nil {
			logrus.Error("index handler: user not found in request context")
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		quotas := make([]*iquota.Quota, 0)

		for _, q := range app.defaultUserQuota {
			quotas = append(quotas, q)
		}

		for _, q := range app.defaultGroupQuota {
			quotas = append(quotas, q)
		}

		out, err := json.Marshal(quotas)
		if err != nil {
			logrus.Printf("Error encoding data as json: %s", err)
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.Write(out)
	})
}

func UserQuotaHandler(app *Application) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user := context.Get(r, "user").(*User)
		if user == nil {
			logrus.Error("user quota handler: user not found in request context")
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		qp := new(iquota.QuotaParams)
		app.decoder.Decode(qp, r.URL.Query())

		if len(qp.Path) == 0 {
			errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Path is required"})
			return
		}

		uid := user.Uid
		if len(qp.User) != 0 && qp.User != uid {
			if !user.IsAdmin() {
				errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Access denied"})
				return
			}

			uid = qp.User
		}

		var qres *iquota.QuotaResponse

		if viper.GetBool("enable_cache") {
			cqres, err := FetchUserQuotaCache(qp.Path, uid)
			if err == nil {
				qres = cqres
			}
		}

		if qres == nil {
			c := NewOnefsClient()
			res, err := c.FetchUserQuota(qp.Path, uid)
			if err != nil {
				logrus.WithFields(logrus.Fields{
					"err": err.Error(),
					"uid": uid,
				}).Error("Failed to fetch user quota")
				if ierr, ok := err.(*iquota.IsiError); ok {
					errorHandler(app, w, http.StatusBadRequest, ierr)
				} else {
					errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Fatal system error"})
				}

				return
			}

			qres = res

			if viper.GetBool("enable_cache") {
				SetUserQuotaCache(qp.Path, uid, qres)
			}
		}

		qr := &iquota.QuotaRestResponse{Quotas: qres.Quotas}
		qr.Default, _ = app.defaultUserQuota[qp.Path]

		out, err := json.Marshal(qr)
		if err != nil {
			logrus.Printf("Error encoding data as json: %s", err)
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.Write(out)
	})
}

func GroupQuotaHandler(app *Application) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user := context.Get(r, "user").(*User)
		if user == nil {
			logrus.Error("group quota handler: user not found in request context")
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		qp := new(iquota.QuotaParams)
		app.decoder.Decode(qp, r.URL.Query())

		if len(qp.Path) == 0 {
			errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Path is required"})
			return
		}

		groups := user.Groups
		if len(qp.Group) != 0 {
			if !user.IsAdmin() {
				errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Access denied"})
				return
			}

			groups = []string{qp.Group}
		}

		c := NewOnefsClient()
		gquotas := make([]*iquota.Quota, 0)

		for _, group := range groups {

			var qres *iquota.QuotaResponse

			if viper.GetBool("enable_cache") {
				cqres, err := FetchGroupQuotaCache(qp.Path, group)
				if err != nil {
					if ierr, ok := err.(*iquota.IsiError); ok {
						if ierr.Code == "AEC_NOT_FOUND" && len(qp.Group) == 0 {
							continue
						}
						logrus.WithFields(logrus.Fields{
							"err":   ierr.Error(),
							"group": group,
						}).Error("Failed to fetch group quota")
						errorHandler(app, w, http.StatusBadRequest, ierr)
						return
					}
				}

				qres = cqres
			}

			if qres == nil {
				res, err := c.FetchGroupQuota(qp.Path, group)
				if err != nil {
					if ierr, ok := err.(*iquota.IsiError); ok {
						if ierr.Code == "AEC_NOT_FOUND" && viper.GetBool("enable_cache") {
							SetGroupNegCache(qp.Path, group)
						}

						if ierr.Code == "AEC_NOT_FOUND" && len(qp.Group) == 0 {
							continue
						}
						logrus.WithFields(logrus.Fields{
							"err":   ierr.Error(),
							"group": group,
						}).Error("Failed to fetch group quota")
						errorHandler(app, w, http.StatusBadRequest, ierr)
					} else {
						logrus.WithFields(logrus.Fields{
							"err":   err.Error(),
							"group": group,
						}).Error("Failed to fetch group quota")
						errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Fatal system error"})
					}

					return
				}

				qres = res
				if viper.GetBool("enable_cache") {
					SetGroupQuotaCache(qp.Path, group, qres)
				}
			}

			gquotas = append(gquotas, qres.Quotas...)
		}

		qr := &iquota.QuotaRestResponse{Quotas: gquotas}
		qr.Default, _ = app.defaultGroupQuota[qp.Path]

		out, err := json.Marshal(qr)
		if err != nil {
			logrus.Printf("Error encoding data as json: %s", err)
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.Write(out)
	})
}
<commit_msg>Use onefs session when querying multiple groups<commit_after>// Copyright 2015 iquota Authors. All rights reserved.
// Use of this source code is governed by a BSD style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/gorilla/context"
	"github.com/spf13/viper"
	"github.com/ubccr/iquota"
)

func errorHandler(app *Application, w http.ResponseWriter, status int, err *iquota.IsiError) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	if err != nil {
		out, err := json.Marshal(err)
		if err != nil {
			logrus.Printf("Error encoding error message as json: %s", err)
			return
		}
		w.Write(out)
	}
}

func IndexHandler(app *Application) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user := context.Get(r, "user").(*User)
		if user == nil {
			logrus.Error("index handler: user not found in request context")
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		quotas := make([]*iquota.Quota, 0)

		for _, q := range app.defaultUserQuota {
			quotas = append(quotas, q)
		}

		for _, q := range app.defaultGroupQuota {
			quotas = append(quotas, q)
		}

		out, err := json.Marshal(quotas)
		if err != nil {
			logrus.Printf("Error encoding data as json: %s", err)
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.Write(out)
	})
}

func UserQuotaHandler(app *Application) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user := context.Get(r, "user").(*User)
		if user == nil {
			logrus.Error("user quota handler: user not found in request context")
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		qp := new(iquota.QuotaParams)
		app.decoder.Decode(qp, r.URL.Query())

		if len(qp.Path) == 0 {
			errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Path is required"})
			return
		}

		uid := user.Uid
		if len(qp.User) != 0 && qp.User != uid {
			if !user.IsAdmin() {
				errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Access denied"})
				return
			}

			uid = qp.User
		}

		var qres *iquota.QuotaResponse

		if viper.GetBool("enable_cache") {
			cqres, err := FetchUserQuotaCache(qp.Path, uid)
			if err == nil {
				qres = cqres
			}
		}

		if qres == nil {
			c := NewOnefsClient()
			res, err := c.FetchUserQuota(qp.Path, uid)
			if err != nil {
				logrus.WithFields(logrus.Fields{
					"err": err.Error(),
					"uid": uid,
				}).Error("Failed to fetch user quota")
				if ierr, ok := err.(*iquota.IsiError); ok {
					errorHandler(app, w, http.StatusBadRequest, ierr)
				} else {
					errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Fatal system error"})
				}

				return
			}

			qres = res

			if viper.GetBool("enable_cache") {
				SetUserQuotaCache(qp.Path, uid, qres)
			}
		}

		qr := &iquota.QuotaRestResponse{Quotas: qres.Quotas}
		qr.Default, _ = app.defaultUserQuota[qp.Path]

		out, err := json.Marshal(qr)
		if err != nil {
			logrus.Printf("Error encoding data as json: %s", err)
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.Write(out)
	})
}

func GroupQuotaHandler(app *Application) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		user := context.Get(r, "user").(*User)
		if user == nil {
			logrus.Error("group quota handler: user not found in request context")
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		qp := new(iquota.QuotaParams)
		app.decoder.Decode(qp, r.URL.Query())

		if len(qp.Path) == 0 {
			errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Path is required"})
			return
		}

		groups := user.Groups
		if len(qp.Group) != 0 {
			if !user.IsAdmin() {
				errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Access denied"})
				return
			}

			groups = []string{qp.Group}
		}

		c := NewOnefsClient()
		c.NewSession()
		gquotas := make([]*iquota.Quota, 0)

		for _, group := range groups {

			var qres *iquota.QuotaResponse

			if viper.GetBool("enable_cache") {
				cqres, err := FetchGroupQuotaCache(qp.Path, group)
				if err != nil {
					if ierr, ok := err.(*iquota.IsiError); ok {
						if ierr.Code == "AEC_NOT_FOUND" && len(qp.Group) == 0 {
							continue
						}
						logrus.WithFields(logrus.Fields{
							"err":   ierr.Error(),
							"group": group,
						}).Error("Failed to fetch group quota")
						errorHandler(app, w, http.StatusBadRequest, ierr)
						return
					}
				}

				qres = cqres
			}

			if qres == nil {
				res, err := c.FetchGroupQuota(qp.Path, group)
				if err != nil {
					if ierr, ok := err.(*iquota.IsiError); ok {
						if ierr.Code == "AEC_NOT_FOUND" && viper.GetBool("enable_cache") {
							SetGroupNegCache(qp.Path, group)
						}

						if ierr.Code == "AEC_NOT_FOUND" && len(qp.Group) == 0 {
							continue
						}
						logrus.WithFields(logrus.Fields{
							"err":   ierr.Error(),
							"group": group,
						}).Error("Failed to fetch group quota")
						errorHandler(app, w, http.StatusBadRequest, ierr)
					} else {
						logrus.WithFields(logrus.Fields{
							"err":   err.Error(),
							"group": group,
						}).Error("Failed to fetch group quota")
						errorHandler(app, w, http.StatusBadRequest, &iquota.IsiError{Code: "AEC_BAD_REQUEST", Message: "Fatal system error"})
					}

					return
				}

				qres = res
				if viper.GetBool("enable_cache") {
					SetGroupQuotaCache(qp.Path, group, qres)
				}
			}

			gquotas = append(gquotas, qres.Quotas...)
		}

		qr := &iquota.QuotaRestResponse{Quotas: gquotas}
		qr.Default, _ = app.defaultGroupQuota[qp.Path]

		out, err := json.Marshal(qr)
		if err != nil {
			logrus.Printf("Error encoding data as json: %s", err)
			errorHandler(app, w, http.StatusInternalServerError, nil)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.Write(out)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/mitchellh/go-homedir"
	"github.com/peterh/liner"
)

func configSetup(c *cli.Context) error {
	var (
		configFile, home, wd string
		err                  error
	)

	if home, err = homedir.Dir(); err != nil {
		log.Fatal(err)
	}
	if wd, err = os.Getwd(); err != nil {
		log.Fatal(err)
	}

	// try loading a configuration file
	if c.GlobalIsSet(`config`) {
		if path.IsAbs(c.GlobalString(`config`)) {
			configFile = c.GlobalString(`config`)
		} else {
			configFile = path.Join(wd, c.GlobalString(`config`))
		}
	} else {
		configFile = path.Join(home, `.soma`, `dbctl`, `somadbctl.conf`)
	}

	if err = Cfg.populateFromFile(configFile); err != nil {
		if c.GlobalIsSet(`config`) {
			// missing cli argument file is fatal error
			log.Fatal(err)
		}
		log.Print(fmt.Sprintf("No configuration file found: %s", configFile))
	}

	// finish setting up runtime configuration
	params := []string{"host", "port", "database", "user", "timeout", "tls"}

	for p := range params {
		// update configuration with cli argument overrides
		if c.GlobalIsSet(params[p]) {
			log.Printf("Setting cli value override for %s", params[p])
			switch params[p] {
			case "host":
				Cfg.Database.Host = c.GlobalString(params[p])
			case "port":
				Cfg.Database.Port = strconv.Itoa(c.GlobalInt(params[p]))
			case "database":
				Cfg.Database.Name = c.GlobalString(params[p])
			case "user":
				Cfg.Database.User = c.GlobalString(params[p])
			case "timeout":
				Cfg.Timeout = strconv.Itoa(c.GlobalInt(params[p]))
			case "tls":
				Cfg.TlsMode = c.GlobalString(params[p])
			}
			continue
		}
		// set default values for unset configuration parameters
		switch params[p] {
		case "host":
			if Cfg.Database.Host == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.Host = "localhost"
			}
		case "port":
			if Cfg.Database.Port == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.Port = strconv.Itoa(5432)
			}
		case "database":
			if Cfg.Database.Name == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.Name = "soma"
			}
		case "user":
			if Cfg.Database.User == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.User = "soma_dba"
			}
		case "timeout":
			if Cfg.Timeout == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Timeout = strconv.Itoa(3)
			}
		case "tls":
			if Cfg.TlsMode == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.TlsMode = "verify-full"
			}
		}
	}

	// prompt for password if the cli flag was set
	if c.GlobalBool(`password`) || Cfg.Database.Pass == "" {
		line := liner.NewLiner()
		defer line.Close()
		line.SetCtrlCAborts(true)

		Cfg.Database.Pass, err = line.PasswordPrompt(`Enter database password: `)
		if err == liner.ErrPromptAborted {
			os.Exit(0)
		} else if err != nil {
			log.Fatal(err)
		}
	}

	// abort if we have no connection password at this point
	if Cfg.Database.Pass == "" {
		log.Fatal(`Can not continue without database connection password`)
	}
	return nil
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>somadbctl: ignore configuration on -n|--no-execute<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/mitchellh/go-homedir"
	"github.com/peterh/liner"
)

func configSetup(c *cli.Context) error {
	if c.GlobalBool(`no-execute`) {
		return nil
	}

	var (
		configFile, home, wd string
		err                  error
	)

	if home, err = homedir.Dir(); err != nil {
		log.Fatal(err)
	}
	if wd, err = os.Getwd(); err != nil {
		log.Fatal(err)
	}

	// try loading a configuration file
	if c.GlobalIsSet(`config`) {
		if path.IsAbs(c.GlobalString(`config`)) {
			configFile = c.GlobalString(`config`)
		} else {
			configFile = path.Join(wd, c.GlobalString(`config`))
		}
	} else {
		configFile = path.Join(home, `.soma`, `dbctl`, `somadbctl.conf`)
	}

	if err = Cfg.populateFromFile(configFile); err != nil {
		if c.GlobalIsSet(`config`) {
			// missing cli argument file is fatal error
			log.Fatal(err)
		}
		log.Print(fmt.Sprintf("No configuration file found: %s", configFile))
	}

	// finish setting up runtime configuration
	params := []string{"host", "port", "database", "user", "timeout", "tls"}

	for p := range params {
		// update configuration with cli argument overrides
		if c.GlobalIsSet(params[p]) {
			log.Printf("Setting cli value override for %s", params[p])
			switch params[p] {
			case "host":
				Cfg.Database.Host = c.GlobalString(params[p])
			case "port":
				Cfg.Database.Port = strconv.Itoa(c.GlobalInt(params[p]))
			case "database":
				Cfg.Database.Name = c.GlobalString(params[p])
			case "user":
				Cfg.Database.User = c.GlobalString(params[p])
			case "timeout":
				Cfg.Timeout = strconv.Itoa(c.GlobalInt(params[p]))
			case "tls":
				Cfg.TlsMode = c.GlobalString(params[p])
			}
			continue
		}
		// set default values for unset configuration parameters
		switch params[p] {
		case "host":
			if Cfg.Database.Host == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.Host = "localhost"
			}
		case "port":
			if Cfg.Database.Port == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.Port = strconv.Itoa(5432)
			}
		case "database":
			if Cfg.Database.Name == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.Name = "soma"
			}
		case "user":
			if Cfg.Database.User == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Database.User = "soma_dba"
			}
		case "timeout":
			if Cfg.Timeout == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.Timeout = strconv.Itoa(3)
			}
		case "tls":
			if Cfg.TlsMode == "" {
				log.Printf("Setting default value for %s", params[p])
				Cfg.TlsMode = "verify-full"
			}
		}
	}

	// prompt for password if the cli flag was set
	if c.GlobalBool(`password`) || Cfg.Database.Pass == "" {
		line := liner.NewLiner()
		defer line.Close()
		line.SetCtrlCAborts(true)

		Cfg.Database.Pass, err = line.PasswordPrompt(`Enter database password: `)
		if err == liner.ErrPromptAborted {
			os.Exit(0)
		} else if err != nil {
			log.Fatal(err)
		}
	}

	// abort if we have no connection password at this point
	if Cfg.Database.Pass == "" {
		log.Fatal(`Can not continue without database connection password`)
	}
	return nil
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/blevesearch/bleve"
	"net/http"
	"strconv"
)

// Please to de-dupe in wof-airportcity-index

type WOFRecord struct {
	Id        int64
	Names     []string
	Latitude  float64
	Longitude float64
}

func search(index bleve.Index, q string) ([]*WOFRecord, error) {

	fields := make([]string, 0)
	fields = append(fields, "Latitude")
	fields = append(fields, "Longitude")
	fields = append(fields, "Names")

	query := bleve.NewQueryStringQuery(q)
	req := bleve.NewSearchRequest(query)
	req.Fields = fields

	rsp, err := index.Search(req)

	if err != nil {
		return nil, err
	}

	results := make([]*WOFRecord, 0)

	for _, r := range rsp.Hits {

		id, _ := strconv.ParseInt(r.ID, 10, 64)
		f := r.Fields

		lat := f["Latitude"].(float64)
		lon := f["Longitude"].(float64)
		names := make([]string, 0)

		/*
			for _, n := range f["Names"] {
			    fmt.Println(n)
			}
		*/

		record := &WOFRecord{Id: id, Latitude: lat, Longitude: lon, Names: names}
		results = append(results, record)
	}

	return results, nil
}

func main() {

	var host = flag.String("host", "localhost", "The hostname to listen for requests on")
	var port = flag.Int("port", 8080, "The port number to listen for requests on")
	var cors = flag.Bool("cors", false, "Enable CORS headers")
	var db = flag.String("db", "", "The path to your search database")

	flag.Parse()

	index, _ := bleve.Open(*db)

	handler := func(rsp http.ResponseWriter, req *http.Request) {

		query := req.URL.Query()

		q := query.Get("q")

		if q == "" {
			http.Error(rsp, "Missing query parameter", http.StatusInternalServerError)
		}

		results, err := search(index, q)

		if err != nil {
			http.Error(rsp, err.Error(), http.StatusInternalServerError)
		}

		js, err := json.Marshal(results)

		if err != nil {
			http.Error(rsp, err.Error(), http.StatusInternalServerError)
			return
		}

		if *cors {
			rsp.Header().Set("Access-Control-Allow-Origin", "*")
		}

		rsp.Header().Set("Content-Type", "application/json")
		rsp.Write(js)
	}

	endpoint := fmt.Sprintf("%s:%d", *host, *port)

	fmt.Println(endpoint)

	http.HandleFunc("/", handler)
	http.ListenAndServe(endpoint, nil)
}
<commit_msg>return names in API responses per issue #1<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/blevesearch/bleve"
	"net/http"
	"strconv"
)

// Please to de-dupe in wof-airportcity-index

type WOFRecord struct {
	Id        int64
	Names     []string
	Latitude  float64
	Longitude float64
}

func search(index bleve.Index, q string) ([]*WOFRecord, error) {

	fields := make([]string, 0)
	fields = append(fields, "Latitude")
	fields = append(fields, "Longitude")
	fields = append(fields, "Names")

	query := bleve.NewQueryStringQuery(q)
	req := bleve.NewSearchRequest(query)
	req.Fields = fields

	rsp, err := index.Search(req)

	if err != nil {
		return nil, err
	}

	results := make([]*WOFRecord, 0)

	for _, r := range rsp.Hits {

		id, _ := strconv.ParseInt(r.ID, 10, 64)
		f := r.Fields

		lat := f["Latitude"].(float64)
		lon := f["Longitude"].(float64)
		names := make([]string, 0)

		for x, n := range f["Names"].([]interface{}) {
			names = append(names, n.(string))
		}

		record := &WOFRecord{Id: id, Latitude: lat, Longitude: lon, Names: names}
		results = append(results, record)
	}

	return results, nil
}

func main() {

	var host = flag.String("host", "localhost", "The hostname to listen for requests on")
	var port = flag.Int("port", 8080, "The port number to listen for requests on")
	var cors = flag.Bool("cors", false, "Enable CORS headers")
	var db = flag.String("db", "", "The path to your search database")

	flag.Parse()

	index, _ := bleve.Open(*db)

	handler := func(rsp http.ResponseWriter, req *http.Request) {

		query := req.URL.Query()

		q := query.Get("q")

		if q == "" {
			http.Error(rsp, "Missing query parameter", http.StatusInternalServerError)
		}

		results, err := search(index, q)

		if err != nil {
			http.Error(rsp, err.Error(), http.StatusInternalServerError)
		}

		js, err := json.Marshal(results)

		if err != nil {
			http.Error(rsp, err.Error(), http.StatusInternalServerError)
			return
		}

		if *cors {
			rsp.Header().Set("Access-Control-Allow-Origin", "*")
		}

		rsp.Header().Set("Content-Type", "application/json")
		rsp.Write(js)
	}

	endpoint := fmt.Sprintf("%s:%d", *host, *port)

	fmt.Println(endpoint)

	http.HandleFunc("/", handler)
	http.ListenAndServe(endpoint, nil)
}
<|endoftext|>