<commit_before>package app_runner

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/lattice/ltc/logs/reserved_app_ids"
	"github.com/cloudfoundry-incubator/lattice/ltc/route_helpers"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

type MonitorMethod int

//go:generate counterfeiter -o fake_app_runner/fake_app_runner.go . AppRunner
type AppRunner interface {
	CreateApp(params CreateAppParams) error
	SubmitLrp(lrpJSON []byte) (string, error)
	ScaleApp(name string, instances int) error
	UpdateAppRoutes(name string, routes RouteOverrides) error
	RemoveApp(name string) error
}

type MonitorConfig struct {
	Method  MonitorMethod
	URI     string
	Port    uint16
	Timeout time.Duration
}

type RouteOverrides []RouteOverride

type RouteOverride struct {
	HostnamePrefix string
	Port           uint16
}

type AppEnvironmentParams struct {
	EnvironmentVariables map[string]string
	Privileged           bool
	Monitor              MonitorConfig
	Instances            int
	CPUWeight            uint
	MemoryMB             int
	DiskMB               int
	ExposedPorts         []uint16
	WorkingDir           string
	RouteOverrides       RouteOverrides
	NoRoutes             bool
}

type CreateAppParams struct {
	AppEnvironmentParams

	Name         string
	StartCommand string
	RootFS       string
	AppArgs      []string
	Timeout      time.Duration
	Setup        models.Action
}

const (
	NoMonitor MonitorMethod = iota
	PortMonitor
	URLMonitor

	AttemptedToCreateLatticeDebugErrorMessage = reserved_app_ids.LatticeDebugLogStreamAppId + " is a reserved app name. It is used internally to stream debug logs for lattice components."
)

const (
	healthcheckDownloadUrl string = "http://file_server.service.dc1.consul:8080/v1/static/healthcheck.tgz"
	lrpDomain              string = "lattice"
)

type appRunner struct {
	receptorClient receptor.Client
	systemDomain   string
}

func New(receptorClient receptor.Client, systemDomain string) AppRunner {
	return &appRunner{receptorClient, systemDomain}
}

func (appRunner *appRunner) CreateApp(params CreateAppParams) error {
	if params.Name == reserved_app_ids.LatticeDebugLogStreamAppId {
		return errors.New(AttemptedToCreateLatticeDebugErrorMessage)
	}
	if exists, err := appRunner.desiredLRPExists(params.Name); err != nil {
		return err
	} else if exists {
		return newExistingAppError(params.Name)
	}

	if err := appRunner.receptorClient.UpsertDomain(lrpDomain, 0); err != nil {
		return err
	}

	return appRunner.desireLrp(params)
}

func (appRunner *appRunner) SubmitLrp(lrpJSON []byte) (string, error) {
	desiredLRP := receptor.DesiredLRPCreateRequest{}

	if err := json.Unmarshal(lrpJSON, &desiredLRP); err != nil {
		return "", err
	}

	if desiredLRP.ProcessGuid == reserved_app_ids.LatticeDebugLogStreamAppId {
		return desiredLRP.ProcessGuid, errors.New(AttemptedToCreateLatticeDebugErrorMessage)
	}

	if exists, err := appRunner.desiredLRPExists(desiredLRP.ProcessGuid); err != nil {
		return desiredLRP.ProcessGuid, err
	} else if exists {
		return desiredLRP.ProcessGuid, newExistingAppError(desiredLRP.ProcessGuid)
	}

	if err := appRunner.receptorClient.UpsertDomain(lrpDomain, 0); err != nil {
		return desiredLRP.ProcessGuid, err
	}

	return desiredLRP.ProcessGuid, appRunner.receptorClient.CreateDesiredLRP(desiredLRP)
}

func (appRunner *appRunner) ScaleApp(name string, instances int) error {
	if exists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !exists {
		return newAppNotStartedError(name)
	}

	return appRunner.updateLrpInstances(name, instances)
}

func (appRunner *appRunner) UpdateAppRoutes(name string, routes RouteOverrides) error {
	if exists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !exists {
		return newAppNotStartedError(name)
	}

	return appRunner.updateLrpRoutes(name, routes)
}

func (appRunner *appRunner) RemoveApp(name string) error {
	if lrpExists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !lrpExists {
		return newAppNotStartedError(name)
	}

	return appRunner.receptorClient.DeleteDesiredLRP(name)
}

func (appRunner *appRunner) desiredLRPExists(name string) (exists bool, err error) {
	desiredLRPs, err := appRunner.receptorClient.DesiredLRPs()
	if err != nil {
		return false, err
	}

	for _, desiredLRP := range desiredLRPs {
		if desiredLRP.ProcessGuid == name {
			return true, nil
		}
	}

	return false, nil
}

func (appRunner *appRunner) desireLrp(params CreateAppParams) error {
	var primaryPort uint16
	if params.Monitor.Port != 0 {
		primaryPort = params.Monitor.Port
	} else {
		primaryPort = params.ExposedPorts[0]
	}

	envVars := buildEnvironmentVariables(params.EnvironmentVariables)
	envVars = append(envVars, receptor.EnvironmentVariable{Name: "PORT", Value: fmt.Sprintf("%d", primaryPort)})

	var appRoutes route_helpers.AppRoutes
	if params.NoRoutes {
		appRoutes = route_helpers.AppRoutes{}
	} else if len(params.RouteOverrides) > 0 {
		routeMap := make(map[uint16][]string)
		for _, override := range params.RouteOverrides {
			routeMap[override.Port] = append(routeMap[override.Port], fmt.Sprintf("%s.%s", override.HostnamePrefix, appRunner.systemDomain))
		}
		for port, hostnames := range routeMap {
			appRoutes = append(appRoutes, route_helpers.AppRoute{
				Hostnames: hostnames,
				Port:      port,
			})
		}
	} else {
		appRoutes = appRunner.buildDefaultRoutingInfo(params.Name, params.ExposedPorts, primaryPort)
	}

	req := receptor.DesiredLRPCreateRequest{
		ProcessGuid:          params.Name,
		Domain:               lrpDomain,
		RootFS:               params.RootFS,
		Instances:            params.Instances,
		Routes:               appRoutes.RoutingInfo(),
		CPUWeight:            params.CPUWeight,
		MemoryMB:             params.MemoryMB,
		DiskMB:               params.DiskMB,
		Privileged:           params.Privileged,
		Ports:                params.ExposedPorts,
		LogGuid:              params.Name,
		LogSource:            "APP",
		MetricsGuid:          params.Name,
		EnvironmentVariables: envVars,
		Setup:                params.Setup,
		Action: &models.RunAction{
			Path: params.StartCommand,
			Args: params.AppArgs,
			Dir:  params.WorkingDir,
		},
	}

	var healthCheckArgs []string
	if params.Monitor.Timeout != 0 {
		healthCheckArgs = append(healthCheckArgs, "-timeout", fmt.Sprint(params.Monitor.Timeout))
	}
	switch params.Monitor.Method {
	case PortMonitor:
		req.Monitor = &models.RunAction{
			Path:      "/tmp/healthcheck",
			Args:      append(healthCheckArgs, "-port", fmt.Sprint(params.Monitor.Port)),
			LogSource: "HEALTH",
		}
	case URLMonitor:
		req.Monitor = &models.RunAction{
			Path:      "/tmp/healthcheck",
			Args:      append(healthCheckArgs, "-port", fmt.Sprint(params.Monitor.Port), "-uri", params.Monitor.URI),
			LogSource: "HEALTH",
		}
	}

	return appRunner.receptorClient.CreateDesiredLRP(req)
}

func (appRunner *appRunner) updateLrpInstances(name string, instances int) error {
	err := appRunner.receptorClient.UpdateDesiredLRP(
		name,
		receptor.DesiredLRPUpdateRequest{
			Instances: &instances,
		},
	)

	return err
}

func (appRunner *appRunner) updateLrpRoutes(name string, routes RouteOverrides) error {
	appRoutes := route_helpers.AppRoutes{}

	routeMap := make(map[uint16][]string)
	for _, override := range routes {
		routeMap[override.Port] = append(routeMap[override.Port], fmt.Sprintf("%s.%s", override.HostnamePrefix, appRunner.systemDomain))
	}
	for port, hostnames := range routeMap {
		appRoutes = append(appRoutes, route_helpers.AppRoute{
			Hostnames: hostnames,
			Port:      port,
		})
	}

	err := appRunner.receptorClient.UpdateDesiredLRP(
		name,
		receptor.DesiredLRPUpdateRequest{
			Routes: appRoutes.RoutingInfo(),
		},
	)

	return err
}

func (appRunner *appRunner) buildDefaultRoutingInfo(appName string, exposedPorts []uint16, primaryPort uint16) route_helpers.AppRoutes {
	appRoutes := route_helpers.AppRoutes{}

	for _, port := range exposedPorts {
		hostnames := []string{}
		if port == primaryPort {
			hostnames = append(hostnames, fmt.Sprintf("%s.%s", appName, appRunner.systemDomain))
		}

		hostnames = append(hostnames, fmt.Sprintf("%s-%s.%s", appName, strconv.Itoa(int(port)), appRunner.systemDomain))
		appRoutes = append(appRoutes, route_helpers.AppRoute{
			Hostnames: hostnames,
			Port:      port,
		})
	}

	return appRoutes
}

func buildEnvironmentVariables(environmentVariables map[string]string) []receptor.EnvironmentVariable {
	appEnvVars := make([]receptor.EnvironmentVariable, 0, len(environmentVariables)+1)
	for name, value := range environmentVariables {
		appEnvVars = append(appEnvVars, receptor.EnvironmentVariable{Name: name, Value: value})
	}
	return appEnvVars
}
<commit_msg>Fixing crash if there are no ports at all.<commit_after>package app_runner

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/lattice/ltc/logs/reserved_app_ids"
	"github.com/cloudfoundry-incubator/lattice/ltc/route_helpers"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

type MonitorMethod int

//go:generate counterfeiter -o fake_app_runner/fake_app_runner.go . AppRunner
type AppRunner interface {
	CreateApp(params CreateAppParams) error
	SubmitLrp(lrpJSON []byte) (string, error)
	ScaleApp(name string, instances int) error
	UpdateAppRoutes(name string, routes RouteOverrides) error
	RemoveApp(name string) error
}

type MonitorConfig struct {
	Method  MonitorMethod
	URI     string
	Port    uint16
	Timeout time.Duration
}

type RouteOverrides []RouteOverride

type RouteOverride struct {
	HostnamePrefix string
	Port           uint16
}

type AppEnvironmentParams struct {
	EnvironmentVariables map[string]string
	Privileged           bool
	Monitor              MonitorConfig
	Instances            int
	CPUWeight            uint
	MemoryMB             int
	DiskMB               int
	ExposedPorts         []uint16
	WorkingDir           string
	RouteOverrides       RouteOverrides
	NoRoutes             bool
}

type CreateAppParams struct {
	AppEnvironmentParams

	Name         string
	StartCommand string
	RootFS       string
	AppArgs      []string
	Timeout      time.Duration
	Setup        models.Action
}

const (
	NoMonitor MonitorMethod = iota
	PortMonitor
	URLMonitor

	AttemptedToCreateLatticeDebugErrorMessage = reserved_app_ids.LatticeDebugLogStreamAppId + " is a reserved app name. It is used internally to stream debug logs for lattice components."
)

const (
	healthcheckDownloadUrl string = "http://file_server.service.dc1.consul:8080/v1/static/healthcheck.tgz"
	lrpDomain              string = "lattice"
)

type appRunner struct {
	receptorClient receptor.Client
	systemDomain   string
}

func New(receptorClient receptor.Client, systemDomain string) AppRunner {
	return &appRunner{receptorClient, systemDomain}
}

func (appRunner *appRunner) CreateApp(params CreateAppParams) error {
	if params.Name == reserved_app_ids.LatticeDebugLogStreamAppId {
		return errors.New(AttemptedToCreateLatticeDebugErrorMessage)
	}
	if exists, err := appRunner.desiredLRPExists(params.Name); err != nil {
		return err
	} else if exists {
		return newExistingAppError(params.Name)
	}

	if err := appRunner.receptorClient.UpsertDomain(lrpDomain, 0); err != nil {
		return err
	}

	return appRunner.desireLrp(params)
}

func (appRunner *appRunner) SubmitLrp(lrpJSON []byte) (string, error) {
	desiredLRP := receptor.DesiredLRPCreateRequest{}

	if err := json.Unmarshal(lrpJSON, &desiredLRP); err != nil {
		return "", err
	}

	if desiredLRP.ProcessGuid == reserved_app_ids.LatticeDebugLogStreamAppId {
		return desiredLRP.ProcessGuid, errors.New(AttemptedToCreateLatticeDebugErrorMessage)
	}

	if exists, err := appRunner.desiredLRPExists(desiredLRP.ProcessGuid); err != nil {
		return desiredLRP.ProcessGuid, err
	} else if exists {
		return desiredLRP.ProcessGuid, newExistingAppError(desiredLRP.ProcessGuid)
	}

	if err := appRunner.receptorClient.UpsertDomain(lrpDomain, 0); err != nil {
		return desiredLRP.ProcessGuid, err
	}

	return desiredLRP.ProcessGuid, appRunner.receptorClient.CreateDesiredLRP(desiredLRP)
}

func (appRunner *appRunner) ScaleApp(name string, instances int) error {
	if exists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !exists {
		return newAppNotStartedError(name)
	}

	return appRunner.updateLrpInstances(name, instances)
}

func (appRunner *appRunner) UpdateAppRoutes(name string, routes RouteOverrides) error {
	if exists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !exists {
		return newAppNotStartedError(name)
	}

	return appRunner.updateLrpRoutes(name, routes)
}

func (appRunner *appRunner) RemoveApp(name string) error {
	if lrpExists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !lrpExists {
		return newAppNotStartedError(name)
	}

	return appRunner.receptorClient.DeleteDesiredLRP(name)
}

func (appRunner *appRunner) desiredLRPExists(name string) (exists bool, err error) {
	desiredLRPs, err := appRunner.receptorClient.DesiredLRPs()
	if err != nil {
		return false, err
	}

	for _, desiredLRP := range desiredLRPs {
		if desiredLRP.ProcessGuid == name {
			return true, nil
		}
	}

	return false, nil
}

func (appRunner *appRunner) desireLrp(params CreateAppParams) error {
	primaryPort := uint16(0)
	if params.Monitor.Port != 0 {
		primaryPort = params.Monitor.Port
	} else if len(params.ExposedPorts) > 0 {
		primaryPort = params.ExposedPorts[0]
	}

	envVars := buildEnvironmentVariables(params.EnvironmentVariables)
	envVars = append(envVars, receptor.EnvironmentVariable{Name: "PORT", Value: fmt.Sprintf("%d", primaryPort)})

	var appRoutes route_helpers.AppRoutes
	if params.NoRoutes {
		appRoutes = route_helpers.AppRoutes{}
	} else if len(params.RouteOverrides) > 0 {
		routeMap := make(map[uint16][]string)
		for _, override := range params.RouteOverrides {
			routeMap[override.Port] = append(routeMap[override.Port], fmt.Sprintf("%s.%s", override.HostnamePrefix, appRunner.systemDomain))
		}
		for port, hostnames := range routeMap {
			appRoutes = append(appRoutes, route_helpers.AppRoute{
				Hostnames: hostnames,
				Port:      port,
			})
		}
	} else {
		appRoutes = appRunner.buildDefaultRoutingInfo(params.Name, params.ExposedPorts, primaryPort)
	}

	req := receptor.DesiredLRPCreateRequest{
		ProcessGuid:          params.Name,
		Domain:               lrpDomain,
		RootFS:               params.RootFS,
		Instances:            params.Instances,
		Routes:               appRoutes.RoutingInfo(),
		CPUWeight:            params.CPUWeight,
		MemoryMB:             params.MemoryMB,
		DiskMB:               params.DiskMB,
		Privileged:           params.Privileged,
		Ports:                params.ExposedPorts,
		LogGuid:              params.Name,
		LogSource:            "APP",
		MetricsGuid:          params.Name,
		EnvironmentVariables: envVars,
		Setup:                params.Setup,
		Action: &models.RunAction{
			Path: params.StartCommand,
			Args: params.AppArgs,
			Dir:  params.WorkingDir,
		},
	}

	var healthCheckArgs []string
	if params.Monitor.Timeout != 0 {
		healthCheckArgs = append(healthCheckArgs, "-timeout", fmt.Sprint(params.Monitor.Timeout))
	}
	switch params.Monitor.Method {
	case PortMonitor:
		req.Monitor = &models.RunAction{
			Path:      "/tmp/healthcheck",
			Args:      append(healthCheckArgs, "-port", fmt.Sprint(params.Monitor.Port)),
			LogSource: "HEALTH",
		}
	case URLMonitor:
		req.Monitor = &models.RunAction{
			Path:      "/tmp/healthcheck",
			Args:      append(healthCheckArgs, "-port", fmt.Sprint(params.Monitor.Port), "-uri", params.Monitor.URI),
			LogSource: "HEALTH",
		}
	}

	return appRunner.receptorClient.CreateDesiredLRP(req)
}

func (appRunner *appRunner) updateLrpInstances(name string, instances int) error {
	err := appRunner.receptorClient.UpdateDesiredLRP(
		name,
		receptor.DesiredLRPUpdateRequest{
			Instances: &instances,
		},
	)

	return err
}

func (appRunner *appRunner) updateLrpRoutes(name string, routes RouteOverrides) error {
	appRoutes := route_helpers.AppRoutes{}

	routeMap := make(map[uint16][]string)
	for _, override := range routes {
		routeMap[override.Port] = append(routeMap[override.Port], fmt.Sprintf("%s.%s", override.HostnamePrefix, appRunner.systemDomain))
	}
	for port, hostnames := range routeMap {
		appRoutes = append(appRoutes, route_helpers.AppRoute{
			Hostnames: hostnames,
			Port:      port,
		})
	}

	err := appRunner.receptorClient.UpdateDesiredLRP(
		name,
		receptor.DesiredLRPUpdateRequest{
			Routes: appRoutes.RoutingInfo(),
		},
	)

	return err
}

func (appRunner *appRunner) buildDefaultRoutingInfo(appName string, exposedPorts []uint16, primaryPort uint16) route_helpers.AppRoutes {
	appRoutes := route_helpers.AppRoutes{}

	for _, port := range exposedPorts {
		hostnames := []string{}
		if port == primaryPort {
			hostnames = append(hostnames, fmt.Sprintf("%s.%s", appName, appRunner.systemDomain))
		}

		hostnames = append(hostnames, fmt.Sprintf("%s-%s.%s", appName, strconv.Itoa(int(port)), appRunner.systemDomain))
		appRoutes = append(appRoutes, route_helpers.AppRoute{
			Hostnames: hostnames,
			Port:      port,
		})
	}

	return appRoutes
}

func buildEnvironmentVariables(environmentVariables map[string]string) []receptor.EnvironmentVariable {
	appEnvVars := make([]receptor.EnvironmentVariable, 0, len(environmentVariables)+1)
	for name, value := range environmentVariables {
		appEnvVars = append(appEnvVars, receptor.EnvironmentVariable{Name: name, Value: value})
	}
	return appEnvVars
}
<|endoftext|>


<commit_before>package resources

import (
	"database/sql/driver"
	"encoding/json"
	"errors"
	"testing"

	testdb "github.com/erikstmartin/go-testdb"
	"github.com/jinzhu/gorm"
	"github.com/pborman/uuid"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
)

func TestCreateVersionDoesNotStoreWhenKubeFails(t *testing.T) {
	version := &Version{Name: "foo"}
	mockClient := new(mockKubeClient)
	mockClient.On("CreateDeployment", mock.Anything).Return(errors.New("foo error"))

	db, _ := gorm.Open("testdb", "")
	result, err := CreateVersion(db, mockClient, version)
	assert.False(t, result)
	assert.Equal(t, "foo error", err.Error())

	mockClient.AssertExpectations(t)
}

func TestCanCreateAVersion(t *testing.T) {
	id := uuid.New()
	version := Version{
		Name: "foobar",
		Slug: "foobar",
		UUID: id,
	}

	mockClient := new(mockKubeClient)
	mockClient.On("CreateDeployment", mock.Anything).Return(nil)

	db, _ := gorm.Open("testdb", "")
	var query string
	var args []driver.Value

	testdb.SetExecWithArgsFunc(func(q string, a []driver.Value) (driver.Result, error) {
		query = q
		args = a
		return driver.ResultNoRows, nil
	})

	CreateVersion(db, mockClient, &version)
	expectQuery := "INSERT INTO \"versions\" (\"created_at\",\"updated_at\",\"uuid\",\"name\",\"slug\",\"image\",\"environment_id\",\"spec\") VALUES (?,?,?,?,?,?,?,?)"
	assert.Equal(t, expectQuery, query)
	assert.Equal(t, id, args[2])
	assert.Equal(t, "foobar", args[3])

	deployment := versionDeploymentDefinition(&version)
	expectSpec, err := json.Marshal(deployment)
	assert.Nil(t, err)
	assert.Equal(t, string(expectSpec), args[7])
}

func TestVersionDeploymentDefinition(t *testing.T) {
	app := Application{Slug: "my-app", UUID: "app-uuid123"}
	environment := Environment{Slug: "dev", UUID: "env-uuid123"}
	environment.Application = app
	version := &Version{
		Environment: environment,
		Name:        "version 1",
		Slug:        "version-1",
		Image:       "foo:latest",
		UUID:        "version-uuid123",
		Replicas:    3,
	}

	deployment := versionDeploymentDefinition(version)
	assert.Equal(t, "my-app-dev", deployment.Name)
	assert.Equal(t, "brizo", deployment.Namespace)
	assert.Equal(t, int32(version.Replicas), *deployment.Spec.Replicas)
	assert.Equal(t, "foo:latest", deployment.Spec.Template.Spec.Containers[0].Image)

	expectDeploymentLabels := map[string]string{"brizoManaged": "true", "appUUID": "app-uuid123", "envUUID": "env-uuid123"}
	assert.Equal(t, expectDeploymentLabels, deployment.Labels)

	expectSelector := map[string]string{"envUUID": "env-uuid123", "versionUUID": "version-uuid123"}
	assert.Equal(t, expectSelector, deployment.Spec.Selector.MatchLabels)

	expectTemplateLabels := map[string]string{"brizoManaged": "true", "appUUID": "app-uuid123", "envUUID": "env-uuid123", "versionUUID": "version-uuid123"}
	assert.Equal(t, expectTemplateLabels, deployment.Spec.Template.Labels)
}

// Mock kube client

type mockKubeClient struct {
	mock.Mock
}

func (m *mockKubeClient) Health() error {
	return nil
}

func (m *mockKubeClient) GetPods(name string, options v1.ListOptions) ([]v1.Pod, error) {
	return []v1.Pod{}, nil
}

func (m *mockKubeClient) CreateDeployment(deployment *v1beta1.Deployment) error {
	args := m.Called(deployment)
	return args.Error(0)
}

func (m *mockKubeClient) DeleteDeployment(deployment *v1beta1.Deployment) error {
	return nil
}

func (m *mockKubeClient) FindDeploymentByName(namespace, name string) (*v1beta1.Deployment, error) {
	return &v1beta1.Deployment{}, nil
}

func (m *mockKubeClient) GetServices(name string, options v1.ListOptions) ([]v1.Service, error) {
	return []v1.Service{}, nil
}

func (m *mockKubeClient) CreateService(service *v1.Service) error {
	args := m.Called(service)
	return args.Error(0)
}

func (m *mockKubeClient) UpdateService(service *v1.Service) error {
	args := m.Called(service)
	return args.Error(0)
}
<commit_msg>updated test<commit_after>package resources

import (
	"database/sql/driver"
	"encoding/json"
	"errors"
	"testing"

	testdb "github.com/erikstmartin/go-testdb"
	"github.com/jinzhu/gorm"
	"github.com/pborman/uuid"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
)

func TestCreateVersionDoesNotStoreWhenKubeFails(t *testing.T) {
	version := &Version{Name: "foo"}
	mockClient := new(mockKubeClient)
	mockClient.On("CreateDeployment", mock.Anything).Return(errors.New("foo error"))

	db, _ := gorm.Open("testdb", "")
	result, err := CreateVersion(db, mockClient, version)
	assert.False(t, result)
	assert.Equal(t, "foo error", err.Error())

	mockClient.AssertExpectations(t)
}

func TestCanCreateAVersion(t *testing.T) {
	id := uuid.New()
	version := Version{
		Name: "foobar",
		Slug: "foobar",
		UUID: id,
	}

	mockClient := new(mockKubeClient)
	mockClient.On("CreateDeployment", mock.Anything).Return(nil)

	db, _ := gorm.Open("testdb", "")
	var query string
	var args []driver.Value

	testdb.SetExecWithArgsFunc(func(q string, a []driver.Value) (driver.Result, error) {
		query = q
		args = a
		return driver.ResultNoRows, nil
	})

	CreateVersion(db, mockClient, &version)
	expectQuery := "INSERT INTO \"versions\" (\"created_at\",\"updated_at\",\"uuid\",\"name\",\"slug\",\"image\",\"environment_id\",\"spec\") VALUES (?,?,?,?,?,?,?,?)"
	assert.Equal(t, expectQuery, query)
	assert.Equal(t, id, args[2])
	assert.Equal(t, "foobar", args[3])

	deployment := versionDeploymentDefinition(&version)
	expectSpec, err := json.Marshal(deployment)
	assert.Nil(t, err)
	assert.Equal(t, string(expectSpec), args[7])
}

func TestVersionDeploymentDefinition(t *testing.T) {
	app := Application{Slug: "my-app", UUID: "app-uuid123"}
	environment := Environment{Slug: "dev", UUID: "env-uuid123"}
	environment.Application = app
	version := &Version{
		Environment: environment,
		Name:        "version 1",
		Slug:        "version-1",
		Image:       "foo:latest",
		UUID:        "version-uuid123",
		Replicas:    3,
	}

	deployment := versionDeploymentDefinition(version)
	assert.Equal(t, "my-app-dev", deployment.Name)
	assert.Equal(t, "brizo", deployment.Namespace)
	assert.Equal(t, int32(version.Replicas), *deployment.Spec.Replicas)
	assert.Equal(t, "foo:latest", deployment.Spec.Template.Spec.Containers[0].Image)

	expectDeploymentLabels := map[string]string{"brizoManaged": "true", "appUUID": "app-uuid123", "envUUID": "env-uuid123"}
	assert.Equal(t, expectDeploymentLabels, deployment.Labels)

	expectSelector := map[string]string{"envUUID": "env-uuid123", "versionUUID": "version-uuid123"}
	assert.Equal(t, expectSelector, deployment.Spec.Selector.MatchLabels)

	expectTemplateLabels := map[string]string{"brizoManaged": "true", "appUUID": "app-uuid123", "envUUID": "env-uuid123", "versionUUID": "version-uuid123"}
	assert.Equal(t, expectTemplateLabels, deployment.Spec.Template.Labels)
}

// Mock kube client

type mockKubeClient struct {
	mock.Mock
}

func (m *mockKubeClient) Health() error {
	return nil
}

func (m *mockKubeClient) GetPods(name string, options v1.ListOptions) ([]v1.Pod, error) {
	return []v1.Pod{}, nil
}

func (m *mockKubeClient) CreateDeployment(deployment *v1beta1.Deployment) error {
	args := m.Called(deployment)
	return args.Error(0)
}

func (m *mockKubeClient) DeleteDeployment(deployment *v1beta1.Deployment) error {
	return nil
}

func (m *mockKubeClient) FindDeploymentByName(namespace, name string) (*v1beta1.Deployment, error) {
	return &v1beta1.Deployment{}, nil
}

func (m *mockKubeClient) GetServices(name string, options v1.ListOptions) ([]v1.Service, error) {
	return []v1.Service{}, nil
}

func (m *mockKubeClient) CreateService(service *v1.Service) error {
	args := m.Called(service)
	return args.Error(0)
}

func (m *mockKubeClient) UpdateService(service *v1.Service) error {
	return nil
}
<|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
)

// MkfsOptions represents options for filesystem creation.
type MkfsOptions struct {
	Label string
}

func wipeDirectory(path string) error {
	// List all entries
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
	}

	// Individually wipe all entries
	for _, entry := range entries {
		entryPath := filepath.Join(path, entry.Name())
		err := os.RemoveAll(entryPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func forceUnmount(path string) (bool, error) {
	unmounted := false

	for {
		// Check if already unmounted
		if !shared.IsMountPoint(path) {
			return unmounted, nil
		}

		// Try a clean unmount first
		err := unix.Unmount(path, 0)
		if err != nil {
			// Fallback to lazy unmounting
			err = unix.Unmount(path, unix.MNT_DETACH)
			if err != nil {
				return false, err
			}
		}

		unmounted = true
	}
}

func mountReadOnly(srcPath string, dstPath string) (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(dstPath) {
		return false, nil
	}

	// Create a mount entry.
	err := tryMount(srcPath, dstPath, "none", unix.MS_BIND, "")
	if err != nil {
		return false, err
	}

	// Make it read-only.
	err = tryMount("", dstPath, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "")
	if err != nil {
		forceUnmount(dstPath)
		return false, err
	}

	return true, nil
}

func sameMount(srcPath string, dstPath string) bool {
	// Get the source vfs path information
	var srcFsStat unix.Statfs_t
	err := unix.Statfs(srcPath, &srcFsStat)
	if err != nil {
		return false
	}

	// Get the destination vfs path information
	var dstFsStat unix.Statfs_t
	err = unix.Statfs(dstPath, &dstFsStat)
	if err != nil {
		return false
	}

	// Compare statfs
	if srcFsStat.Type != dstFsStat.Type || srcFsStat.Fsid != dstFsStat.Fsid {
		return false
	}

	// Get the source path information
	var srcStat unix.Stat_t
	err = unix.Stat(srcPath, &srcStat)
	if err != nil {
		return false
	}

	// Get the destination path information
	var dstStat unix.Stat_t
	err = unix.Stat(dstPath, &dstStat)
	if err != nil {
		return false
	}

	// Compare inode
	if srcStat.Ino != dstStat.Ino {
		return false
	}

	return true
}

func tryMount(src string, dst string, fs string, flags uintptr, options string) error {
	var err error

	// Attempt 20 mounts over 10s
	for i := 0; i < 20; i++ {
		err = unix.Mount(src, dst, fs, flags, options)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
	return shared.VarPath("storage-pools", poolName)
}

// GetVolumeMountPath returns the mount path for a specific volume based on its pool and type and
// whether it is a snapshot or not.
// For VolumeTypeImage the volName is the image fingerprint.
func GetVolumeMountPath(poolName string, volType VolumeType, volName string) string {
	if shared.IsSnapshot(volName) {
		return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), project.Prefix("default", volName))
	}

	return shared.VarPath("storage-pools", poolName, string(volType), project.Prefix("default", volName))
}

// GetVolumeSnapshotDir gets the snapshot mount directory for the parent volume.
func GetVolumeSnapshotDir(poolName string, volType VolumeType, volName string) string {
	parent, _, _ := shared.InstanceGetParentAndSnapshotName(volName)
	return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), project.Prefix("default", parent))
}

// GetSnapshotVolumeName returns the full volume name for a parent volume and snapshot name.
func GetSnapshotVolumeName(parentName, snapshotName string) string {
	return fmt.Sprintf("%s%s%s", parentName, shared.SnapshotDelimiter, snapshotName)
}

// createParentSnapshotDirIfMissing creates the parent directory for volume snapshots
func createParentSnapshotDirIfMissing(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it's missing, create it.
	if !shared.PathExists(snapshotsPath) {
		return os.Mkdir(snapshotsPath, 0700)
	}

	return nil
}

// deleteParentSnapshotDirIfEmpty removes the parent snapshot directory if it is empty.
// It accepts the pool name, volume type and parent volume name.
func deleteParentSnapshotDirIfEmpty(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it exists, try to delete it.
	if shared.PathExists(snapshotsPath) {
		isEmpty, err := shared.PathIsEmpty(snapshotsPath)
		if err != nil {
			return err
		}

		if isEmpty {
			err := os.Remove(snapshotsPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// createSparseFile creates a sparse empty file at specified location with specified size.
func createSparseFile(filePath string, sizeBytes int64) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("Failed to open %s: %s", filePath, err)
	}
	defer f.Close()

	err = f.Chmod(0600)
	if err != nil {
		return fmt.Errorf("Failed to chmod %s: %s", filePath, err)
	}

	err = f.Truncate(sizeBytes)
	if err != nil {
		return fmt.Errorf("Failed to create sparse file %s: %s", filePath, err)
	}

	return nil
}

// MakeFSType creates the provided filesystem.
func MakeFSType(path string, fsType string, options *MkfsOptions) (string, error) {
	var err error
	var msg string

	fsOptions := options
	if fsOptions == nil {
		fsOptions = &MkfsOptions{}
	}

	cmd := []string{fmt.Sprintf("mkfs.%s", fsType), path}
	if fsOptions.Label != "" {
		cmd = append(cmd, "-L", fsOptions.Label)
	}

	if fsType == "ext4" {
		cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0")
	}

	msg, err = shared.TryRunCommand(cmd[0], cmd[1:]...)
	if err != nil {
		return msg, err
	}

	return "", nil
}

// mountOption represents an individual mount option.
type mountOption struct {
	capture bool
	flag    uintptr
}

// mountOptions represents a list of possible mount options.
var mountOptions = map[string]mountOption{
	"async":         {false, unix.MS_SYNCHRONOUS},
	"atime":         {false, unix.MS_NOATIME},
	"bind":          {true, unix.MS_BIND},
	"defaults":      {true, 0},
	"dev":           {false, unix.MS_NODEV},
	"diratime":      {false, unix.MS_NODIRATIME},
	"dirsync":       {true, unix.MS_DIRSYNC},
	"exec":          {false, unix.MS_NOEXEC},
	"lazytime":      {true, unix.MS_LAZYTIME},
	"mand":          {true, unix.MS_MANDLOCK},
	"noatime":       {true, unix.MS_NOATIME},
	"nodev":         {true, unix.MS_NODEV},
	"nodiratime":    {true, unix.MS_NODIRATIME},
	"noexec":        {true, unix.MS_NOEXEC},
	"nomand":        {false, unix.MS_MANDLOCK},
	"norelatime":    {false, unix.MS_RELATIME},
	"nostrictatime": {false, unix.MS_STRICTATIME},
	"nosuid":        {true, unix.MS_NOSUID},
	"rbind":         {true, unix.MS_BIND | unix.MS_REC},
	"relatime":      {true, unix.MS_RELATIME},
	"remount":       {true, unix.MS_REMOUNT},
	"ro":            {true, unix.MS_RDONLY},
	"rw":            {false, unix.MS_RDONLY},
	"strictatime":   {true, unix.MS_STRICTATIME},
	"suid":          {false, unix.MS_NOSUID},
	"sync":          {true, unix.MS_SYNCHRONOUS},
}

// ResolveMountOptions resolves the provided mount options.
func ResolveMountOptions(options string) (uintptr, string) {
	mountFlags := uintptr(0)
	tmp := strings.SplitN(options, ",", -1)
	for i := 0; i < len(tmp); i++ {
		opt := tmp[i]
		do, ok := mountOptions[opt]
		if !ok {
			continue
		}

		if do.capture {
			mountFlags |= do.flag
		} else {
			mountFlags &= ^do.flag
		}

		copy(tmp[i:], tmp[i+1:])
		tmp[len(tmp)-1] = ""
		tmp = tmp[:len(tmp)-1]
		i--
	}

	return mountFlags, strings.Join(tmp, ",")
}
<commit_msg>lxd/storage/drivers: Re-order utils<commit_after>package drivers

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
)

func wipeDirectory(path string) error {
	// List all entries
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
	}

	// Individually wipe all entries
	for _, entry := range entries {
		entryPath := filepath.Join(path, entry.Name())
		err := os.RemoveAll(entryPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func forceUnmount(path string) (bool, error) {
	unmounted := false

	for {
		// Check if already unmounted
		if !shared.IsMountPoint(path) {
			return unmounted, nil
		}

		// Try a clean unmount first
		err := unix.Unmount(path, 0)
		if err != nil {
			// Fallback to lazy unmounting
			err = unix.Unmount(path, unix.MNT_DETACH)
			if err != nil {
				return false, err
			}
		}

		unmounted = true
	}
}

func mountReadOnly(srcPath string, dstPath string) (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(dstPath) {
		return false, nil
	}

	// Create a mount entry.
	err := tryMount(srcPath, dstPath, "none", unix.MS_BIND, "")
	if err != nil {
		return false, err
	}

	// Make it read-only.
	err = tryMount("", dstPath, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "")
	if err != nil {
		forceUnmount(dstPath)
		return false, err
	}

	return true, nil
}

func sameMount(srcPath string, dstPath string) bool {
	// Get the source vfs path information
	var srcFsStat unix.Statfs_t
	err := unix.Statfs(srcPath, &srcFsStat)
	if err != nil {
		return false
	}

	// Get the destination vfs path information
	var dstFsStat unix.Statfs_t
	err = unix.Statfs(dstPath, &dstFsStat)
	if err != nil {
		return false
	}

	// Compare statfs
	if srcFsStat.Type != dstFsStat.Type || srcFsStat.Fsid != dstFsStat.Fsid {
		return false
	}

	// Get the source path information
	var srcStat unix.Stat_t
	err = unix.Stat(srcPath, &srcStat)
	if err != nil {
		return false
	}

	// Get the destination path information
	var dstStat unix.Stat_t
	err = unix.Stat(dstPath, &dstStat)
	if err != nil {
		return false
	}

	// Compare inode
	if srcStat.Ino != dstStat.Ino {
		return false
	}

	return true
}

func tryMount(src string, dst string, fs string, flags uintptr, options string) error {
	var err error

	// Attempt 20 mounts over 10s
	for i := 0; i < 20; i++ {
		err = unix.Mount(src, dst, fs, flags, options)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
	return shared.VarPath("storage-pools", poolName)
}

// GetVolumeMountPath returns the mount path for a specific volume based on its pool and type and
// whether it is a snapshot or not.
// For VolumeTypeImage the volName is the image fingerprint.
func GetVolumeMountPath(poolName string, volType VolumeType, volName string) string {
	if shared.IsSnapshot(volName) {
		return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), project.Prefix("default", volName))
	}

	return shared.VarPath("storage-pools", poolName, string(volType), project.Prefix("default", volName))
}

// GetVolumeSnapshotDir gets the snapshot mount directory for the parent volume.
func GetVolumeSnapshotDir(poolName string, volType VolumeType, volName string) string {
	parent, _, _ := shared.InstanceGetParentAndSnapshotName(volName)
	return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), project.Prefix("default", parent))
}

// GetSnapshotVolumeName returns the full volume name for a parent volume and snapshot name.
func GetSnapshotVolumeName(parentName, snapshotName string) string {
	return fmt.Sprintf("%s%s%s", parentName, shared.SnapshotDelimiter, snapshotName)
}

// createParentSnapshotDirIfMissing creates the parent directory for volume snapshots
func createParentSnapshotDirIfMissing(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it's missing, create it.
	if !shared.PathExists(snapshotsPath) {
		return os.Mkdir(snapshotsPath, 0700)
	}

	return nil
}

// deleteParentSnapshotDirIfEmpty removes the parent snapshot directory if it is empty.
// It accepts the pool name, volume type and parent volume name.
func deleteParentSnapshotDirIfEmpty(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it exists, try to delete it.
	if shared.PathExists(snapshotsPath) {
		isEmpty, err := shared.PathIsEmpty(snapshotsPath)
		if err != nil {
			return err
		}

		if isEmpty {
			err := os.Remove(snapshotsPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// createSparseFile creates a sparse empty file at specified location with specified size.
func createSparseFile(filePath string, sizeBytes int64) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("Failed to open %s: %s", filePath, err)
	}
	defer f.Close()

	err = f.Chmod(0600)
	if err != nil {
		return fmt.Errorf("Failed to chmod %s: %s", filePath, err)
	}

	err = f.Truncate(sizeBytes)
	if err != nil {
		return fmt.Errorf("Failed to create sparse file %s: %s", filePath, err)
	}

	return nil
}

// MkfsOptions represents options for filesystem creation.
type MkfsOptions struct {
	Label string
}

// MakeFSType creates the provided filesystem.
func MakeFSType(path string, fsType string, options *MkfsOptions) (string, error) {
	var err error
	var msg string

	fsOptions := options
	if fsOptions == nil {
		fsOptions = &MkfsOptions{}
	}

	cmd := []string{fmt.Sprintf("mkfs.%s", fsType), path}
	if fsOptions.Label != "" {
		cmd = append(cmd, "-L", fsOptions.Label)
	}

	if fsType == "ext4" {
		cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0")
	}

	msg, err = shared.TryRunCommand(cmd[0], cmd[1:]...)
	if err != nil {
		return msg, err
	}

	return "", nil
}

// mountOption represents an individual mount option.
type mountOption struct {
	capture bool
	flag    uintptr
}

// mountOptions represents a list of possible mount options.
var mountOptions = map[string]mountOption{
	"async":         {false, unix.MS_SYNCHRONOUS},
	"atime":         {false, unix.MS_NOATIME},
	"bind":          {true, unix.MS_BIND},
	"defaults":      {true, 0},
	"dev":           {false, unix.MS_NODEV},
	"diratime":      {false, unix.MS_NODIRATIME},
	"dirsync":       {true, unix.MS_DIRSYNC},
	"exec":          {false, unix.MS_NOEXEC},
	"lazytime":      {true, unix.MS_LAZYTIME},
	"mand":          {true, unix.MS_MANDLOCK},
	"noatime":       {true, unix.MS_NOATIME},
	"nodev":         {true, unix.MS_NODEV},
	"nodiratime":    {true, unix.MS_NODIRATIME},
	"noexec":        {true, unix.MS_NOEXEC},
	"nomand":        {false, unix.MS_MANDLOCK},
	"norelatime":    {false, unix.MS_RELATIME},
	"nostrictatime": {false, unix.MS_STRICTATIME},
	"nosuid":        {true, unix.MS_NOSUID},
	"rbind":         {true, unix.MS_BIND | unix.MS_REC},
	"relatime":      {true, unix.MS_RELATIME},
	"remount":       {true, unix.MS_REMOUNT},
	"ro":            {true, unix.MS_RDONLY},
	"rw":            {false, unix.MS_RDONLY},
	"strictatime":   {true, unix.MS_STRICTATIME},
	"suid":          {false, unix.MS_NOSUID},
	"sync":          {true, unix.MS_SYNCHRONOUS},
}

// ResolveMountOptions resolves the provided mount options.
func ResolveMountOptions(options string) (uintptr, string) {
	mountFlags := uintptr(0)
	tmp := strings.SplitN(options, ",", -1)
	for i := 0; i < len(tmp); i++ {
		opt := tmp[i]
		do, ok := mountOptions[opt]
		if !ok {
			continue
		}

		if do.capture {
			mountFlags |= do.flag
		} else {
			mountFlags &= ^do.flag
		}

		copy(tmp[i:], tmp[i+1:])
		tmp[len(tmp)-1] = ""
		tmp = tmp[:len(tmp)-1]
		i--
	}

	return mountFlags, strings.Join(tmp, ",")
}
<|endoftext|>


<commit_before>// meek-server is the server transport plugin for the meek pluggable transport.
// It acts as an HTTP server, keeps track of session ids, and forwards received
// data to a local OR port.
//
// Sample usage in torrc:
// 	ServerTransportPlugin meek exec ./meek-server --port 8443 --cert cert.pem --key key.pem --log meek-server.log
// Plain HTTP usage:
// 	ServerTransportPlugin meek exec ./meek-server --port 8080 --disable-tls --log meek-server.log
//
// The server runs in HTTPS mode by default, and the --cert and --key options
// are required. Use the --disable-tls option to run with plain HTTP.
package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"sync"
	"syscall"
	"time"
)

import "git.torproject.org/pluggable-transports/goptlib.git"

const (
	ptMethodName = "meek"
	// Reject session ids shorter than this, as a weak defense against
	// client bugs that send an empty session id or something similarly
	// likely to collide.
	minSessionIdLength = 32
	// The largest request body we are willing to process, and the largest
	// chunk of data we'll send back in a response.
	maxPayloadLength = 0x10000
	// How long we try to read something back from the OR port before
	// returning the response.
	turnaroundTimeout = 10 * time.Millisecond
	// Passed as ReadTimeout and WriteTimeout when constructing the
	// http.Server.
	readWriteTimeout = 20 * time.Second
	// Cull unused session ids (with their corresponding OR port connection)
	// if we haven't seen any activity for this long.
	maxSessionStaleness = 120 * time.Second
)

var ptInfo pt.ServerInfo

// When a connection handler starts, +1 is written to this channel; when it
// ends, -1 is written.
var handlerChan = make(chan int)

func httpBadRequest(w http.ResponseWriter) {
	http.Error(w, "Bad request.\n", http.StatusBadRequest)
}

func httpInternalServerError(w http.ResponseWriter) {
	http.Error(w, "Internal server error.\n", http.StatusInternalServerError)
}

// Every session id maps to an existing OR port connection, which we keep open
// between received requests. The first time we see a new session id, we create
// a new OR port connection.
type Session struct {
	Or       *net.TCPConn
	LastSeen time.Time
}

// Mark a session as having been seen just now.
func (session *Session) Touch() {
	session.LastSeen = time.Now()
}

// Is this session old enough to be culled?
func (session *Session) IsExpired() bool {
	return time.Since(session.LastSeen) > maxSessionStaleness
}

// There is one state per HTTP listener. In the usual case there is just one
// listener, so there is just one global state. State also serves as the http
// Handler.
type State struct {
	sessionMap map[string]*Session
	lock       sync.Mutex
}

func NewState() *State {
	state := new(State)
	state.sessionMap = make(map[string]*Session)
	return state
}

func (state *State) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	handlerChan <- 1
	defer func() {
		handlerChan <- -1
	}()

	switch req.Method {
	case "GET":
		state.Get(w, req)
	case "POST":
		state.Post(w, req)
	default:
		httpBadRequest(w)
	}
}

// Handle a GET request. This doesn't have any purpose apart from diagnostics.
func (state *State) Get(w http.ResponseWriter, req *http.Request) {
	if path.Clean(req.URL.Path) != "/" {
		http.NotFound(w, req)
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("I’m just a happy little web server.\n"))
}

// Look up a session by id, or create a new one (with its OR port connection) if
// it doesn't already exist.
func (state *State) GetSession(sessionId string, req *http.Request) (*Session, error) {
	state.lock.Lock()
	defer state.lock.Unlock()

	session := state.sessionMap[sessionId]
	if session == nil {
		// log.Printf("unknown session id %q; creating new session", sessionId)

		or, err := pt.DialOr(&ptInfo, req.RemoteAddr, ptMethodName)
		if err != nil {
			return nil, err
		}
		session = &Session{Or: or}
		state.sessionMap[sessionId] = session
	}
	session.Touch()

	return session, nil
}

// Feed the body of req into the OR port, and write any data read from the OR
// port back to w.
func transact(session *Session, w http.ResponseWriter, req *http.Request) error {
	body := http.MaxBytesReader(w, req.Body, maxPayloadLength+1)
	_, err := io.Copy(session.Or, body)
	if err != nil {
		return fmt.Errorf("copying body to ORPort: %s", err)
	}

	buf := make([]byte, maxPayloadLength)
	session.Or.SetReadDeadline(time.Now().Add(turnaroundTimeout))
	n, err := session.Or.Read(buf)
	if err != nil {
		if e, ok := err.(net.Error); !ok || !e.Timeout() {
			httpInternalServerError(w)
			return fmt.Errorf("reading from ORPort: %s", err)
		}
	}
	// log.Printf("read %d bytes from ORPort: %q", n, buf[:n])
	// Set a Content-Type to prevent Go and the CDN from trying to guess.
	w.Header().Set("Content-Type", "application/octet-stream")
	n, err = w.Write(buf[:n])
	if err != nil {
		return fmt.Errorf("writing to response: %s", err)
	}
	// log.Printf("wrote %d bytes to response", n)
	return nil
}

// Handle a POST request. Look up the session id and then do a transaction.
func (state *State) Post(w http.ResponseWriter, req *http.Request) {
	sessionId := req.Header.Get("X-Session-Id")
	if len(sessionId) < minSessionIdLength {
		httpBadRequest(w)
		return
	}

	session, err := state.GetSession(sessionId, req)
	if err != nil {
		log.Print(err)
		httpInternalServerError(w)
		return
	}

	err = transact(session, w, req)
	if err != nil {
		log.Print(err)
		state.CloseSession(sessionId)
		return
	}
}

// Remove a session from the map and closes its corresponding OR port
// connection. Does nothing if the session id is not known.
func (state *State) CloseSession(sessionId string) {
	state.lock.Lock()
	defer state.lock.Unlock()
	// log.Printf("closing session %q", sessionId)
	session, ok := state.sessionMap[sessionId]
	if ok {
		session.Or.Close()
		delete(state.sessionMap, sessionId)
	}
}

// Loop forever, checking for expired sessions and removing them.
func (state *State) ExpireSessions() {
	for {
		time.Sleep(maxSessionStaleness / 2)
		state.lock.Lock()
		for sessionId, session := range state.sessionMap {
			if session.IsExpired() {
				// log.Printf("deleting expired session %q", sessionId)
				session.Or.Close()
				delete(state.sessionMap, sessionId)
			}
		}
		state.lock.Unlock()
	}
}

func listenTLS(network string, addr *net.TCPAddr, certFilename, keyFilename string) (net.Listener, error) {
	// This is cribbed from the source of net/http.Server.ListenAndServeTLS.
	// We have to separate the Listen and Serve parts because we need to
	// report the listening address before entering Serve (which is an
	// infinite loop).
	// https://groups.google.com/d/msg/Golang-nuts/3F1VRCCENp8/3hcayZiwYM8J
	config := &tls.Config{}
	config.NextProtos = []string{"http/1.1"}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFilename, keyFilename)
	if err != nil {
		return nil, err
	}

	conn, err := net.ListenTCP(network, addr)
	if err != nil {
		return nil, err
	}

	tlsListener := tls.NewListener(conn, config)

	return tlsListener, nil
}

func startListener(network string, addr *net.TCPAddr) (net.Listener, error) {
	ln, err := net.ListenTCP(network, addr)
	if err != nil {
		return nil, err
	}
	log.Printf("listening with plain HTTP on %s", ln.Addr())
	return startServer(ln)
}

func startListenerTLS(network string, addr *net.TCPAddr, certFilename, keyFilename string) (net.Listener, error) {
	ln, err := listenTLS(network, addr, certFilename, keyFilename)
	if err != nil {
		return nil, err
	}
	log.Printf("listening with HTTPS on %s", ln.Addr())
	return startServer(ln)
}

func startServer(ln net.Listener) (net.Listener, error) {
	state := NewState()
	go state.ExpireSessions()
	server := &http.Server{
		Handler:      state,
		ReadTimeout:  readWriteTimeout,
		WriteTimeout: readWriteTimeout,
	}
	go func() {
		defer ln.Close()
		err := server.Serve(ln)
		if err != nil {
			log.Printf("Error in Serve: %s", err)
		}
	}()
	return ln, nil
}

func main() {
	var disableTLS bool
	var certFilename, keyFilename string
	var logFilename string
	var port int

	flag.BoolVar(&disableTLS, "disable-tls", false, "don't use HTTPS")
	flag.StringVar(&certFilename, "cert", "", "TLS certificate file (required without --disable-tls)")
	flag.StringVar(&keyFilename, "key", "", "TLS private key file (required without --disable-tls)")
	flag.StringVar(&logFilename, "log", "", "name of log file")
	flag.IntVar(&port, "port", 0, "port to listen on")
	flag.Parse()

	if logFilename != "" {
		f, err := os.OpenFile(logFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			log.Fatalf("error opening log file: %s", err)
		}
		defer f.Close()
		log.SetOutput(f)
	}

	if disableTLS {
		if certFilename != "" || keyFilename != "" {
			log.Fatalf("The --cert and --key options are not allowed with --disable-tls.\n")
		}
	} else {
		if certFilename == "" || keyFilename == "" {
			log.Fatalf("The --cert and --key options are required.\n")
		}
	}

	var err error
	ptInfo, err = pt.ServerSetup([]string{ptMethodName})
	if err != nil {
		log.Fatalf("error in ServerSetup: %s", err)
	}

	log.Printf("starting")
	listeners := make([]net.Listener, 0)
	for _, bindaddr := range ptInfo.Bindaddrs {
		if port != 0 {
			bindaddr.Addr.Port = port
		}
		switch bindaddr.MethodName {
		case ptMethodName:
			var ln net.Listener
			if disableTLS {
				ln, err = startListener("tcp", bindaddr.Addr)
			} else {
				ln, err = startListenerTLS("tcp", bindaddr.Addr, certFilename, keyFilename)
			}
			if err != nil {
				pt.SmethodError(bindaddr.MethodName, err.Error())
				break
			}
			pt.Smethod(bindaddr.MethodName, ln.Addr())
			listeners = append(listeners, ln)
		default:
			pt.SmethodError(bindaddr.MethodName, "no such method")
		}
	}
	pt.SmethodsDone()

	var numHandlers int = 0
	var sig os.Signal
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// Wait for first signal.
	sig = nil
	for sig == nil {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case sig = <-sigChan:
			log.Printf("got signal %s", sig)
		}
	}
	for _, ln := range listeners {
		ln.Close()
	}

	if sig == syscall.SIGTERM {
		log.Printf("done")
		return
	}

	// Wait for second signal or no more handlers.
	sig = nil
	for sig == nil && numHandlers != 0 {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case sig = <-sigChan:
			log.Printf("got second signal %s", sig)
		}
	}

	log.Printf("done")
}
<commit_msg>Don't show error text that may contain public IPs.<commit_after>// meek-server is the server transport plugin for the meek pluggable transport.
// It acts as an HTTP server, keeps track of session ids, and forwards received
// data to a local OR port.
//
// Sample usage in torrc:
// 	ServerTransportPlugin meek exec ./meek-server --port 8443 --cert cert.pem --key key.pem --log meek-server.log
// Plain HTTP usage:
// 	ServerTransportPlugin meek exec ./meek-server --port 8080 --disable-tls --log meek-server.log
//
// The server runs in HTTPS mode by default, and the --cert and --key options
// are required. Use the --disable-tls option to run with plain HTTP.
package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"sync"
	"syscall"
	"time"
)

import "git.torproject.org/pluggable-transports/goptlib.git"

const (
	ptMethodName = "meek"
	// Reject session ids shorter than this, as a weak defense against
	// client bugs that send an empty session id or something similarly
	// likely to collide.
	minSessionIdLength = 32
	// The largest request body we are willing to process, and the largest
	// chunk of data we'll send back in a response.
	maxPayloadLength = 0x10000
	// How long we try to read something back from the OR port before
	// returning the response.
	turnaroundTimeout = 10 * time.Millisecond
	// Passed as ReadTimeout and WriteTimeout when constructing the
	// http.Server.
	readWriteTimeout = 20 * time.Second
	// Cull unused session ids (with their corresponding OR port connection)
	// if we haven't seen any activity for this long.
	maxSessionStaleness = 120 * time.Second
)

var ptInfo pt.ServerInfo

// When a connection handler starts, +1 is written to this channel; when it
// ends, -1 is written.
var handlerChan = make(chan int)

func httpBadRequest(w http.ResponseWriter) {
	http.Error(w, "Bad request.\n", http.StatusBadRequest)
}

func httpInternalServerError(w http.ResponseWriter) {
	http.Error(w, "Internal server error.\n", http.StatusInternalServerError)
}

// Every session id maps to an existing OR port connection, which we keep open
// between received requests. The first time we see a new session id, we create
// a new OR port connection.
type Session struct {
	Or       *net.TCPConn
	LastSeen time.Time
}

// Mark a session as having been seen just now.
func (session *Session) Touch() {
	session.LastSeen = time.Now()
}

// Is this session old enough to be culled?
func (session *Session) IsExpired() bool {
	return time.Since(session.LastSeen) > maxSessionStaleness
}

// There is one state per HTTP listener. In the usual case there is just one
// listener, so there is just one global state. State also serves as the http
// Handler.
type State struct {
	sessionMap map[string]*Session
	lock       sync.Mutex
}

func NewState() *State {
	state := new(State)
	state.sessionMap = make(map[string]*Session)
	return state
}

func (state *State) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	handlerChan <- 1
	defer func() {
		handlerChan <- -1
	}()

	switch req.Method {
	case "GET":
		state.Get(w, req)
	case "POST":
		state.Post(w, req)
	default:
		httpBadRequest(w)
	}
}

// Handle a GET request. This doesn't have any purpose apart from diagnostics.
func (state *State) Get(w http.ResponseWriter, req *http.Request) {
	if path.Clean(req.URL.Path) != "/" {
		http.NotFound(w, req)
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("I’m just a happy little web server.\n"))
}

// Look up a session by id, or create a new one (with its OR port connection) if
// it doesn't already exist.
func (state *State) GetSession(sessionId string, req *http.Request) (*Session, error) {
	state.lock.Lock()
	defer state.lock.Unlock()

	session := state.sessionMap[sessionId]
	if session == nil {
		// log.Printf("unknown session id %q; creating new session", sessionId)

		or, err := pt.DialOr(&ptInfo, req.RemoteAddr, ptMethodName)
		if err != nil {
			return nil, err
		}
		session = &Session{Or: or}
		state.sessionMap[sessionId] = session
	}
	session.Touch()

	return session, nil
}

// Feed the body of req into the OR port, and write any data read from the OR
// port back to w.
func transact(session *Session, w http.ResponseWriter, req *http.Request) error {
	body := http.MaxBytesReader(w, req.Body, maxPayloadLength+1)
	_, err := io.Copy(session.Or, body)
	if err != nil {
		// Omit err because it contains an IP address.
		return fmt.Errorf("error copying body to ORPort")
	}

	buf := make([]byte, maxPayloadLength)
	session.Or.SetReadDeadline(time.Now().Add(turnaroundTimeout))
	n, err := session.Or.Read(buf)
	if err != nil {
		if e, ok := err.(net.Error); !ok || !e.Timeout() {
			httpInternalServerError(w)
			return fmt.Errorf("reading from ORPort: %s", err)
		}
	}
	// log.Printf("read %d bytes from ORPort: %q", n, buf[:n])
	// Set a Content-Type to prevent Go and the CDN from trying to guess.
	w.Header().Set("Content-Type", "application/octet-stream")
	n, err = w.Write(buf[:n])
	if err != nil {
		// Omit err because it contains an IP address.
		return fmt.Errorf("error writing to response")
	}
	// log.Printf("wrote %d bytes to response", n)
	return nil
}

// Handle a POST request. Look up the session id and then do a transaction.
func (state *State) Post(w http.ResponseWriter, req *http.Request) {
	sessionId := req.Header.Get("X-Session-Id")
	if len(sessionId) < minSessionIdLength {
		httpBadRequest(w)
		return
	}

	session, err := state.GetSession(sessionId, req)
	if err != nil {
		log.Print(err)
		httpInternalServerError(w)
		return
	}

	err = transact(session, w, req)
	if err != nil {
		log.Print(err)
		state.CloseSession(sessionId)
		return
	}
}

// Remove a session from the map and closes its corresponding OR port
// connection. Does nothing if the session id is not known.
func (state *State) CloseSession(sessionId string) {
	state.lock.Lock()
	defer state.lock.Unlock()
	// log.Printf("closing session %q", sessionId)
	session, ok := state.sessionMap[sessionId]
	if ok {
		session.Or.Close()
		delete(state.sessionMap, sessionId)
	}
}

// Loop forever, checking for expired sessions and removing them.
func (state *State) ExpireSessions() {
	for {
		time.Sleep(maxSessionStaleness / 2)
		state.lock.Lock()
		for sessionId, session := range state.sessionMap {
			if session.IsExpired() {
				// log.Printf("deleting expired session %q", sessionId)
				session.Or.Close()
				delete(state.sessionMap, sessionId)
			}
		}
		state.lock.Unlock()
	}
}

func listenTLS(network string, addr *net.TCPAddr, certFilename, keyFilename string) (net.Listener, error) {
	// This is cribbed from the source of net/http.Server.ListenAndServeTLS.
	// We have to separate the Listen and Serve parts because we need to
	// report the listening address before entering Serve (which is an
	// infinite loop).
	// https://groups.google.com/d/msg/Golang-nuts/3F1VRCCENp8/3hcayZiwYM8J
	config := &tls.Config{}
	config.NextProtos = []string{"http/1.1"}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFilename, keyFilename)
	if err != nil {
		return nil, err
	}

	conn, err := net.ListenTCP(network, addr)
	if err != nil {
		return nil, err
	}

	tlsListener := tls.NewListener(conn, config)

	return tlsListener, nil
}

func startListener(network string, addr *net.TCPAddr) (net.Listener, error) {
	ln, err := net.ListenTCP(network, addr)
	if err != nil {
		return nil, err
	}
	log.Printf("listening with plain HTTP on %s", ln.Addr())
	return startServer(ln)
}

func startListenerTLS(network string, addr *net.TCPAddr, certFilename, keyFilename string) (net.Listener, error) {
	ln, err := listenTLS(network, addr, certFilename, keyFilename)
	if err != nil {
		return nil, err
	}
	log.Printf("listening with HTTPS on %s", ln.Addr())
	return startServer(ln)
}

func startServer(ln net.Listener) (net.Listener, error) {
	state := NewState()
	go state.ExpireSessions()
	server := &http.Server{
		Handler:      state,
		ReadTimeout:  readWriteTimeout,
		WriteTimeout: readWriteTimeout,
	}
	go func() {
		defer ln.Close()
		err := server.Serve(ln)
		if err != nil {
			log.Printf("Error in Serve: %s", err)
		}
	}()
	return ln, nil
}

func main() {
	var disableTLS bool
	var certFilename, keyFilename string
	var logFilename string
	var port int

	flag.BoolVar(&disableTLS, "disable-tls", false, "don't use HTTPS")
	flag.StringVar(&certFilename, "cert", "", "TLS certificate file (required without --disable-tls)")
	flag.StringVar(&keyFilename, "key", "", "TLS private key file (required without --disable-tls)")
	flag.StringVar(&logFilename, "log", "", "name of log file")
	flag.IntVar(&port, "port", 0, "port to listen on")
	flag.Parse()

	if logFilename != "" {
		f, err := os.OpenFile(logFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			log.Fatalf("error opening log file: %s", err)
		}
		defer f.Close()
		log.SetOutput(f)
	}

	if disableTLS {
		if certFilename != "" || keyFilename != "" {
			log.Fatalf("The --cert and --key options are not allowed with --disable-tls.\n")
		}
	} else {
		if certFilename == "" || keyFilename == "" {
			log.Fatalf("The --cert and --key options are required.\n")
		}
	}

	var err error
	ptInfo, err = pt.ServerSetup([]string{ptMethodName})
	if err != nil {
		log.Fatalf("error in ServerSetup: %s", err)
	}

	log.Printf("starting")
	listeners := make([]net.Listener, 0)
	for _, bindaddr := range ptInfo.Bindaddrs {
		if port != 0 {
			bindaddr.Addr.Port = port
		}
		switch bindaddr.MethodName {
		case ptMethodName:
			var ln net.Listener
			if disableTLS {
				ln, err = startListener("tcp", bindaddr.Addr)
			} else {
				ln, err = startListenerTLS("tcp", bindaddr.Addr, certFilename, keyFilename)
			}
			if err != nil {
				pt.SmethodError(bindaddr.MethodName, err.Error())
				break
			}
			pt.Smethod(bindaddr.MethodName, ln.Addr())
			listeners = append(listeners, ln)
		default:
			pt.SmethodError(bindaddr.MethodName, "no such method")
		}
	}
	pt.SmethodsDone()

	var numHandlers int = 0
	var sig os.Signal
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// Wait for first signal.
	sig = nil
	for sig == nil {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case sig = <-sigChan:
			log.Printf("got signal %s", sig)
		}
	}
	for _, ln := range listeners {
		ln.Close()
	}

	if sig == syscall.SIGTERM {
		log.Printf("done")
		return
	}

	// Wait for second signal or no more handlers.
	sig = nil
	for sig == nil && numHandlers != 0 {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case sig = <-sigChan:
			log.Printf("got second signal %s", sig)
		}
	}

	log.Printf("done")
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package feed

import (
	"github.com/nmeum/go-feedparser"
	"github.com/nmeum/go-feedparser/atom"
	"github.com/nmeum/go-feedparser/rss"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"net/http"
	"strings"
	"sync"
	"time"
)

var parsers = []feedparser.FeedFunc{rss.Parse, atom.Parse}

type Module struct {
	feeds    map[string]time.Time
	URLs     []string `json:"urls"`
	Interval string   `json:"interval"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "feed"
}

func (m *Module) Help() string {
	return "Display new entries for RSS/ATOM feeds."
}

func (m *Module) Defaults() {
	m.Interval = "0h15m"
}

func (m *Module) Load(client *irc.Client) error {
	m.feeds = make(map[string]time.Time)
	for _, url := range m.URLs {
		m.feeds[url] = time.Now()
	}

	duration, err := time.ParseDuration(m.Interval)
	if err != nil {
		return err
	}

	newItems := make(chan feedparser.Item)
	go func() {
		for item := range newItems {
			m.notify(client, item)
		}
	}()

	go func() {
		for {
			time.Sleep(duration)
			m.pollFeeds(newItems)
		}
	}()

	return nil
}

func (m *Module) pollFeeds(out chan feedparser.Item) {
	var wg sync.WaitGroup
	for _, url := range m.URLs {
		wg.Add(1)
		go func(u string) {
			defer wg.Done()
			feed, err := m.fetchFeed(u)
			if err != nil {
				return
			}

			latest := m.feeds[u]
			m.feeds[u] = feed.Items[0].Date

			for _, i := range feed.Items {
				if i.Date.Before(latest) || i.Date.Equal(latest) {
					break
				}

				out <- i
			}
		}(url)
	}

	wg.Wait()
}

func (m *Module) fetchFeed(url string) (feed feedparser.Feed, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}

	reader := resp.Body
	defer reader.Close()

	feed, err = feedparser.Parse(reader, parsers)
	if err != nil {
		return
	}

	return
}

func (m *Module) notify(client *irc.Client, item feedparser.Item) {
	for _, ch := range client.Channels {
		client.Write("NOTICE %s :%s -- %s -- %s",
			ch, strings.ToUpper(m.Name()), item.Title, item.Link)
	}
}
<commit_msg>Updated to refactored version of go-feedparser<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package feed

import (
	"github.com/nmeum/go-feedparser"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"net/http"
	"strings"
	"sync"
	"time"
)

type Module struct {
	feeds    map[string]time.Time
	URLs     []string `json:"urls"`
	Interval string   `json:"interval"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "feed"
}

func (m *Module) Help() string {
	return "Display new entries for RSS/ATOM feeds."
}

func (m *Module) Defaults() {
	m.Interval = "0h15m"
}

func (m *Module) Load(client *irc.Client) error {
	m.feeds = make(map[string]time.Time)
	for _, url := range m.URLs {
		m.feeds[url] = time.Now()
	}

	duration, err := time.ParseDuration(m.Interval)
	if err != nil {
		return err
	}

	newItems := make(chan feedparser.Item)
	go func() {
		for item := range newItems {
			m.notify(client, item)
		}
	}()

	go func() {
		for {
			time.Sleep(duration)
			m.pollFeeds(newItems)
		}
	}()

	return nil
}

func (m *Module) pollFeeds(out chan feedparser.Item) {
	var wg sync.WaitGroup
	for _, url := range m.URLs {
		wg.Add(1)
		go func(u string) {
			defer wg.Done()
			feed, err := m.fetchFeed(u)
			if err != nil {
				return
			}

			latest := m.feeds[u]
			m.feeds[u] = feed.Items[0].PubDate

			for _, i := range feed.Items {
				if i.PubDate.Before(latest) || i.PubDate.Equal(latest) {
					break
				}

				out <- i
			}
		}(url)
	}

	wg.Wait()
}

func (m *Module) fetchFeed(url string) (feed feedparser.Feed, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}

	reader := resp.Body
	defer reader.Close()

	feed, err = feedparser.Parse(reader)
	if err != nil {
		return
	}

	return
}

func (m *Module) notify(client *irc.Client, item feedparser.Item) {
	for _, ch := range client.Channels {
		client.Write("NOTICE %s :%s -- %s -- %s",
			ch, strings.ToUpper(m.Name()), item.Title, item.Link)
	}
}
<|endoftext|>


<commit_before>package classpath

import (
	"os"
	"path/filepath"
	"strings"
)

type WildcardEntry struct {
	CompositeEntry
}

func newWildcardEntry(path string) *WildcardEntry {
	baseDir := path[:len(path)-1] // remove *
	entry := &WildcardEntry{}

	walkFn := func(path string, info os.FileInfo, err error) error {
		if info.IsDir() && path != baseDir {
			return filepath.SkipDir
		}
		if strings.HasSuffix(path, ".jar") || strings.HasSuffix(path, ".JAR") {
			jarEntry := newZipEntry(path)
			entry.addEntry(jarEntry)
		}
		return nil
	}

	filepath.Walk(baseDir, walkFn)

	return entry
}
<commit_msg>fixed: panic when `GetFileAttributesEx` error<commit_after>package classpath

import (
	"os"
	"path/filepath"
	"strings"
)

type WildcardEntry struct {
	CompositeEntry
}

func newWildcardEntry(path string) *WildcardEntry {
	baseDir := path[:len(path)-1] // remove *
	entry := &WildcardEntry{}

	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != baseDir {
			return filepath.SkipDir
		}
		if strings.HasSuffix(path, ".jar") || strings.HasSuffix(path, ".JAR") {
			jarEntry := newZipEntry(path)
			entry.addEntry(jarEntry)
		}
		return nil
	}

	filepath.Walk(baseDir, walkFn)

	return entry
}
<|endoftext|>


<commit_before>package jsonapi

import (
	"io"
	"reflect"

	"github.com/nstratos/jsonapi"
)

func EncodeOne(w io.Writer, v interface{}) error {
	return jsonapi.MarshalOnePayload(w, v)
}

func EncodeMany(w io.Writer, v interface{}) error {
	return jsonapi.MarshalManyPayload(w, v)
}

func DecodeOne(r io.Reader, v interface{}) error {
	return jsonapi.UnmarshalPayload(r, v)
}

func DecodeMany(r io.Reader, t reflect.Type) ([]interface{}, Offset, error) {
	v, links, err := jsonapi.UnmarshalManyPayloadWithLinks(r, t)
	if err != nil {
		return nil, Offset{}, err
	}

	o := Offset{}
	var perr error
	if links != nil {
		o, perr = parseOffset(*links)
		if perr != nil {
			return nil, Offset{}, perr
		}
	}
	return v, o, err
}
<commit_msg>Combine DecodeOne and DecodeMany into Decode<commit_after>package jsonapi

import (
	"fmt"
	"io"
	"reflect"
	"runtime"

	"github.com/nstratos/jsonapi"
)

func isZeroOfUnderlyingType(x interface{}) bool {
	return reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface())
}

func Encode(w io.Writer, v interface{}) (err error) {
	const errFormat = "cannot encode type %T, need pointer to struct or slice of pointers to structs"
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}
			err = fmt.Errorf(errFormat+": %v", v, r.(error))
		}
	}()
	if isZeroOfUnderlyingType(v) {
		return fmt.Errorf("cannot encode nil value of %#v", v)
	}
	t := reflect.TypeOf(v)
	switch t.Kind() {
	default:
		return fmt.Errorf(errFormat, v)
	case reflect.Ptr:
		if t.Elem().Kind() != reflect.Struct {
			return fmt.Errorf(errFormat, v)
		}
		return jsonapi.MarshalOnePayload(w, v)
	case reflect.Slice:
		s := reflect.ValueOf(v)
		if s.Type().Elem().Kind() != reflect.Ptr {
			return fmt.Errorf(errFormat, v)
		}
		if s.Type().Elem().Elem().Kind() != reflect.Struct {
			return fmt.Errorf(errFormat, v)

		}
		return jsonapi.MarshalManyPayload(w, v)
	}
}

func Decode(r io.Reader, ptr interface{}) (Offset, error) {
	const errFormat = "cannot decode to %T, need pointer to struct or pointer to slice"
	if reflect.TypeOf(ptr).Kind() != reflect.Ptr {
		return Offset{}, fmt.Errorf(errFormat, ptr)
	}
	v := reflect.Indirect(reflect.ValueOf(ptr))
	switch v.Kind() {
	default:
		return Offset{}, fmt.Errorf(errFormat, ptr)
	case reflect.Struct:
		return Offset{}, jsonapi.UnmarshalPayload(r, ptr)
	case reflect.Slice:
		data, links, err := jsonapi.UnmarshalManyPayloadWithLinks(r, v.Type().Elem())
		if err != nil {
			return Offset{}, err
		}
		for _, d := range data {
			v.Set(reflect.Append(v, reflect.ValueOf(d)))
		}

		o := Offset{}
		var perr error
		if links != nil {
			o, perr = parseOffset(*links)
			if perr != nil {
				return Offset{}, perr
			}
		}
		return o, nil
	}
}

func EncodeOne(w io.Writer, v interface{}) error {
	return jsonapi.MarshalOnePayload(w, v)
}

func EncodeMany(w io.Writer, v interface{}) error {
	return jsonapi.MarshalManyPayload(w, v)
}

func DecodeOne(r io.Reader, v interface{}) error {
	return jsonapi.UnmarshalPayload(r, v)
}

func DecodeMany(r io.Reader, t reflect.Type) ([]interface{}, Offset, error) {
	v, links, err := jsonapi.UnmarshalManyPayloadWithLinks(r, t)
	if err != nil {
		return nil, Offset{}, err
	}

	o := Offset{}
	var perr error
	if links != nil {
		o, perr = parseOffset(*links)
		if perr != nil {
			return nil, Offset{}, perr
		}
	}
	return v, o, err
}
<|endoftext|>


<commit_before>// +build linux

package modules

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strconv"
)

func init() {
	Modules["geekbench"] = &Geekbench{}
}

type Geekbench struct {
	SingleCore int
	MultiCore  int
	URL        string
	ID         int
}

func (stat *Geekbench) Run() error {
	res, err := http.Get("http://cdn.primatelabs.com/Geekbench-4.1.0-Linux.tar.gz")
	if err != nil {
		return err
	}

	if err := stat.extract(res.Body); err != nil {
		return err
	}

	gb := exec.Command("build.pulse/dist/Geekbench-4.1.0-Linux/geekbench4")

	out, err := gb.Output()
	if err != nil {
		fmt.Printf("%s", err.(*exec.ExitError).Stderr)
		return err
	}

	if err := os.RemoveAll("build.pulse"); err != nil {
		return err
	}

	r, err := regexp.Compile("https://browser.geekbench.com/v4/cpu/([0-9]*)")
	if err != nil {
		return err
	}

	match := r.FindStringSubmatch(string(out))
	if len(match) < 2 {
		return errors.New("geekbench did not return result url")
	}

	stat.URL = match[0]
	id, _ := strconv.Atoi(match[1])
	stat.ID = id

	stat.SingleCore, stat.MultiCore, err = stat.scrape(stat.URL)
	if err != nil {
		return err
	}

	return nil
}

func (stat *Geekbench) Print() {
	fmt.Printf("Single-Core Score  : %d\n", stat.SingleCore)
	fmt.Printf("Multi-Core Score   : %d\n", stat.MultiCore)
	fmt.Printf("Result URL         : %s\n", stat.URL)
}

func (stat *Geekbench) scrape(url string) (int, int, error) {
	res, err := http.Get(url)
	if err != nil {
		return 0, 0, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, 0, err
	}

	singleReg, err := regexp.Compile("Single-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	single := singleReg.FindStringSubmatch(string(body))
	if len(single) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	multiReg, err := regexp.Compile("Multi-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	multi := multiReg.FindStringSubmatch(string(body))
	if len(multi) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	singleScore, _ := strconv.Atoi(single[1])
	multiScore, _ := strconv.Atoi(multi[1])

	return singleScore, multiScore, nil
}

func (stat *Geekbench) extract(r io.Reader) error {

	gzf, err := gzip.NewReader(r)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	tarReader := tar.NewReader(gzf)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}

		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(header.Name, os.ModePerm); err != nil {
				return err
			}
		case tar.TypeReg:
			f, err := os.Create(header.Name)
			if err != nil {
				return err
			}
			defer f.Close()

			if _, err := io.Copy(f, tarReader); err != nil {
				return err
			}

			os.Chmod(header.Name, os.ModePerm)
		default:
			return fmt.Errorf("%s %c %s %s", "unexpected file type", header.Typeflag, "in file", header.Name)
		}
	}

}
<commit_msg>geekbench: err is usually ExitError, not always<commit_after>// +build linux

package modules

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strconv"
)

func init() {
	Modules["geekbench"] = &Geekbench{}
}

type Geekbench struct {
	SingleCore int
	MultiCore  int
	URL        string
	ID         int
}

func (stat *Geekbench) Run() error {
	res, err := http.Get("http://cdn.primatelabs.com/Geekbench-4.1.0-Linux.tar.gz")
	if err != nil {
		return err
	}

	if err := stat.extract(res.Body); err != nil {
		return err
	}

	gb := exec.Command("build.pulse/dist/Geekbench-4.1.0-Linux/geekbench4")

	out, err := gb.Output()
	if err != nil {
		exit, ok := err.(*exec.ExitError)
		if ok {
			fmt.Printf("%s", exit.Stderr)
		}

		return err
	}

	if err := os.RemoveAll("build.pulse"); err != nil {
		return err
	}

	r, err := regexp.Compile("https://browser.geekbench.com/v4/cpu/([0-9]*)")
	if err != nil {
		return err
	}

	match := r.FindStringSubmatch(string(out))
	if len(match) < 2 {
		return errors.New("geekbench did not return result url")
	}

	stat.URL = match[0]
	id, _ := strconv.Atoi(match[1])
	stat.ID = id

	stat.SingleCore, stat.MultiCore, err = stat.scrape(stat.URL)
	if err != nil {
		return err
	}

	return nil
}

func (stat *Geekbench) Print() {
	fmt.Printf("Single-Core Score  : %d\n", stat.SingleCore)
	fmt.Printf("Multi-Core Score   : %d\n", stat.MultiCore)
	fmt.Printf("Result URL         : %s\n", stat.URL)
}

func (stat *Geekbench) scrape(url string) (int, int, error) {
	res, err := http.Get(url)
	if err != nil {
		return 0, 0, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, 0, err
	}

	singleReg, err := regexp.Compile("Single-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	single := singleReg.FindStringSubmatch(string(body))
	if len(single) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	multiReg, err := regexp.Compile("Multi-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	multi := multiReg.FindStringSubmatch(string(body))
	if len(multi) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	singleScore, _ := strconv.Atoi(single[1])
	multiScore, _ := strconv.Atoi(multi[1])

	return singleScore, multiScore, nil
}

func (stat *Geekbench) extract(r io.Reader) error {

	gzf, err := gzip.NewReader(r)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	tarReader := tar.NewReader(gzf)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}

		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(header.Name, os.ModePerm); err != nil {
				return err
			}
		case tar.TypeReg:
			f, err := os.Create(header.Name)
			if err != nil {
				return err
			}
			defer f.Close()

			if _, err := io.Copy(f, tarReader); err != nil {
				return err
			}

			os.Chmod(header.Name, os.ModePerm)
		default:
			return fmt.Errorf("%s %c %s %s", "unexpected file type", header.Typeflag, "in file", header.Name)
		}
	}

}
<|endoftext|>


<commit_before>package openweathermap

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type ForecastSys struct {
	Population int `json:"population"`
}

type Temperature struct {
	Day   float64 `json:"day"`
	Min   float64 `json:"min"`
	Max   float64 `json:"max"`
	Night float64 `json:"night"`
	Eve   float64 `json:"eve"`
	Morn  float64 `json:"morn"`
}

type City struct {
	ID         int         `json:"id"`
	Name       string      `json:"name"`
	Coord      Coordinates `json:"coord"`
	Country    string      `json:"country"`
	Population int         `json:"population"`
	Sys        ForecastSys `json:"sys"`
}

type ForecastWeatherList struct {
	Dt       int         `json:"dt"`
	Temp     Temperature `json:"temp"`
	Pressure float64     `json:"pressure"`
	Humidity int         `json:"humidity"`
	Weather  []Weather   `json:"weather"`
	Speed    float64     `json:"speed"`
	Deg      int         `json:"deg"`
	Clouds   int         `json:"clouds"`
	Rain     int         `json:"rain"`
}

type ForecastWeatherData struct {
	COD     string                `json:"cod"`
	Message float64               `json:"message"`
	City    City                  `json:"city"`
	Cnt     int                   `json:"cnt"`
	List    []ForecastWeatherList `json:"list"`
	Units   string
}

// NewHistorical returns a new HistoricalWeatherData pointer with the supplied
// arguments.
func NewForecast(unit string) (*ForecastWeatherData, error) {
	unitChoice := strings.ToLower(unit)
	for _, i := range dataUnits {
		if strings.Contains(unitChoice, i) {
			return &ForecastWeatherData{Units: unitChoice}, nil
		}
	}
	return nil, errors.New("ERROR: unit of measure not available")
}

// DailyByName will provide a forecast for the location given for the
// number of days given.
func (f *ForecastWeatherData) DailyByName(location string, days int) {
	response, err := http.Get(fmt.Sprintf(forecastBase, location, f.Units, days))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &f)
	if err != nil {
		log.Fatalln(err)
	}
}

// DailyByID will provide a forecast for the location ID give for the
// number of days given.
func (f *ForecastWeatherData) DailyByID() {}

// DailyByCoordinates will provide a forecast for the coordinates ID give for the
// number of days given.
func (f *ForecastWeatherData) DailyByCoordinates() {}
<commit_msg>forecast for location id<commit_after>package openweathermap

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type ForecastSys struct {
	Population int `json:"population"`
}

type Temperature struct {
	Day   float64 `json:"day"`
	Min   float64 `json:"min"`
	Max   float64 `json:"max"`
	Night float64 `json:"night"`
	Eve   float64 `json:"eve"`
	Morn  float64 `json:"morn"`
}

type City struct {
	ID         int         `json:"id"`
	Name       string      `json:"name"`
	Coord      Coordinates `json:"coord"`
	Country    string      `json:"country"`
	Population int         `json:"population"`
	Sys        ForecastSys `json:"sys"`
}

type ForecastWeatherList struct {
	Dt       int         `json:"dt"`
	Temp     Temperature `json:"temp"`
	Pressure float64     `json:"pressure"`
	Humidity int         `json:"humidity"`
	Weather  []Weather   `json:"weather"`
	Speed    float64     `json:"speed"`
	Deg      int         `json:"deg"`
	Clouds   int         `json:"clouds"`
	Rain     int         `json:"rain"`
}

type ForecastWeatherData struct {
	COD     string                `json:"cod"`
	Message float64               `json:"message"`
	City    City                  `json:"city"`
	Cnt     int                   `json:"cnt"`
	List    []ForecastWeatherList `json:"list"`
	Units   string
}

// NewHistorical returns a new HistoricalWeatherData pointer with the supplied
// arguments.
func NewForecast(unit string) (*ForecastWeatherData, error) {
	unitChoice := strings.ToLower(unit)
	for _, i := range dataUnits {
		if strings.Contains(unitChoice, i) {
			return &ForecastWeatherData{Units: unitChoice}, nil
		}
	}
	return nil, errors.New("ERROR: unit of measure not available")
}

// DailyByName will provide a forecast for the location given for the
// number of days given.
func (f *ForecastWeatherData) DailyByName(location string, days int) {
	response, err := http.Get(fmt.Sprintf(forecastBase, location, f.Units, days))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &f)
	if err != nil {
		log.Fatalln(err)
	}
}

// DailyByCoordinates will provide a forecast for the coordinates ID give for the
// number of days given.
func (f *ForecastWeatherData) DailyByCoordinates(location *Coordinates) {}

// DailyByID will provide a forecast for the location ID give for the
// number of days given.
func (f *ForecastWeatherData) DailyByID(id int, days int) {
	response, err := http.Get(fmt.Sprintf(forecastBase, id, f.Units, days))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &f)
	if err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>package memaccess

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type process struct {
	pid          uint
	mapsFilepath string
	memFilepath  string
}

type mapInfo struct {
	start uintptr
	end   uintptr
}

func newProcessMemoryReaderImpl(pid uint) (process, error, []error) {
	var result process
	result.pid = pid
	result.mapsFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "maps")
	result.memFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "mem")
	// trying to open the maps file, only to see if gives an error
	f, harderror := os.Open(result.mapsFilepath)
	softerrors := make([]error, 0)
	if harderror != nil {
		return process{}, harderror, softerrors
	}
	f.Close()
	return result, nil, softerrors
}

func (p process) Close() (error, []error) {
	return nil, make([]error, 0)
}

// NextReadableMemoryRegion should return a MemoryRegion with address inside or,
// if that's impossible, the next readable MemoryRegion
func (p process) NextReadableMemoryRegion(address uintptr) (MemoryRegion, error, []error) {
	mapsFile, harderror := os.Open(p.mapsFilepath)
	softerrors := make([]error, 0)
	if harderror != nil {
		return MemoryRegion{}, harderror, softerrors
	}
	defer mapsFile.Close()

	mappedAddresses, harderror := getMappedAddresses(mapsFile)
	if harderror != nil {
		return MemoryRegion{}, harderror, softerrors
	}

	mappedRegion, harderror := nextReadableMappedRegion(address, mappedAddresses)
	//TODO: ignore non readable mapped regions and add a softerror
	if harderror != nil {
		return NoRegionAvailable, harderror, softerrors
	}

	if mappedRegion.start != 0 {
		size := uint(mappedRegion.end - mappedRegion.start)
		return MemoryRegion{mappedRegion.start, size}, nil, softerrors
	}
	return NoRegionAvailable, nil, softerrors
}

func nextReadableMappedRegion(address uintptr, mappedAddresses []mapInfo) (mapInfo, error) {
	for _, mapinfo := range mappedAddresses {
		if mapinfo.start <= address && address < mapinfo.end {
			return mapinfo, nil
		}
	}
	// there's no mapped region with address inside it
	// I should return the next one
	for _, mapinfo := range mappedAddresses {
		if address < mapinfo.start {
			return mapinfo, nil
		}
	}
	// there's no mapped region with address inside it and no next region
	return mapInfo{}, nil
}

func getMappedAddresses(mapsFile *os.File) ([]mapInfo, error) {
	res := make([]mapInfo, 0)
	scanner := bufio.NewScanner(mapsFile)

	for scanner.Scan() {
		line := scanner.Text()
		items := strings.Split(line, " ")
		if len(items) <= 1 {
			continue
		}

		fields := strings.Split(items[0], "-")
		start64, err := strconv.ParseUint(fields[0], 16, 64)
		if err != nil {
			return nil, err
		}
		end64, err := strconv.ParseUint(fields[1], 16, 64)
		if err != nil {
			return nil, err
		}
		start := uintptr(start64)
		end := uintptr(end64)
		info := mapInfo{start: start, end: end}
		res = append(res, info)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return res, nil
}

func (p process) CopyMemory(address uintptr, buffer []byte) (error, []error) {
	mem, harderror := os.Open(p.memFilepath)
	softerrors := make([]error, 0)
	if harderror != nil {
		harderror := fmt.Errorf("Error while reading %d bytes starting at %x: %s", len(buffer), address, harderror)
		return harderror, softerrors
	}
	defer mem.Close()

	bytes_read, harderror := mem.ReadAt(buffer, int64(address))
	if harderror != nil {
		return harderror, softerrors
	}

	if bytes_read != len(buffer) {
		return fmt.Errorf("Coul not read the entire buffer"), softerrors
	}

	return nil, softerrors
}
<commit_msg>[memaccess/linux] Fix NextReadableMemoryRegion<commit_after>package memaccess

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

type process struct {
	pid          uint
	mapsFilepath string
	memFilepath  string
}

func newProcessMemoryReaderImpl(pid uint) (process, error, []error) {
	var result process
	result.pid = pid
	result.mapsFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "maps")
	result.memFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "mem")
	softerrors := make([]error, 0)

	// trying to open the maps file, only to see if we have enought privileges
	f, harderror := os.Open(result.mapsFilepath)
	if harderror != nil {
		return process{}, harderror, softerrors
	}
	f.Close()

	return result, nil, softerrors
}

func (p process) Close() (error, []error) {
	return nil, make([]error, 0)
}

func (p process) NextReadableMemoryRegion(address uintptr) (region MemoryRegion, harderror error, softerrors []error) {
	// fmt.Printf("\n\n\nNextReadableMemoryRegion %x\n", address)
	softerrors = make([]error, 0)

	mapsFile, harderror := os.Open(p.mapsFilepath)
	if harderror != nil {
		return
	}
	defer mapsFile.Close()

	region = MemoryRegion{}
	scanner := bufio.NewScanner(mapsFile)
	splitBySpacesRegexp := regexp.MustCompile("\\s+")

	for scanner.Scan() {
		line := scanner.Text()
		items := splitBySpacesRegexp.Split(line, -1)

		if len(items) != 6 {
			return region, fmt.Errorf("Unrecognised maps line: %s", line), softerrors
		}

		start, end, err := parseMemoryLimits(items[0])
		if err != nil {
			return region, err, softerrors
		}

		if end <= address {
			continue
		}

		// Skip vsyscall as it can't be read. It's a special page mapped by the kernel to accelerate some syscalls.
		if items[5] == "[vsyscall]" {
			continue
		}

		// Check if memory is unreadable
		if items[1][0] == '-' {

			// If we were already reading a region this will just finish it. We only report the softerror when we
			// were actually trying to read it.
			if region.Address == 0 {
				return region, nil, softerrors
			}

			softerrors = append(softerrors, fmt.Errorf("Unreadable memory %s - address %x", items[0], address))
			continue
		}

		size := uint(end - start)

		// Begenning of a region
		if region.Address == 0 {
			region = MemoryRegion{Address: start, Size: size}
			continue
		}

		// Continuation of a region
		if region.Address+uintptr(region.Size) == start {
			region.Size += size
			continue
		}

		// This map is outside the current region, so we are ready
		return region, nil, softerrors
	}

	// No region left
	if err := scanner.Err(); err != nil {
		return region, err, softerrors
	}

	return NoRegionAvailable, nil, softerrors
}

func (p process) CopyMemory(address uintptr, buffer []byte) (error, []error) {
	mem, harderror := os.Open(p.memFilepath)
	softerrors := make([]error, 0)
	if harderror != nil {
		harderror := fmt.Errorf("Error while reading %d bytes starting at %x: %s", len(buffer), address, harderror)
		return harderror, softerrors
	}
	defer mem.Close()

	bytes_read, harderror := mem.ReadAt(buffer, int64(address))
	if harderror != nil {
		harderror := fmt.Errorf("Error while reading %d bytes starting at %x: %s", len(buffer), address, harderror)
		return harderror, softerrors
	}

	if bytes_read != len(buffer) {
		return fmt.Errorf("Coul not read the entire buffer"), softerrors
	}

	return nil, softerrors
}

//Parses the memory limits of a mapping as found in /proc/PID/maps
func parseMemoryLimits(limits string) (start uintptr, end uintptr, err error) {
	fields := strings.Split(limits, "-")
	start64, err := strconv.ParseUint(fields[0], 16, 64)
	if err != nil {
		return 0, 0, err
	}
	start = uintptr(start64)

	end64, err := strconv.ParseUint(fields[1], 16, 64)
	if err != nil {
		return 0, 0, err
	}
	end = uintptr(end64)

	return
}
<|endoftext|>


<commit_before>package oai

import "testing"

var testResponse = []byte(`<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="http://www.worldsciencepublisher.org/journals/lib/pkp/xml/oai2.xsl" ?>
<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/
		http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
	<responseDate>2015-10-16T19:24:52Z</responseDate>
	<request verb="ListMetadataFormats">http://www.worldsciencepublisher.org/journals/index.php/ACSA/oai</request>
	<ListMetadataFormats>
		<metadataFormat>
			<metadataPrefix>nlm</metadataPrefix>
			<schema>http://dtd.nlm.nih.gov/publishing/2.3/xsd/journalpublishing.xsd</schema>
			<metadataNamespace>http://td.nlm.nih.gov/publishing/2.3</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>oai_dc</metadataPrefix>
			<schema>http://www.openarchives.org/OAI/2.0/oai_dc.xsd</schema>
			<metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>rfc1807</metadataPrefix>
			<schema>http://www.openarchives.org/OAI/1.1/rfc1807.xsd</schema>
			<metadataNamespace>http://info.internet.isi.edu:80/in-notes/rfc/files/rfc1807.txt</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>marcxml</metadataPrefix>
			<schema>http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</schema>
			<metadataNamespace>http://www.loc.gov/MARC21/slim</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>oai_marc</metadataPrefix>
			<schema>http://www.openarchives.org/OAI/1.1/oai_marc.xsd</schema>
			<metadataNamespace>http://www.openarchives.org/OAI/1.1/oai_marc</metadataNamespace>
		</metadataFormat>
	</ListMetadataFormats>
</OAI-PMH>`)

// TESTMOCK OMIT
type mockFetcher struct {
}

func (mf mockFetcher) Fetch(url string) ([]byte, error) {
	return testResponse, nil // testResponse local []byte holding known good response
}

var oai = &OAI{
	OAIFetcher: mockFetcher{}, // inject the mock
}

// TESTMOCK OMIT

func TestFetchMetadataFormats(t *testing.T) {
	got, err := oai.FetchMetadataFormats("http://test.com")
	if err != nil {
		t.Errorf("Unepxted error from FetchMetadataFormats: %v", err)
	}

	expected := []MDFormat{
		{
			Prefix:    "nlm",
			Schema:    "http://dtd.nlm.nih.gov/publishing/2.3/xsd/journalpublishing.xsd",
			Namespace: "http://td.nlm.nih.gov/publishing/2.3",
		},
		{
			Prefix:    "oai_dc",
			Schema:    "http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
			Namespace: "http://www.openarchives.org/OAI/2.0/oai_dc/",
		},
		{
			Prefix:    "rfc1807",
			Schema:    "http://www.openarchives.org/OAI/1.1/rfc1807.xsd",
			Namespace: "http://info.internet.isi.edu:80/in-notes/rfc/files/rfc1807.txt",
		},
		{
			Prefix:    "marcxml",
			Schema:    "http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd",
			Namespace: "http://www.loc.gov/MARC21/slim",
		},
		{
			Prefix:    "oai_marc",
			Schema:    "http://www.openarchives.org/OAI/1.1/oai_marc.xsd",
			Namespace: "http://www.openarchives.org/OAI/1.1/oai_marc",
		},
	}

	for i, format := range got {
		if format.Prefix != expected[i].Prefix {
			t.Errorf("Test %d. Unexpected Prefix: %s, expected: %s", i, format.Prefix, expected[i].Prefix)
		}
		if format.Schema != expected[i].Schema {
			t.Errorf("Test %d. Unexpected Schema: %s, expected: %s", i, format.Schema, expected[i].Schema)
		}
		if format.Namespace != expected[i].Namespace {
			t.Errorf("Test %d. Unexpected Namespace: %s, expected: %s", i, format.Namespace, expected[i].Namespace)
		}
	}

}
<commit_msg>extra cpmment around deepequal<commit_after>package oai

import "testing"

var testResponse = []byte(`<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="http://www.worldsciencepublisher.org/journals/lib/pkp/xml/oai2.xsl" ?>
<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/
		http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
	<responseDate>2015-10-16T19:24:52Z</responseDate>
	<request verb="ListMetadataFormats">http://www.worldsciencepublisher.org/journals/index.php/ACSA/oai</request>
	<ListMetadataFormats>
		<metadataFormat>
			<metadataPrefix>nlm</metadataPrefix>
			<schema>http://dtd.nlm.nih.gov/publishing/2.3/xsd/journalpublishing.xsd</schema>
			<metadataNamespace>http://td.nlm.nih.gov/publishing/2.3</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>oai_dc</metadataPrefix>
			<schema>http://www.openarchives.org/OAI/2.0/oai_dc.xsd</schema>
			<metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>rfc1807</metadataPrefix>
			<schema>http://www.openarchives.org/OAI/1.1/rfc1807.xsd</schema>
			<metadataNamespace>http://info.internet.isi.edu:80/in-notes/rfc/files/rfc1807.txt</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>marcxml</metadataPrefix>
			<schema>http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</schema>
			<metadataNamespace>http://www.loc.gov/MARC21/slim</metadataNamespace>
		</metadataFormat>
		<metadataFormat>
			<metadataPrefix>oai_marc</metadataPrefix>
			<schema>http://www.openarchives.org/OAI/1.1/oai_marc.xsd</schema>
			<metadataNamespace>http://www.openarchives.org/OAI/1.1/oai_marc</metadataNamespace>
		</metadataFormat>
	</ListMetadataFormats>
</OAI-PMH>`)

// TESTMOCK OMIT
type mockFetcher struct {
}

func (mf mockFetcher) Fetch(url string) ([]byte, error) {
	return testResponse, nil // testResponse local []byte holding known good response
}

var oai = &OAI{
	OAIFetcher: mockFetcher{}, // inject the mock
}

// TESTMOCK OMIT

func TestFetchMetadataFormats(t *testing.T) {
	got, err := oai.FetchMetadataFormats("http://test.com")
	if err != nil {
		t.Errorf("Unepxted error from FetchMetadataFormats: %v", err)
	}

	expected := []MDFormat{
		{
			Prefix:    "nlm",
			Schema:    "http://dtd.nlm.nih.gov/publishing/2.3/xsd/journalpublishing.xsd",
			Namespace: "http://td.nlm.nih.gov/publishing/2.3",
		},
		{
			Prefix:    "oai_dc",
			Schema:    "http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
			Namespace: "http://www.openarchives.org/OAI/2.0/oai_dc/",
		},
		{
			Prefix:    "rfc1807",
			Schema:    "http://www.openarchives.org/OAI/1.1/rfc1807.xsd",
			Namespace: "http://info.internet.isi.edu:80/in-notes/rfc/files/rfc1807.txt",
		},
		{
			Prefix:    "marcxml",
			Schema:    "http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd",
			Namespace: "http://www.loc.gov/MARC21/slim",
		},
		{
			Prefix:    "oai_marc",
			Schema:    "http://www.openarchives.org/OAI/1.1/oai_marc.xsd",
			Namespace: "http://www.openarchives.org/OAI/1.1/oai_marc",
		},
	}

	for i, format := range got {
		if format.Prefix != expected[i].Prefix {
			t.Errorf("Test %d. Unexpected Prefix: %s, expected: %s", i, format.Prefix, expected[i].Prefix)
		}
		if format.Schema != expected[i].Schema {
			t.Errorf("Test %d. Unexpected Schema: %s, expected: %s", i, format.Schema, expected[i].Schema)
		}
		if format.Namespace != expected[i].Namespace {
			t.Errorf("Test %d. Unexpected Namespace: %s, expected: %s", i, format.Namespace, expected[i].Namespace)
		}
		// could replace all the above if with a reflect.DeepEquals call:
		/*
			if !reflect.DeepEqual(format, expected[i]) {
				t.Errorf("Mismatched format record:\nexp:%+v\ngot:%+v", expected[i], format)
			}
		*/
	}

}
<|endoftext|>