<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testing

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"runtime"
	"strings"
	"testing"
	"text/template"
	"time"

	"cloud.google.com/go/profiler/proftest"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"

	compute "google.golang.org/api/compute/v1"
)

var (
	gcsLocation       = flag.String("gcs_location", "", "GCS location for the agent")
	runID             = strings.Replace(time.Now().Format("2006-01-02-15-04-05.000000-0700"), ".", "-", -1)
	benchFinishString = "busybench finished profiling"
	errorString       = "failed to set up or run the benchmark"
)

const (
	cloudScope       = "https://www.googleapis.com/auth/cloud-platform"
	storageReadScope = "https://www.googleapis.com/auth/devstorage.read_only"
)

const startupTemplate = `
{{- template "prologue" . }}

# Install dependencies.
retry apt-get update >/dev/null
retry apt-get install -yq git build-essential python-dev python3-dev >/dev/null

# Install desired version of Python.
{{if .InstallPythonVersion}}
retry add-apt-repository ppa:deadsnakes/ppa >/dev/null
retry apt-get update >/dev/null
retry apt-get install {{.InstallPythonVersion}} >/dev/null
{{end}}

# Install Python dependencies.
retry wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py >/dev/null
retry {{.PythonCommand}} /tmp/get-pip.py >/dev/null
retry {{.PythonCommand}} -m pip install --upgrade pyasn1 >/dev/null

# Fetch agent.
mkdir /tmp/agent
retry gsutil cp gs://{{.GCSLocation}}/* /tmp/agent

# Install agent.
{{.PythonCommand}} -m pip install "$(find /tmp/agent -name "google-cloud-profiler*")"

# Run bench app.
export BENCH_DIR="$HOME/bench"
mkdir -p $BENCH_DIR
cd $BENCH_DIR

cat << EOF > bench.py
import googlecloudprofiler
import sys
import time
import traceback

def python_bench():
  for counter in range(1, 5000):
    pass

def repeat_bench(dur_sec):
  t_end = time.time() + dur_sec
  while time.time() < t_end or dur_sec == 0:
    python_bench()

if __name__ == '__main__':
  if not {{.VersionCheck}}:
    raise EnvironmentError('Python version %s failed to satisfy "{{.VersionCheck}}".' % str(sys.version_info))

  try:
    googlecloudprofiler.start(
      service='{{.Service}}',
      service_version='1.0.0',
      verbose=3)
  except BaseException:
    sys.exit('Failed to start the profiler: %s' % traceback.format_exc())
  repeat_bench(3 * 60)
EOF

# TODO(b/133360821): Stop ignoring exit code SIGALRM when b/133360821 is fixed.
{{.PythonCommand}} bench.py || [ "$?" -eq "142" ]

# Indicate to test that script has finished running.
echo "{{.FinishString}}"

{{ template "epilogue" . -}}
`

type testCase struct {
	proftest.InstanceConfig
	name string
	// Python version to install. Empty string means no installation is needed.
	installPythonVersion string
	// Python command name, e.g "python" or "python3".
	pythonCommand string
	// Used in the bench code to check the Python version, e.g
	// "sys.version_info[:2] == (2.7)".
	versionCheck string
	// Maps profile type to function name wanted for that type. Empty function
	// name means the type should not be profiled.
	wantProfiles map[string]string
}

func (tc *testCase) initializeStartUpScript(template *template.Template) error {
	var buf bytes.Buffer
	err := template.Execute(&buf,
		struct {
			Service              string
			GCSLocation          string
			InstallPythonVersion string
			PythonCommand        string
			VersionCheck         string
			FinishString         string
			ErrorString          string
		}{
			Service:              tc.name,
			GCSLocation:          *gcsLocation,
			InstallPythonVersion: tc.installPythonVersion,
			PythonCommand:        tc.pythonCommand,
			VersionCheck:         tc.versionCheck,
			FinishString:         benchFinishString,
			ErrorString:          errorString,
		})
	if err != nil {
		return fmt.Errorf("failed to render startup script for %s: %v", tc.name, err)
	}
	tc.StartupScript = buf.String()
	return nil
}

func TestAgentIntegration(t *testing.T) {
	projectID := os.Getenv("GCLOUD_TESTS_PYTHON_PROJECT_ID")
	if projectID == "" {
		t.Fatalf("Getenv(GCLOUD_TESTS_PYTHON_PROJECT_ID) got empty string")
	}

	zone := os.Getenv("GCLOUD_TESTS_PYTHON_ZONE")
	if zone == "" {
		t.Fatalf("Getenv(GCLOUD_TESTS_PYTHON_ZONE) got empty string")
	}

	if *gcsLocation == "" {
		t.Fatal("gcsLocation flag is not set")
	}

	ctx := context.Background()

	client, err := google.DefaultClient(ctx, cloudScope)
	if err != nil {
		t.Fatalf("failed to get default client: %v", err)
	}

	computeService, err := compute.New(client)
	if err != nil {
		t.Fatalf("failed to initialize compute Service: %v", err)
	}
	template, err := proftest.BaseStartupTmpl.Parse(startupTemplate)
	if err != nil {
		t.Fatalf("failed to parse startup script template: %v", err)
	}

	gceTr := proftest.GCETestRunner{
		TestRunner: proftest.TestRunner{
			Client: client,
		},
		ComputeService: computeService,
	}

	testcases := []testCase{
		// Test GCE Ubuntu default Python 2, should be Python 2.7.
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Zone:         zone,
				Name:         fmt.Sprintf("profiler-test-python2-%s", runID),
				MachineType:  "n1-standard-1",
				ImageProject: "ubuntu-os-cloud",
				ImageFamily:  "ubuntu-1804-lts",
				Scopes:       []string{storageReadScope},
			},
			name: fmt.Sprintf("profiler-test-python2-%s-gce", runID),
			wantProfiles: map[string]string{
				"WALL": "repeat_bench",
				// CPU profiling should be disabled on Python 2.
				"CPU": "",
			},
			pythonCommand: "python2.7",
			versionCheck:  "sys.version_info[:2] == (2, 7)",
		},
		// Test GCE Ubuntu default Python 3, should be Python 3.6 or higher.
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Zone:         zone,
				Name:         fmt.Sprintf("profiler-test-python3-%s", runID),
				MachineType:  "n1-standard-1",
				ImageProject: "ubuntu-os-cloud",
				ImageFamily:  "ubuntu-1804-lts",
				Scopes:       []string{storageReadScope},
			},
			name: fmt.Sprintf("profiler-test-python3-%s-gce", runID),
			wantProfiles: map[string]string{
				"WALL": "repeat_bench",
				"CPU":  "repeat_bench",
			},
			pythonCommand: "python3",
			versionCheck:  "sys.version_info[:2] >= (3, 6)",
		},
		// Test Python 3.5.
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Zone:         zone,
				Name:         fmt.Sprintf("profiler-test-python35-%s", runID),
				MachineType:  "n1-standard-1",
				ImageProject: "ubuntu-os-cloud",
				// ppa:deadsnakes/ppa is not yet available on Ubuntu 18.10.
				ImageFamily: "ubuntu-1604-lts",
				Scopes:      []string{storageReadScope},
			},
			name: fmt.Sprintf("profiler-test-python35-%s-gce", runID),
			wantProfiles: map[string]string{
				"CPU": "repeat_bench",
				// Wall profiling should be disabled on Python 3.5.
				"WALL": "",
			},
			installPythonVersion: "python3.5",
			pythonCommand:        "python3.5",
			versionCheck:         "sys.version_info[:2] == (3, 5)",
		},
	}

	// Allow test cases to run in parallel.
	runtime.GOMAXPROCS(len(testcases))

	for _, tc := range testcases {
		tc := tc // capture range variable
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if err := tc.initializeStartUpScript(template); err != nil {
				t.Fatalf("failed to initialize startup script: %v", err)
			}

			gceTr.StartInstance(ctx, &tc.InstanceConfig)
			defer func() {
				if gceTr.DeleteInstance(ctx, &tc.InstanceConfig); err != nil {
					t.Fatalf("failed to delete instance: %v", err)
				}
			}()

			timeoutCtx, cancel := context.WithTimeout(ctx, time.Minute*20)
			defer cancel()
			if err := gceTr.PollForSerialOutput(timeoutCtx, &tc.InstanceConfig, benchFinishString, errorString); err != nil {
				t.Fatal(err)
			}

			timeNow := time.Now()
			endTime := timeNow.Format(time.RFC3339)
			startTime := timeNow.Add(-1 * time.Hour).Format(time.RFC3339)
			for pType, function := range tc.wantProfiles {
				pr, err := gceTr.TestRunner.QueryProfiles(tc.ProjectID, tc.name, startTime, endTime, pType)
				if function == "" {
					if err == nil {
						t.Errorf("QueryProfiles(%s, %s, %s, %s, %s) got profile, want no profile", tc.ProjectID, tc.name, startTime, endTime, pType)
					}
					continue
				}

				if err != nil {
					t.Errorf("QueryProfiles(%s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.name, startTime, endTime, pType, err)
					continue
				}

				if err := pr.HasFunction(function); err != nil {
					t.Errorf("Function %s not found in profiles of type %s: %v", function, pType, err)
				}
			}
		})
	}
}
<commit_msg>Confirm zone is set in e2e test (#5)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testing

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"runtime"
	"strings"
	"testing"
	"text/template"
	"time"

	"cloud.google.com/go/profiler/proftest"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"

	compute "google.golang.org/api/compute/v1"
)

var (
	gcsLocation       = flag.String("gcs_location", "", "GCS location for the agent")
	runID             = strings.Replace(time.Now().Format("2006-01-02-15-04-05.000000-0700"), ".", "-", -1)
	benchFinishString = "busybench finished profiling"
	errorString       = "failed to set up or run the benchmark"
)

const (
	cloudScope       = "https://www.googleapis.com/auth/cloud-platform"
	storageReadScope = "https://www.googleapis.com/auth/devstorage.read_only"
)

const startupTemplate = `
{{- template "prologue" . }}

# Install dependencies.
retry apt-get update >/dev/null
retry apt-get install -yq git build-essential python-dev python3-dev >/dev/null

# Install desired version of Python.
{{if .InstallPythonVersion}}
retry add-apt-repository ppa:deadsnakes/ppa >/dev/null
retry apt-get update >/dev/null
retry apt-get install {{.InstallPythonVersion}} >/dev/null
{{end}}

# Install Python dependencies.
retry wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py >/dev/null
retry {{.PythonCommand}} /tmp/get-pip.py >/dev/null
retry {{.PythonCommand}} -m pip install --upgrade pyasn1 >/dev/null

# Fetch agent.
mkdir /tmp/agent
retry gsutil cp gs://{{.GCSLocation}}/* /tmp/agent

# Install agent.
{{.PythonCommand}} -m pip install "$(find /tmp/agent -name "google-cloud-profiler*")"

# Run bench app.
export BENCH_DIR="$HOME/bench"
mkdir -p $BENCH_DIR
cd $BENCH_DIR

cat << EOF > bench.py
import googlecloudprofiler
import sys
import time
import traceback

def python_bench():
  for counter in range(1, 5000):
    pass

def repeat_bench(dur_sec):
  t_end = time.time() + dur_sec
  while time.time() < t_end or dur_sec == 0:
    python_bench()

if __name__ == '__main__':
  if not {{.VersionCheck}}:
    raise EnvironmentError('Python version %s failed to satisfy "{{.VersionCheck}}".' % str(sys.version_info))

  try:
    googlecloudprofiler.start(
      service='{{.Service}}',
      service_version='1.0.0',
      verbose=3)
  except BaseException:
    sys.exit('Failed to start the profiler: %s' % traceback.format_exc())
  repeat_bench(3 * 60)
EOF

# TODO(b/133360821): Stop ignoring exit code SIGALRM when b/133360821 is fixed.
{{.PythonCommand}} bench.py || [ "$?" -eq "142" ]

# Indicate to test that script has finished running.
echo "{{.FinishString}}"

{{ template "epilogue" . -}}
`

type testCase struct {
	proftest.InstanceConfig
	name string
	// Python version to install. Empty string means no installation is needed.
	installPythonVersion string
	// Python command name, e.g "python" or "python3".
	pythonCommand string
	// Used in the bench code to check the Python version, e.g
	// "sys.version_info[:2] == (2.7)".
	versionCheck string
	// Maps profile type to function name wanted for that type. Empty function
	// name means the type should not be profiled.
	wantProfiles map[string]string
}

func (tc *testCase) initializeStartUpScript(template *template.Template) error {
	var buf bytes.Buffer
	err := template.Execute(&buf,
		struct {
			Service              string
			GCSLocation          string
			InstallPythonVersion string
			PythonCommand        string
			VersionCheck         string
			FinishString         string
			ErrorString          string
		}{
			Service:              tc.name,
			GCSLocation:          *gcsLocation,
			InstallPythonVersion: tc.installPythonVersion,
			PythonCommand:        tc.pythonCommand,
			VersionCheck:         tc.versionCheck,
			FinishString:         benchFinishString,
			ErrorString:          errorString,
		})
	if err != nil {
		return fmt.Errorf("failed to render startup script for %s: %v", tc.name, err)
	}
	tc.StartupScript = buf.String()
	return nil
}

func TestAgentIntegration(t *testing.T) {
	projectID := os.Getenv("GCLOUD_TESTS_PYTHON_PROJECT_ID")
	if projectID == "" {
		t.Fatalf("Getenv(GCLOUD_TESTS_PYTHON_PROJECT_ID) got empty string")
	}

	zone := os.Getenv("GCLOUD_TESTS_PYTHON_ZONE")
	if zone == "" {
		t.Fatalf("Getenv(GCLOUD_TESTS_PYTHON_ZONE) got empty string")
	}

	if *gcsLocation == "" {
		t.Fatal("gcsLocation flag is not set")
	}

	ctx := context.Background()

	client, err := google.DefaultClient(ctx, cloudScope)
	if err != nil {
		t.Fatalf("failed to get default client: %v", err)
	}

	computeService, err := compute.New(client)
	if err != nil {
		t.Fatalf("failed to initialize compute Service: %v", err)
	}
	template, err := proftest.BaseStartupTmpl.Parse(startupTemplate)
	if err != nil {
		t.Fatalf("failed to parse startup script template: %v", err)
	}

	gceTr := proftest.GCETestRunner{
		TestRunner: proftest.TestRunner{
			Client: client,
		},
		ComputeService: computeService,
	}

	testcases := []testCase{
		// Test GCE Ubuntu default Python 2, should be Python 2.7.
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Zone:         zone,
				Name:         fmt.Sprintf("profiler-test-python2-%s", runID),
				MachineType:  "n1-standard-1",
				ImageProject: "ubuntu-os-cloud",
				ImageFamily:  "ubuntu-1804-lts",
				Scopes:       []string{storageReadScope},
			},
			name: fmt.Sprintf("profiler-test-python2-%s-gce", runID),
			wantProfiles: map[string]string{
				"WALL": "repeat_bench",
				// CPU profiling should be disabled on Python 2.
				"CPU": "",
			},
			pythonCommand: "python2.7",
			versionCheck:  "sys.version_info[:2] == (2, 7)",
		},
		// Test GCE Ubuntu default Python 3, should be Python 3.6 or higher.
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Zone:         zone,
				Name:         fmt.Sprintf("profiler-test-python3-%s", runID),
				MachineType:  "n1-standard-1",
				ImageProject: "ubuntu-os-cloud",
				ImageFamily:  "ubuntu-1804-lts",
				Scopes:       []string{storageReadScope},
			},
			name: fmt.Sprintf("profiler-test-python3-%s-gce", runID),
			wantProfiles: map[string]string{
				"WALL": "repeat_bench",
				"CPU":  "repeat_bench",
			},
			pythonCommand: "python3",
			versionCheck:  "sys.version_info[:2] >= (3, 6)",
		},
		// Test Python 3.5.
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Zone:         zone,
				Name:         fmt.Sprintf("profiler-test-python35-%s", runID),
				MachineType:  "n1-standard-1",
				ImageProject: "ubuntu-os-cloud",
				// ppa:deadsnakes/ppa is not yet available on Ubuntu 18.10.
				ImageFamily: "ubuntu-1604-lts",
				Scopes:      []string{storageReadScope},
			},
			name: fmt.Sprintf("profiler-test-python35-%s-gce", runID),
			wantProfiles: map[string]string{
				"CPU": "repeat_bench",
				// Wall profiling should be disabled on Python 3.5.
				"WALL": "",
			},
			installPythonVersion: "python3.5",
			pythonCommand:        "python3.5",
			versionCheck:         "sys.version_info[:2] == (3, 5)",
		},
	}

	// Allow test cases to run in parallel.
	runtime.GOMAXPROCS(len(testcases))

	for _, tc := range testcases {
		tc := tc // capture range variable
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if err := tc.initializeStartUpScript(template); err != nil {
				t.Fatalf("failed to initialize startup script: %v", err)
			}

			gceTr.StartInstance(ctx, &tc.InstanceConfig)
			defer func() {
				if gceTr.DeleteInstance(ctx, &tc.InstanceConfig); err != nil {
					t.Fatalf("failed to delete instance: %v", err)
				}
			}()

			timeoutCtx, cancel := context.WithTimeout(ctx, time.Minute*20)
			defer cancel()
			if err := gceTr.PollForSerialOutput(timeoutCtx, &tc.InstanceConfig, benchFinishString, errorString); err != nil {
				t.Fatal(err)
			}

			timeNow := time.Now()
			endTime := timeNow.Format(time.RFC3339)
			startTime := timeNow.Add(-1 * time.Hour).Format(time.RFC3339)
			for pType, function := range tc.wantProfiles {
				pr, err := gceTr.TestRunner.QueryProfilesWithZone(tc.ProjectID, tc.name, startTime, endTime, pType, zone)
				if function == "" {
					if err == nil {
						t.Errorf("QueryProfilesWithZone(%s, %s, %s, %s, %s, %s) got profile, want no profile", tc.ProjectID, tc.name, startTime, endTime, pType, zone)
					}
					continue
				}

				if err != nil {
					t.Errorf("QueryProfiles(%s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.name, startTime, endTime, pType, err)
					continue
				}

				if err := pr.HasFunction(function); err != nil {
					t.Errorf("Function %s not found in profiles of type %s: %v", function, pType, err)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package misc

import (
	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/kola/register"
)

func init() {
	register.Register(&register.Test{
		Run:         SelinuxEnforce,
		ClusterSize: 1,
		Name:        "coreos.selinux.enforce",
	})
}

// SelinuxEnforce checks that some basic things work after `setenforce 1`
func SelinuxEnforce(c cluster.TestCluster) {
	m := c.Machines()[0]

	for _, cmd := range []struct {
		cmdline     string
		checkoutput bool
		output      string
	}{
		{"sudo setenforce 1", false, ""},
		{"getenforce", true, "Enforcing"},
		{"systemctl --no-pager is-active system.slice", true, "active"},
	} {
		output, err := m.SSH(cmd.cmdline)
		if err != nil {
			c.Fatalf("failed to run %q: output: %q status: %q", cmd.cmdline, output, err)
		}

		if cmd.checkoutput && string(output) != cmd.output {
			c.Fatalf("command %q has unexpected output: want %q got %q", cmd.cmdline, cmd.output, string(output))
		}
	}
}
<commit_msg>kola/tests/misc: test if selinux survives reboots<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package misc

import (
	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/kola/register"
)

func init() {
	register.Register(&register.Test{
		Run:         SelinuxEnforce,
		ClusterSize: 1,
		Name:        "coreos.selinux.enforce",
		Flags:       []register.Flag{register.NoEnableSelinux},
	})
}

// SelinuxEnforce checks that some basic things work after `setenforce 1`
func SelinuxEnforce(c cluster.TestCluster) {
	m := c.Machines()[0]

	for _, cmd := range []struct {
		cmdline     string
		checkoutput bool
		output      string
	}{
		{"getenforce", true, "Permissive"},
		{"sudo setenforce 1", false, ""},
		{"getenforce", true, "Enforcing"},
		{"systemctl --no-pager is-active system.slice", true, "active"},
		{"sudo cp --remove-destination $(readlink -f /etc/selinux/config) /etc/selinux/config", false, ""},
		{"sudo sed -i 's/SELINUX=permissive/SELINUX=enforcing/' /etc/selinux/config", false, ""},
	} {
		output, err := m.SSH(cmd.cmdline)
		if err != nil {
			c.Fatalf("failed to run %q: output: %q status: %q", cmd.cmdline, output, err)
		}

		if cmd.checkoutput && string(output) != cmd.output {
			c.Fatalf("command %q has unexpected output: want %q got %q", cmd.cmdline, cmd.output, string(output))
		}
	}

	err := m.Reboot()
	if err != nil {
		c.Fatalf("failed to reboot machine: %v", err)
	}

	output, err := m.SSH("getenforce")
	if err != nil {
		c.Fatalf("failed to run \"getenforce\": output: %q status: %q", output, err)
	}

	if string(output) != "Enforcing" {
		c.Fatalf("command \"getenforce\" has unexpected output: want \"Enforcing\" got %q", string(output))
	}
}
<|endoftext|>


<commit_before>//go:build !linux
// +build !linux

package selinux

import (
	"testing"
)

func TestSELinux(t *testing.T) {
	if GetEnabled() {
		t.Fatal("SELinux enabled on non-linux.")
	}

	if _, err := FileLabel("/etc"); err != nil {
		t.Fatal(err)
	}

	if err := SetFileLabel("/etc", "foobar"); err != nil {
		t.Fatal(err)
	}

	if err := SetFSCreateLabel("foobar"); err != nil {
		t.Fatal(err)
	}

	if _, err := FSCreateLabel(); err != nil {
		t.Fatal(err)
	}
	if _, err := CurrentLabel(); err != nil {
		t.Fatal(err)
	}

	if _, err := PidLabel(0); err != nil {
		t.Fatal(err)
	}

	ClearLabels()

	ReserveLabel("foobar")
	ReleaseLabel("foobar")
	_, _ = DupSecOpt("foobar")
	DisableSecOpt()
	SetDisabled()
	if enabled := GetEnabled(); enabled {
		t.Fatal("Should not be enabled")
	}
	if err := SetExecLabel("foobar"); err != nil {
		t.Fatal(err)
	}
	if err := SetTaskLabel("foobar"); err != nil {
		t.Fatal(err)
	}
	if _, err := ExecLabel(); err != nil {
		t.Fatal(err)
	}
	if _, err := CanonicalizeContext("foobar"); err != nil {
		t.Fatal(err)
	}
	if _, err := ComputeCreateContext("foo", "bar", "foobar"); err != nil {
		t.Fatal(err)
	}
	if err := SetSocketLabel("foobar"); err != nil {
		t.Fatal(err)
	}
	if _, err := ClassIndex("foobar"); err != nil {
		t.Fatal(err)
	}
	if _, err := SocketLabel(); err != nil {
		t.Fatal(err)
	}
	if _, err := PeerLabel(0); err != nil {
		t.Fatal(err)
	}
	if err := SetKeyLabel("foobar"); err != nil {
		t.Fatal(err)
	}
	if _, err := KeyLabel(); err != nil {
		t.Fatal(err)
	}
	con, err := NewContext("foobar")
	if err != nil {
		t.Fatal(err)
	}
	con.Get()
	if err := SetEnforceMode(1); err != nil {
		t.Fatal(err)
	}
	DefaultEnforceMode()
	EnforceMode()
	ROFileLabel()
	ContainerLabels()
	if err := SecurityCheckContext("foobar"); err != nil {
		t.Fatal(err)
	}
	if _, err := CopyLevel("foo", "bar"); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>go-selinux: improve stub tests coverage<commit_after>//go:build !linux
// +build !linux

package selinux

import (
	"testing"
)

const testLabel = "foobar"

func TestSELinuxStubs(t *testing.T) {
	if GetEnabled() {
		t.Error("SELinux enabled on non-linux.")
	}

	if _, err := FileLabel("/etc"); err != nil {
		t.Error(err)
	}

	if err := SetFileLabel("/etc", testLabel); err != nil {
		t.Error(err)
	}

	if err := SetFSCreateLabel(testLabel); err != nil {
		t.Error(err)
	}

	if _, err := FSCreateLabel(); err != nil {
		t.Error(err)
	}
	if _, err := CurrentLabel(); err != nil {
		t.Error(err)
	}

	if _, err := PidLabel(0); err != nil {
		t.Error(err)
	}

	ClearLabels()

	ReserveLabel(testLabel)
	ReleaseLabel(testLabel)
	if _, err := DupSecOpt(testLabel); err != nil {
		t.Error(err)
	}
	if v := DisableSecOpt(); len(v) != 1 || v[0] != "disable" {
		t.Errorf(`expected "disabled", got %v`, v)
	}
	SetDisabled()
	if enabled := GetEnabled(); enabled {
		t.Error("Should not be enabled")
	}
	if err := SetExecLabel(testLabel); err != nil {
		t.Error(err)
	}
	if err := SetTaskLabel(testLabel); err != nil {
		t.Error(err)
	}
	if _, err := ExecLabel(); err != nil {
		t.Error(err)
	}
	if _, err := CanonicalizeContext(testLabel); err != nil {
		t.Error(err)
	}
	if _, err := ComputeCreateContext("foo", "bar", testLabel); err != nil {
		t.Error(err)
	}
	if err := SetSocketLabel(testLabel); err != nil {
		t.Error(err)
	}
	if _, err := ClassIndex(testLabel); err != nil {
		t.Error(err)
	}
	if _, err := SocketLabel(); err != nil {
		t.Error(err)
	}
	if _, err := PeerLabel(0); err != nil {
		t.Error(err)
	}
	if err := SetKeyLabel(testLabel); err != nil {
		t.Error(err)
	}
	if _, err := KeyLabel(); err != nil {
		t.Error(err)
	}
	if err := SetExecLabel(testLabel); err != nil {
		t.Error(err)
	}
	if _, err := ExecLabel(); err != nil {
		t.Error(err)
	}
	con, err := NewContext(testLabel)
	if err != nil {
		t.Error(err)
	}
	con.Get()
	if err = SetEnforceMode(1); err != nil {
		t.Error(err)
	}
	if v := DefaultEnforceMode(); v != Disabled {
		t.Errorf("expected %d, got %d", Disabled, v)
	}
	if v := EnforceMode(); v != Disabled {
		t.Errorf("expected %d, got %d", Disabled, v)
	}
	if v := ROFileLabel(); v != "" {
		t.Errorf(`expected "", got %q`, v)
	}
	if processLbl, fileLbl := ContainerLabels(); processLbl != "" || fileLbl != "" {
		t.Errorf(`expected fileLbl="", fileLbl="" got processLbl=%q, fileLbl=%q`, processLbl, fileLbl)
	}
	if err = SecurityCheckContext(testLabel); err != nil {
		t.Error(err)
	}
	if _, err = CopyLevel("foo", "bar"); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"os"
	"path/filepath"
	"sort"
	"strings"

	sysl "github.com/anz-bank/sysl/src/proto"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"gopkg.in/alecthomas/kingpin.v2"
)

const (
	syslRootMarker          = ".sysl"
	noRootMarkerWarning     = "%s is not defined but root flag is defined in %s"
	rootMarkerExistsWarning = "%s found in %s but will use %s instead"
	noRootWarning           = "root and %s are undefined, %s will be used instead"
)

type cmdRunner struct {
	commands map[string]Command

	Root   string
	module string
}

func (r *cmdRunner) Run(which string, fs afero.Fs, logger *logrus.Logger) error {
	if cmd, ok := r.commands[which]; ok {
		if cmd.Name() == which {
			var mod *sysl.Module
			var err error
			if cmd.RequireSyslModule() {
				err = r.getProjectRoot(fs, logger)
				if err != nil {
					return err
				}

				mod, _, err = LoadSyslModule(r.Root, r.module, fs, logger)
				if err != nil {
					return err
				}
			}

			return cmd.Execute(ExecuteArgs{mod, fs, logger})
		}
	}
	return nil
}

func (r *cmdRunner) Configure(app *kingpin.Application) error {
	app.UsageTemplate(kingpin.SeparateOptionalFlagsUsageTemplate)

	commands := []Command{
		&protobuf{},
		&intsCmd{},
		&datamodelCmd{},
		&codegenCmd{},
		&sequenceDiagramCmd{},
		&importCmd{},
		&infoCmd{},
		&validateCmd{},
		&exportCmd{},
	}
	r.commands = map[string]Command{}

	app.Flag("root",
		"sysl root directory for input model file (default: .)").
		Default("").StringVar(&r.Root)

	sort.Slice(commands, func(i, j int) bool {
		return strings.Compare(commands[i].Name(), commands[j].Name()) < 0
	})
	for _, cmd := range commands {
		c := cmd.Configure(app)
		if cmd.RequireSyslModule() {
			c.Arg("MODULE", "input files without .sysl extension and with leading /, eg: "+
				"/project_dir/my_models combine with --root if needed").
				Required().StringVar(&r.module)
		}
		r.commands[cmd.Name()] = cmd
	}

	return nil
}

// Helper function to validate that a set of command flags are not empty values
func EnsureFlagsNonEmpty(cmd *kingpin.CmdClause, excludes ...string) {
	inExcludes := func(s string) bool {
		for _, e := range excludes {
			if s == e {
				return true
			}
		}
		return false
	}
	fn := func(c *kingpin.ParseContext) error {
		var errorMsg strings.Builder
		for _, f := range cmd.Model().Flags {
			if inExcludes(f.Name) {
				continue
			}
			val := f.Value.String()

			if val != "" {
				val = strings.Trim(val, " ")
				if val == "" {
					errorMsg.WriteString("'" + f.Name + "'" + " value passed is empty\n")
				}
			} else if len(f.Default) > 0 {
				errorMsg.WriteString("'" + f.Name + "'" + " value passed is empty\n")
			}
		}
		if errorMsg.Len() > 0 {
			return errors.New(errorMsg.String())
		}
		return nil
	}

	cmd.PreAction(fn)
}

func (r *cmdRunner) getProjectRoot(fs afero.Fs, logger *logrus.Logger) error {
	rootIsDefined := r.Root != ""

	modulePath := r.module
	if rootIsDefined {
		modulePath = filepath.Join(r.Root, r.module)
	}

	syslRootPath, err := findRootFromSyslModule(modulePath, fs)
	if err != nil {
		return err
	}

	rootMarkerExists := syslRootPath != ""

	switch {
	case rootIsDefined:
		if rootMarkerExists {
			logger.Warningf(rootMarkerExistsWarning, syslRootMarker, syslRootPath, r.Root)
		} else {
			logger.Warningf(noRootMarkerWarning, syslRootMarker, r.Root)
		}
	case !rootIsDefined && !rootMarkerExists:
		// uses the module directory as the root, changing the module to be relative to the root
		r.Root = filepath.Dir(r.module)
		r.module = filepath.Base(r.module)
		logger.Warningf(noRootWarning, syslRootMarker, r.Root)
	case !rootIsDefined && rootMarkerExists:
		r.Root = syslRootPath
		absModulePath, err := filepath.Abs(r.module)
		if err != nil {
			return err
		}
		r.module, err = filepath.Rel(r.Root, absModulePath)
		if err != nil {
			return err
		}
	}

	return nil
}

func findRootFromSyslModule(modulePath string, fs afero.Fs) (string, error) {
	// Takes the closest root marker
	currentPath, err := filepath.Abs(modulePath)
	if err != nil {
		return "", err
	}

	systemRoot, err := filepath.Abs(string(os.PathSeparator))
	if err != nil {
		return "", err
	}

	for {
		// Keep walking up the directories
		currentPath = filepath.Dir(currentPath)
		exists, err := afero.Exists(fs, filepath.Join(currentPath, syslRootMarker))
		reachedRoot := currentPath == systemRoot || (err != nil && os.IsPermission(err))
		switch {
		case exists:
			return currentPath, nil
		case reachedRoot:
			return "", nil
		case err != nil:
			return "", err
		}
	}
}
<commit_msg>Changed root flag description<commit_after>package main

import (
	"errors"
	"os"
	"path/filepath"
	"sort"
	"strings"

	sysl "github.com/anz-bank/sysl/src/proto"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"gopkg.in/alecthomas/kingpin.v2"
)

const (
	syslRootMarker          = ".sysl"
	noRootMarkerWarning     = "%s is not defined but root flag is defined in %s"
	rootMarkerExistsWarning = "%s found in %s but will use %s instead"
	noRootWarning           = "root and %s are undefined, %s will be used instead"
)

type cmdRunner struct {
	commands map[string]Command

	Root   string
	module string
}

func (r *cmdRunner) Run(which string, fs afero.Fs, logger *logrus.Logger) error {
	if cmd, ok := r.commands[which]; ok {
		if cmd.Name() == which {
			var mod *sysl.Module
			var err error
			if cmd.RequireSyslModule() {
				err = r.getProjectRoot(fs, logger)
				if err != nil {
					return err
				}

				mod, _, err = LoadSyslModule(r.Root, r.module, fs, logger)
				if err != nil {
					return err
				}
			}

			return cmd.Execute(ExecuteArgs{mod, fs, logger})
		}
	}
	return nil
}

func (r *cmdRunner) Configure(app *kingpin.Application) error {
	app.UsageTemplate(kingpin.SeparateOptionalFlagsUsageTemplate)

	commands := []Command{
		&protobuf{},
		&intsCmd{},
		&datamodelCmd{},
		&codegenCmd{},
		&sequenceDiagramCmd{},
		&importCmd{},
		&infoCmd{},
		&validateCmd{},
		&exportCmd{},
	}
	r.commands = map[string]Command{}

	app.Flag("root",
		"sysl root directory for input model file. If root is not found, the module directory becomes the root").
		Default("").StringVar(&r.Root)

	sort.Slice(commands, func(i, j int) bool {
		return strings.Compare(commands[i].Name(), commands[j].Name()) < 0
	})
	for _, cmd := range commands {
		c := cmd.Configure(app)
		if cmd.RequireSyslModule() {
			c.Arg("MODULE", "input files without .sysl extension and with leading /, eg: "+
				"/project_dir/my_models combine with --root if needed").
				Required().StringVar(&r.module)
		}
		r.commands[cmd.Name()] = cmd
	}

	return nil
}

// Helper function to validate that a set of command flags are not empty values
func EnsureFlagsNonEmpty(cmd *kingpin.CmdClause, excludes ...string) {
	inExcludes := func(s string) bool {
		for _, e := range excludes {
			if s == e {
				return true
			}
		}
		return false
	}
	fn := func(c *kingpin.ParseContext) error {
		var errorMsg strings.Builder
		for _, f := range cmd.Model().Flags {
			if inExcludes(f.Name) {
				continue
			}
			val := f.Value.String()

			if val != "" {
				val = strings.Trim(val, " ")
				if val == "" {
					errorMsg.WriteString("'" + f.Name + "'" + " value passed is empty\n")
				}
			} else if len(f.Default) > 0 {
				errorMsg.WriteString("'" + f.Name + "'" + " value passed is empty\n")
			}
		}
		if errorMsg.Len() > 0 {
			return errors.New(errorMsg.String())
		}
		return nil
	}

	cmd.PreAction(fn)
}

func (r *cmdRunner) getProjectRoot(fs afero.Fs, logger *logrus.Logger) error {
	rootIsDefined := r.Root != ""

	modulePath := r.module
	if rootIsDefined {
		modulePath = filepath.Join(r.Root, r.module)
	}

	syslRootPath, err := findRootFromSyslModule(modulePath, fs)
	if err != nil {
		return err
	}

	rootMarkerExists := syslRootPath != ""

	switch {
	case rootIsDefined:
		if rootMarkerExists {
			logger.Warningf(rootMarkerExistsWarning, syslRootMarker, syslRootPath, r.Root)
		} else {
			logger.Warningf(noRootMarkerWarning, syslRootMarker, r.Root)
		}
	case !rootIsDefined && !rootMarkerExists:
		// uses the module directory as the root, changing the module to be relative to the root
		r.Root = filepath.Dir(r.module)
		r.module = filepath.Base(r.module)
		logger.Warningf(noRootWarning, syslRootMarker, r.Root)
	case !rootIsDefined && rootMarkerExists:
		r.Root = syslRootPath
		absModulePath, err := filepath.Abs(r.module)
		if err != nil {
			return err
		}
		r.module, err = filepath.Rel(r.Root, absModulePath)
		if err != nil {
			return err
		}
	}

	return nil
}

func findRootFromSyslModule(modulePath string, fs afero.Fs) (string, error) {
	// Takes the closest root marker
	currentPath, err := filepath.Abs(modulePath)
	if err != nil {
		return "", err
	}

	systemRoot, err := filepath.Abs(string(os.PathSeparator))
	if err != nil {
		return "", err
	}

	for {
		// Keep walking up the directories
		currentPath = filepath.Dir(currentPath)
		exists, err := afero.Exists(fs, filepath.Join(currentPath, syslRootMarker))
		reachedRoot := currentPath == systemRoot || (err != nil && os.IsPermission(err))
		switch {
		case exists:
			return currentPath, nil
		case reachedRoot:
			return "", nil
		case err != nil:
			return "", err
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	// We use our own fork to get the connection termination behaviour
	// we need.
	// "net/http/cgi"
	"github.com/scraperwiki/cobalt/go/daemons/cgi-endpoint/go/cgi"

	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/stretchr/graceful"
)

var (
	cobaltHome   = "/var/lib/cobalt/home" // Location of boxes outside chroot
	boxHome      = "/home"                // Location of $HOME inside one chroot
	globalCGI    = "/tools/global-cgi"    // Location of global CGI scripts
	checkToken   = "http://localhost:23423"
	inProduction = false // Running in production
)

func init() {
	if os.Getenv("COBALT_HOME") != "" {
		cobaltHome = os.Getenv("COBALT_HOME")
	}
	if os.Getenv("COBALT_BOX_HOME") != "" {
		boxHome = os.Getenv("COBALT_BOX_HOME")
	}
	if os.Getenv("COBALT_GLOBAL_CGI") != "" {
		globalCGI = os.Getenv("COBALT_GLOBAL_CGI")
	}
	if os.Getenv("COBALT_CHECKTOKEN") != "" {
		checkToken = os.Getenv("COBALT_CHECKTOKEN")
	}

	inProduction = os.Getenv("SCRAPERWIKI_ENV") == "production"
}

// Split the string into the bit which identifies the box and task
// ({boxname}, {publishToken}, {task e.g cgi-bin}) and (script path to invoke)
func GetTarget(r *http.Request) (prefix, target string) {
	result := strings.SplitN(r.URL.Path, "/", 5)
	if len(result) != 5 {
		log.Panic("Request URI not of the right form: %q", r.URL.RequestURI())
	}
	prefix = strings.Join(result[:4], "/")
	target = "/" + result[4]
	return
}

func HandleHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	user := vars["box"]

	prefix, _ := GetTarget(r)

	root := http.Dir(path.Join(cobaltHome, user, "http"))
	staticHandler := http.StripPrefix(prefix, http.FileServer(root))
	staticHandler.ServeHTTP(w, r)
}

// Note: this assumes running in a context with the UID or GID of the
// user/group of the file. This is good enough for us.
// In an ideal world the file would just be invoked, but it's not possible
// to do this here, since we don't want to run the whole CGI handler repeatedly
// for every possible file, since it's rather expensive.
func isExecutable(mode os.FileMode) bool {
	return int(mode&0111) != 0
}

// Can we execute this as a CGI handler?
// ("Does it exist for our purposes")
func Exists(path string) bool {
	// log.Println("Tried", path)
	s, err := os.Stat(path)
	if err == nil && !s.IsDir() {
		return isExecutable(s.Mode())
	}
	return false
}

// Locate `target` in `users` box, returning the resulting path and true if
// found. It also searches for matching default scripts, and in the
// /tool/cgi-bin directory, and in the /tool/globals directory.
func FindCgiScript(user, target string) (fullpath, uri string, ok bool) {
	var thisOutsideChrootHome = path.Join(cobaltHome, user)

	// Path to $HOME inside box doesn't contain $USER in production.
	var thisBoxHome = boxHome
	if !inProduction {
		// If we're in a production environment, then we find home at `boxHome`,
		// otherwise it's at `{boxHome}/{username}`.
		thisBoxHome = path.Join(boxHome, user)
	}

	// Search for a script at {root}/{target}, then look for matching scripts
	// living in directories named `default`.
	// Returns the path to the script to invoke relative to `root` when found,
	// which may be a script named `default`.
	lookForScript := func(root, target string) (uri string, ok bool) {
		ok = Exists(path.Join(root, target))
		if ok {
			return target, true
		}
		for target != "/" {
			uri := path.Join(target, "default")
			ok = Exists(path.Join(root, uri))
			if ok {
				return uri, true
			}
			target = path.Dir(target)
		}
		return "", false
	}

	roots := []struct {
		outsideChrootBase, base, place string
	}{
		{thisOutsideChrootHome, thisBoxHome, "cgi-bin"},
		{thisOutsideChrootHome, thisBoxHome, "tool/cgi-bin"},
		{globalCGI, globalCGI, "cgi-bin"},
	}

	for _, root := range roots {
		uri, ok := lookForScript(path.Join(root.outsideChrootBase, root.place), target)
		if ok {
			return path.Join(root.base, root.place, uri), uri, ok
		}
	}

	return "", "", false
}

func HandleCGI(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	user := vars["box"]
	prefix, target := GetTarget(r)

	// Prefix contains /{boxname}/{boxToken}
	// scriptUri contains /cgi-bin/foo.

	scriptPath, scriptUri, ok := FindCgiScript(user, target)
	if !ok {
		http.NotFound(w, r)
		return
	}
	// log.Printf("%v = FindCgiScript(%v) -> %v", ok, target, scriptPath)

	var cgipath = ""
	var cgiargs = []string{}

	// We have to use shell because we're using su to become the right user.
	// We also can't specify these varibles directly because they're
	// overwritten by the CGI handler.
	const code = `
		# SCRIPT_NAME is the URI to the script itself (may be equal to the
		# location of "default")
		export SCRIPT_NAME="$1"; shift
		# These two are the full path of the script being invoked
		export SCRIPT_FILENAME="$1"
		export SCRIPT_PATH="$1"
		cd "$(dirname "$SCRIPT_PATH")"
		exec "$@"
	`

	args := []string{path.Join(prefix, scriptUri), scriptPath}

	if inProduction {
		cgipath = "/bin/su"
		cgiargs = append([]string{"-c", code, user, "--", "-sh"}, args...)
	} else {
		// sh doesn't take `user`, nor `$0`.
		cgipath = "/bin/sh"
		cgiargs = append([]string{"-c", code, "--"}, args...)
	}

	// on Dir: In the usual case where we're su'ing into a box, setting Dir has
	// no effect because the PAM chroot module changes the current directory
	// (to be / in the box's chrooted environment).
	// on $HOME: The cgi module only sets certain environment variables, and
	// leaves HOME unset. We set the directory within the command invoked.
	handler := &cgi.Handler{
		Path: cgipath,
		Args: cgiargs,
		Env: []string{
			"SERVER_SOFTWARE=github.com/scraperwiki/cobalt/go/daemons/cgi-endpoint",
		}}

	handler.ServeHTTP(w, r)
}

func Listen(host, port string) (l net.Listener, err error) {
	if len(port) == 0 {
		err = fmt.Errorf("Bad listen address, please set PORT and optionally HOST")
		return
	}
	if host == "unix" {
		l, err = net.Listen("unix", port)
	} else {
		addr := host + ":" + port
		l, err = net.Listen("tcp", addr)
	}
	return
}

func tokenVerifier(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	vars := mux.Vars(r)

	if checkToken == "off" {
		next(rw, r)
		return
	}

	endpoint := checkToken + "/" + path.Join(vars["box"], vars["publishToken"])
	resp, err := http.Get(endpoint)
	if err != nil {
		log.Println("Unable to access", endpoint, "err =", err)
		http.Error(rw, "503 Service Unavailable", http.StatusServiceUnavailable)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		rw.WriteHeader(resp.StatusCode)
		// Discard error.
		_, _ = io.Copy(rw, resp.Body)
		return
	}

	next(rw, r)
}

// Wrap the given handler with a token verifier
func WrapTokenVerifier(handler http.Handler) http.Handler {

	middleware := negroni.New()
	middleware.Use(negroni.HandlerFunc(tokenVerifier))
	middleware.UseHandler(handler)

	top := mux.NewRouter()
	top.PathPrefix("/{box}/{publishToken}/").Handler(middleware)

	return top
}

func NewHandler() http.Handler {

	box := mux.NewRouter().PathPrefix("/{box}/{publishToken}/").Subrouter()

	box.PathPrefix("/cgi-bin/").HandlerFunc(HandleCGI)
	box.PathPrefix("/http/").HandlerFunc(HandleHTTP)

	n := negroni.Classic()
	n.Use(gzip.Gzip(1))
	n.UseHandler(WrapTokenVerifier(box))

	return n
}

func main() {

	log.Println("COBALT_HOME =", cobaltHome)
	log.Println("COBALT_BOX_HOME =", boxHome)
	log.Println("COBALT_GLOBAL_CGI =", globalCGI)
	log.Println("COBALT_CHECKTOKEN =", checkToken)
	log.Println("Production Environment =", inProduction)

	l, err := Listen(os.Getenv("HOST"), os.Getenv("PORT"))
	if err != nil {
		log.Fatalln("Error listening:", err)
	}
	defer l.Close()

	log.Printf("Listening on %s:%s", os.Getenv("HOST"), os.Getenv("PORT"))

	s := &http.Server{Handler: NewHandler()}

	// Graceful shutdown servers immediately stop listening on CTRL-C, but give
	// ongoing connections a  chance to finish before terminating.
	const gracefulShutdownTime = 5 * time.Second
	graceful.Serve(s, l, gracefulShutdownTime)
}
<commit_msg>Fix for environment vulnerability<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	// We use our own fork to get the connection termination behaviour
	// we need.
	// "net/http/cgi"
	"github.com/scraperwiki/cobalt/go/daemons/cgi-endpoint/go/cgi"

	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/stretchr/graceful"
)

var (
	cobaltHome   = "/var/lib/cobalt/home" // Location of boxes outside chroot
	boxHome      = "/home"                // Location of $HOME inside one chroot
	globalCGI    = "/tools/global-cgi"    // Location of global CGI scripts
	checkToken   = "http://localhost:23423"
	inProduction = false // Running in production
)

func init() {
	if os.Getenv("COBALT_HOME") != "" {
		cobaltHome = os.Getenv("COBALT_HOME")
	}
	if os.Getenv("COBALT_BOX_HOME") != "" {
		boxHome = os.Getenv("COBALT_BOX_HOME")
	}
	if os.Getenv("COBALT_GLOBAL_CGI") != "" {
		globalCGI = os.Getenv("COBALT_GLOBAL_CGI")
	}
	if os.Getenv("COBALT_CHECKTOKEN") != "" {
		checkToken = os.Getenv("COBALT_CHECKTOKEN")
	}

	inProduction = os.Getenv("SCRAPERWIKI_ENV") == "production"
}

// Split the string into the bit which identifies the box and task
// ({boxname}, {publishToken}, {task e.g cgi-bin}) and (script path to invoke)
func GetTarget(r *http.Request) (prefix, target string) {
	result := strings.SplitN(r.URL.Path, "/", 5)
	if len(result) != 5 {
		log.Panic("Request URI not of the right form: %q", r.URL.RequestURI())
	}
	prefix = strings.Join(result[:4], "/")
	target = "/" + result[4]
	return
}

func HandleHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	user := vars["box"]

	prefix, _ := GetTarget(r)

	root := http.Dir(path.Join(cobaltHome, user, "http"))
	staticHandler := http.StripPrefix(prefix, http.FileServer(root))
	staticHandler.ServeHTTP(w, r)
}

// Note: this assumes running in a context with the UID or GID of the
// user/group of the file. This is good enough for us.
// In an ideal world the file would just be invoked, but it's not possible
// to do this here, since we don't want to run the whole CGI handler repeatedly
// for every possible file, since it's rather expensive.
func isExecutable(mode os.FileMode) bool {
	return int(mode&0111) != 0
}

// Can we execute this as a CGI handler?
// ("Does it exist for our purposes")
func Exists(path string) bool {
	// log.Println("Tried", path)
	s, err := os.Stat(path)
	if err == nil && !s.IsDir() {
		return isExecutable(s.Mode())
	}
	return false
}

// Locate `target` in `users` box, returning the resulting path and true if
// found. It also searches for matching default scripts, and in the
// /tool/cgi-bin directory, and in the /tool/globals directory.
func FindCgiScript(user, target string) (fullpath, uri string, ok bool) {
	var thisOutsideChrootHome = path.Join(cobaltHome, user)

	// Path to $HOME inside box doesn't contain $USER in production.
	var thisBoxHome = boxHome
	if !inProduction {
		// If we're in a production environment, then we find home at `boxHome`,
		// otherwise it's at `{boxHome}/{username}`.
		thisBoxHome = path.Join(boxHome, user)
	}

	// Search for a script at {root}/{target}, then look for matching scripts
	// living in directories named `default`.
	// Returns the path to the script to invoke relative to `root` when found,
	// which may be a script named `default`.
	lookForScript := func(root, target string) (uri string, ok bool) {
		ok = Exists(path.Join(root, target))
		if ok {
			return target, true
		}
		for target != "/" {
			uri := path.Join(target, "default")
			ok = Exists(path.Join(root, uri))
			if ok {
				return uri, true
			}
			target = path.Dir(target)
		}
		return "", false
	}

	roots := []struct {
		outsideChrootBase, base, place string
	}{
		{thisOutsideChrootHome, thisBoxHome, "cgi-bin"},
		{thisOutsideChrootHome, thisBoxHome, "tool/cgi-bin"},
		{globalCGI, globalCGI, "cgi-bin"},
	}

	for _, root := range roots {
		uri, ok := lookForScript(path.Join(root.outsideChrootBase, root.place), target)
		if ok {
			return path.Join(root.base, root.place, uri), uri, ok
		}
	}

	return "", "", false
}

func HandleCGI(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	user := vars["box"]
	prefix, target := GetTarget(r)

	// Prefix contains /{boxname}/{boxToken}
	// scriptUri contains /cgi-bin/foo.

	scriptPath, scriptUri, ok := FindCgiScript(user, target)
	if !ok {
		http.NotFound(w, r)
		return
	}
	// log.Printf("%v = FindCgiScript(%v) -> %v", ok, target, scriptPath)

	var cgipath = ""
	var cgiargs = []string{}

	// We have to use shell because we're using su to become the right user.
	// We also can't specify these varibles directly because they're
	// overwritten by the CGI handler.
	const code = `
		# SCRIPT_NAME is the URI to the script itself (may be equal to the
		# location of "default")
		export SCRIPT_NAME="$1"; shift
		# These two are the full path of the script being invoked
		export SCRIPT_FILENAME="$1"
		export SCRIPT_PATH="$1"
		cd "$(dirname "$SCRIPT_PATH")"
		exec "$@"
	`

	args := []string{path.Join(prefix, scriptUri), scriptPath}

	if inProduction {
		cgipath = "/bin/su"
		cgiargs = append([]string{"--shell=/bin/sh", "-c", code, user, "--", "-sh"}, args...)
	} else {
		// sh doesn't take `user`, nor `$0`.
		cgipath = "/bin/sh"
		cgiargs = append([]string{"-c", code, "--"}, args...)
	}

	// on Dir: In the usual case where we're su'ing into a box, setting Dir has
	// no effect because the PAM chroot module changes the current directory
	// (to be / in the box's chrooted environment).
	// on $HOME: The cgi module only sets certain environment variables, and
	// leaves HOME unset. We set the directory within the command invoked.
	handler := &cgi.Handler{
		Path: cgipath,
		Args: cgiargs,
		Env: []string{
			"SERVER_SOFTWARE=github.com/scraperwiki/cobalt/go/daemons/cgi-endpoint",
		}}

	handler.ServeHTTP(w, r)
}

func Listen(host, port string) (l net.Listener, err error) {
	if len(port) == 0 {
		err = fmt.Errorf("Bad listen address, please set PORT and optionally HOST")
		return
	}
	if host == "unix" {
		l, err = net.Listen("unix", port)
	} else {
		addr := host + ":" + port
		l, err = net.Listen("tcp", addr)
	}
	return
}

func tokenVerifier(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	vars := mux.Vars(r)

	if checkToken == "off" {
		next(rw, r)
		return
	}

	endpoint := checkToken + "/" + path.Join(vars["box"], vars["publishToken"])
	resp, err := http.Get(endpoint)
	if err != nil {
		log.Println("Unable to access", endpoint, "err =", err)
		http.Error(rw, "503 Service Unavailable", http.StatusServiceUnavailable)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		rw.WriteHeader(resp.StatusCode)
		// Discard error.
		_, _ = io.Copy(rw, resp.Body)
		return
	}

	next(rw, r)
}

// Wrap the given handler with a token verifier
func WrapTokenVerifier(handler http.Handler) http.Handler {

	middleware := negroni.New()
	middleware.Use(negroni.HandlerFunc(tokenVerifier))
	middleware.UseHandler(handler)

	top := mux.NewRouter()
	top.PathPrefix("/{box}/{publishToken}/").Handler(middleware)

	return top
}

func NewHandler() http.Handler {

	box := mux.NewRouter().PathPrefix("/{box}/{publishToken}/").Subrouter()

	box.PathPrefix("/cgi-bin/").HandlerFunc(HandleCGI)
	box.PathPrefix("/http/").HandlerFunc(HandleHTTP)

	n := negroni.Classic()
	n.Use(gzip.Gzip(1))
	n.UseHandler(WrapTokenVerifier(box))

	return n
}

func main() {

	log.Println("COBALT_HOME =", cobaltHome)
	log.Println("COBALT_BOX_HOME =", boxHome)
	log.Println("COBALT_GLOBAL_CGI =", globalCGI)
	log.Println("COBALT_CHECKTOKEN =", checkToken)
	log.Println("Production Environment =", inProduction)

	l, err := Listen(os.Getenv("HOST"), os.Getenv("PORT"))
	if err != nil {
		log.Fatalln("Error listening:", err)
	}
	defer l.Close()

	log.Printf("Listening on %s:%s", os.Getenv("HOST"), os.Getenv("PORT"))

	s := &http.Server{Handler: NewHandler()}

	// Graceful shutdown servers immediately stop listening on CTRL-C, but give
	// ongoing connections a  chance to finish before terminating.
	const gracefulShutdownTime = 5 * time.Second
	graceful.Serve(s, l, gracefulShutdownTime)
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"io"
	"io/ioutil"
	"strings"
)

//MaxContentSize represent max allowed expandable content size
var MaxContentSize = 1024 * 64

func canExpand(content []byte) bool {
	if len(content) == 0 {
		return false
	}
	limit := 100
	if limit >= len(content) {
		limit = len(content) - 1
	}
	return toolbox.IsPrintText(string(content[:limit]))
}

//NewExpandedContentHandler return a new reader that can substitute content with state map, replacement data provided in replacement map.
func NewExpandedContentHandler(context *endly.Context, replaceMap map[string]string, expand bool) func(reader io.ReadCloser) (io.ReadCloser, error) {
	return func(reader io.ReadCloser) (io.ReadCloser, error) {
		var replaced = false
		defer reader.Close()
		content, err := ioutil.ReadAll(reader)
		if err != nil {
			return nil, err
		}
		if len(content) > MaxContentSize {
			return ioutil.NopCloser(bytes.NewReader(content)), nil
		}

		var result = string(content)
		if expand && canExpand(content) {
			result = context.Expand(result)
			if err != nil {
				return nil, err
			}
			replaced = len(result) != len(content)
		}

		for k, v := range replaceMap {
			if !replaced && strings.Contains(result, k) {
				replaced = true
			}
			result = strings.Replace(result, k, v, len(result))
		}
		if replaced {
			return ioutil.NopCloser(strings.NewReader(toolbox.AsString(result))), nil
		}
		return ioutil.NopCloser(bytes.NewReader(content)), nil
	}
}

//UseMemoryService sets flag on context to always use memory service (testing only)
func UseMemoryService(context *endly.Context) storage.Service {
	state := context.State()
	state.Put(useMemoryService, true)
	return storage.NewMemoryService()
}

//GetStorageService return toolbox storage service
func GetStorageService(context *endly.Context, resource *url.Resource) (storage.Service, error) {
	var state = context.State()
	if state.Has(useMemoryService) {
		return storage.NewMemoryService(), nil
	}
	if resource.Credentials != "" {
		var err error
		if resource.Credentials, err = context.Secrets.CredentialsLocation(resource.Credentials); err != nil {
			return nil, err
		}
	}
	return storage.NewServiceForURL(resource.URL, resource.Credentials)
}

//IsShellCompressable returns true if resource can be compress via shell command.
func IsShellCompressable(protScheme string) bool {
	return protScheme == "scp" || protScheme == "file" || protScheme == "ssh"
}

//Copy transfers data for provided transfer definition.
func Copy(context *endly.Context, transfers ...*Transfer) (interface{}, error) {
	if transfers == nil {
		return nil, nil
	}
	transferService, err := context.Service(ServiceID)
	if err != nil {
		return nil, err
	}
	response := transferService.Run(context, &CopyRequest{Transfers: transfers})
	if response.Err != nil {
		return nil, response.Err
	}
	return nil, nil
}

func joinIfNeeded(parent *url.Resource, URI string) (result *url.Resource) {
	defer func() {
		if parent != nil {
			result.Credentials = parent.Credentials
		}
	}()

	if strings.Contains(URI, ":/") {
		result = url.NewResource(URI)
	} else if !(strings.HasPrefix(URI, "/") || strings.HasPrefix(URI, "$")) {
		var hostname = parent.ParsedURL.Hostname()
		if hostname == "" || hostname == "127.0.0.1" || hostname == "localhost" {
			var candidate = url.NewResource(URI)
			if toolbox.FileExists(candidate.ParsedURL.Path) {
				result = candidate
			}
		}
		if result == nil {
			result = url.NewResource(toolbox.URLPathJoin(parent.URL, URI))
		}

	} else if parent != nil {
		result = url.NewResource(toolbox.URLPathJoin(parent.URL, URI))
	} else {
		result = url.NewResource(URI)
	}
	if strings.HasPrefix(URI, "$") { //has to expand to be re-evaluated
		result.URL = URI
	}
	return result
}
<commit_msg>reformatted<commit_after>package storage

import (
	"bytes"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"io"
	"io/ioutil"
	"strings"
)

//MaxContentSize represent max allowed expandable content size
var MaxContentSize = 1024 * 64

func canExpand(content []byte) bool {
	if len(content) == 0 {
		return false
	}
	limit := 100
	if limit >= len(content) {
		limit = len(content) - 1
	}
	return toolbox.IsPrintText(string(content[:limit]))
}

//NewExpandedContentHandler return a new reader that can substitute content with state map, replacement data provided in replacement map.
func NewExpandedContentHandler(context *endly.Context, replaceMap map[string]string, expand bool) func(reader io.ReadCloser) (io.ReadCloser, error) {
	return func(reader io.ReadCloser) (io.ReadCloser, error) {
		var replaced = false
		defer reader.Close()
		content, err := ioutil.ReadAll(reader)
		if err != nil {
			return nil, err
		}
		if len(content) > MaxContentSize {
			return ioutil.NopCloser(bytes.NewReader(content)), nil
		}

		var result = string(content)
		if expand && canExpand(content) {
			result = context.Expand(result)
			if err != nil {
				return nil, err
			}
			replaced = len(result) != len(content)
		}

		for k, v := range replaceMap {
			if !replaced && strings.Contains(result, k) {
				replaced = true
			}
			result = strings.Replace(result, k, v, len(result))
		}
		if replaced {
			return ioutil.NopCloser(strings.NewReader(toolbox.AsString(result))), nil
		}
		return ioutil.NopCloser(bytes.NewReader(content)), nil
	}
}

//UseMemoryService sets flag on context to always use memory service (testing only)
func UseMemoryService(context *endly.Context) storage.Service {
	state := context.State()
	state.Put(useMemoryService, true)
	return storage.NewMemoryService()
}

//GetStorageService return toolbox storage service
func GetStorageService(context *endly.Context, resource *url.Resource) (storage.Service, error) {
	var state = context.State()
	if state.Has(useMemoryService) {
		return storage.NewMemoryService(), nil
	}
	if resource.Credentials != "" {
		var err error
		if resource.Credentials, err = context.Secrets.CredentialsLocation(resource.Credentials); err != nil {
			return nil, err
		}
	}
	return storage.NewServiceForURL(resource.URL, resource.Credentials)
}

//IsShellCompressable returns true if resource can be compress via shell command.
func IsShellCompressable(protScheme string) bool {
	return protScheme == "scp" || protScheme == "file" || protScheme == "ssh"
}

//Copy transfers data for provided transfer definition.
func Copy(context *endly.Context, transfers ...*Transfer) (interface{}, error) {
	if transfers == nil {
		return nil, nil
	}
	transferService, err := context.Service(ServiceID)
	if err != nil {
		return nil, err
	}
	response := transferService.Run(context, &CopyRequest{Transfers: transfers})
	if response.Err != nil {
		return nil, response.Err
	}
	return nil, nil
}

func joinIfNeeded(parent *url.Resource, URI string) (result *url.Resource) {
	defer func() {
		if parent != nil {
			result.Credentials = parent.Credentials
		}
	}()

	if strings.Contains(URI, ":/") {
		result = url.NewResource(URI)
	} else if !(strings.HasPrefix(URI, "/") || strings.HasPrefix(URI, "$")) {
		var hostname = parent.ParsedURL.Hostname()
		if hostname == "" || hostname == "127.0.0.1" || hostname == "localhost" {
			var candidate = url.NewResource(URI)
			if toolbox.FileExists(candidate.ParsedURL.Path) {
				result = candidate
			}
		}
		if result == nil {
			result = url.NewResource(toolbox.URLPathJoin(parent.URL, URI))
		}

	} else if parent != nil {
		result = url.NewResource(toolbox.URLPathJoin(parent.URL, URI))
	} else {
		result = url.NewResource(URI)
	}
	if strings.HasPrefix(URI, "$") { //has to expand to be re-evaluated
		result.URL = URI
	}
	return result
}
<|endoftext|>


<commit_before>package eventexporter

import kodingmetrics "github.com/koding/metrics"

type DatadogExporter struct {
	datadog *kodingmetrics.DogStatsD
}

// NewDatadogExporter initializes DatadogExporter struct
// and NewDatadogExporter implements Exporter interface with Send and Close functions
func NewDatadogExporter(d *kodingmetrics.DogStatsD) *DatadogExporter {
	return &DatadogExporter{datadog: d}
}

func (d *DatadogExporter) Send(m *Event) error {
	eventName, tags := eventSeperator(m)

	return d.datadog.Count(eventName, 1, tags, 1)
}

func (d *DatadogExporter) Close() error {
	return nil
}
<commit_msg>datadog: ignore errors because we are trying to write on localhost<commit_after>package eventexporter

import kodingmetrics "github.com/koding/metrics"

type DatadogExporter struct {
	datadog *kodingmetrics.DogStatsD
}

// NewDatadogExporter initializes DatadogExporter struct
// and NewDatadogExporter implements Exporter interface with Send and Close functions
func NewDatadogExporter(d *kodingmetrics.DogStatsD) *DatadogExporter {
	return &DatadogExporter{datadog: d}
}

func (d *DatadogExporter) Send(m *Event) error {
	eventName, tags := eventSeperator(m)
	_ = d.datadog.Count(eventName, 1, tags, 1)
	return nil
}

func (d *DatadogExporter) Close() error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dashboard

// This file handles the front page.

import (
	"bytes"
	"html/template"
	"io"
	"net/http"
	"sync"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/user"
)

func init() {
	http.HandleFunc("/", handleFront)
	http.HandleFunc("/favicon.ico", http.NotFound)
}

func handleFront(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	data := &frontPageData{
		Reviewers: personList,
		User:      user.Current(c).Email,
		IsAdmin:   user.IsAdmin(c),
	}
	var currentPerson string
	currentPerson, data.UserIsReviewer = emailToPerson[data.User]

	var wg sync.WaitGroup
	errc := make(chan error, 10)
	activeCLs := datastore.NewQuery("CL").
		Filter("Closed =", false).
		Order("-Modified")

	tableFetch := func(index int, f func(tbl *clTable) error) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			start := time.Now()
			if err := f(&data.Tables[index]); err != nil {
				errc <- err
			}
			data.Timing[index] = time.Now().Sub(start)
		}()
	}

	if data.UserIsReviewer {
		tableFetch(0, func(tbl *clTable) error {
			q := activeCLs.Filter("Reviewer =", currentPerson).Limit(10)
			tbl.Title = "CLs assigned to you for review"
			tbl.Assignable = true
			_, err := q.GetAll(c, &tbl.CLs)
			return err
		})
	}

	tableFetch(1, func(tbl *clTable) error {
		q := activeCLs.Filter("Author =", currentPerson).Limit(10)
		tbl.Title = "CLs sent by you"
		tbl.Assignable = true
		_, err := q.GetAll(c, &tbl.CLs)
		return err
	})

	tableFetch(2, func(tbl *clTable) error {
		q := activeCLs.Limit(50)
		tbl.Title = "Other active CLs"
		tbl.Assignable = true
		if _, err := q.GetAll(c, &tbl.CLs); err != nil {
			return err
		}
		// filter
		if data.UserIsReviewer {
			for i := len(tbl.CLs) - 1; i >= 0; i-- {
				cl := tbl.CLs[i]
				if cl.Author == currentPerson || cl.Reviewer == currentPerson {
					// Preserve order.
					copy(tbl.CLs[i:], tbl.CLs[i+1:])
					tbl.CLs = tbl.CLs[:len(tbl.CLs)-1]
				}
			}
		}
		return nil
	})

	tableFetch(3, func(tbl *clTable) error {
		q := datastore.NewQuery("CL").
			Filter("Closed =", true).
			Order("-Modified").
			Limit(10)
		tbl.Title = "Recently closed CLs"
		tbl.Assignable = false
		_, err := q.GetAll(c, &tbl.CLs)
		return err
	})

	// Not really a table fetch.
	tableFetch(0, func(_ *clTable) error {
		var err error
		data.LogoutURL, err = user.LogoutURL(c, "/")
		return err
	})

	wg.Wait()

	select {
	case err := <-errc:
		c.Errorf("%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	default:
	}

	var b bytes.Buffer
	if err := frontPage.ExecuteTemplate(&b, "front", &data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	io.Copy(w, &b)
}

type frontPageData struct {
	Tables [4]clTable
	Timing [4]time.Duration

	Reviewers      []string
	UserIsReviewer bool

	User, LogoutURL string
	IsAdmin         bool
}

type clTable struct {
	Title      string
	Assignable bool
	CLs        []*CL
}

var frontPage = template.Must(template.New("front").Funcs(template.FuncMap{
	"selected": func(a, b string) string {
		if a == b {
			return "selected"
		}
		return ""
	},
}).Parse(`
<!doctype html>

    <title>Go code reviews</title>
    <link rel="icon" type="image/png" href="/static/icon.png" />

Go code reviews

{{range $tbl := .Tables}}

{{$tbl.Title}}

{{if .CLs}}

{{range $cl := .CLs}}

    {{$cl.DisplayOwner}}
    {{if $tbl.Assignable}}

    <select id="cl-rev-{{$cl.Number}}" {{if not $.UserIsReviewer}}disabled{{end}}>
      <option></option>
      {{range $.Reviewers}}
      <option {{selected . $cl.Reviewer}}>{{.}}</option>
      {{end}}
    </select>

    {{end}}

      {{.Number}}: {{.FirstLineHTML}}
      {{if and .LGTMs $tbl.Assignable}}
LGTMs: {{.LGTMHTML}}{{end}}
      {{if and .NotLGTMs $tbl.Assignable}}
NOT LGTMs: {{.NotLGTMHTML}}{{end}}

    {{.ModifiedAgo}}
    {{if $.IsAdmin}}⟳	{{end}}

{{end}}

{{else}}
none
{{end}}
{{end}}

<address>
You are {{.User}} · logout

datastore timing: {{range .Timing}} {{.}}{{end}}
</address>

`))
<commit_msg>misc/dashboard/codereview: fix tag nesting.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dashboard

// This file handles the front page.

import (
	"bytes"
	"html/template"
	"io"
	"net/http"
	"sync"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/user"
)

func init() {
	http.HandleFunc("/", handleFront)
	http.HandleFunc("/favicon.ico", http.NotFound)
}

func handleFront(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	data := &frontPageData{
		Reviewers: personList,
		User:      user.Current(c).Email,
		IsAdmin:   user.IsAdmin(c),
	}
	var currentPerson string
	currentPerson, data.UserIsReviewer = emailToPerson[data.User]

	var wg sync.WaitGroup
	errc := make(chan error, 10)
	activeCLs := datastore.NewQuery("CL").
		Filter("Closed =", false).
		Order("-Modified")

	tableFetch := func(index int, f func(tbl *clTable) error) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			start := time.Now()
			if err := f(&data.Tables[index]); err != nil {
				errc <- err
			}
			data.Timing[index] = time.Now().Sub(start)
		}()
	}

	if data.UserIsReviewer {
		tableFetch(0, func(tbl *clTable) error {
			q := activeCLs.Filter("Reviewer =", currentPerson).Limit(10)
			tbl.Title = "CLs assigned to you for review"
			tbl.Assignable = true
			_, err := q.GetAll(c, &tbl.CLs)
			return err
		})
	}

	tableFetch(1, func(tbl *clTable) error {
		q := activeCLs.Filter("Author =", currentPerson).Limit(10)
		tbl.Title = "CLs sent by you"
		tbl.Assignable = true
		_, err := q.GetAll(c, &tbl.CLs)
		return err
	})

	tableFetch(2, func(tbl *clTable) error {
		q := activeCLs.Limit(50)
		tbl.Title = "Other active CLs"
		tbl.Assignable = true
		if _, err := q.GetAll(c, &tbl.CLs); err != nil {
			return err
		}
		// filter
		if data.UserIsReviewer {
			for i := len(tbl.CLs) - 1; i >= 0; i-- {
				cl := tbl.CLs[i]
				if cl.Author == currentPerson || cl.Reviewer == currentPerson {
					// Preserve order.
					copy(tbl.CLs[i:], tbl.CLs[i+1:])
					tbl.CLs = tbl.CLs[:len(tbl.CLs)-1]
				}
			}
		}
		return nil
	})

	tableFetch(3, func(tbl *clTable) error {
		q := datastore.NewQuery("CL").
			Filter("Closed =", true).
			Order("-Modified").
			Limit(10)
		tbl.Title = "Recently closed CLs"
		tbl.Assignable = false
		_, err := q.GetAll(c, &tbl.CLs)
		return err
	})

	// Not really a table fetch.
	tableFetch(0, func(_ *clTable) error {
		var err error
		data.LogoutURL, err = user.LogoutURL(c, "/")
		return err
	})

	wg.Wait()

	select {
	case err := <-errc:
		c.Errorf("%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	default:
	}

	var b bytes.Buffer
	if err := frontPage.ExecuteTemplate(&b, "front", &data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	io.Copy(w, &b)
}

type frontPageData struct {
	Tables [4]clTable
	Timing [4]time.Duration

	Reviewers      []string
	UserIsReviewer bool

	User, LogoutURL string
	IsAdmin         bool
}

type clTable struct {
	Title      string
	Assignable bool
	CLs        []*CL
}

var frontPage = template.Must(template.New("front").Funcs(template.FuncMap{
	"selected": func(a, b string) string {
		if a == b {
			return "selected"
		}
		return ""
	},
}).Parse(`
<!doctype html>

    <title>Go code reviews</title>
    <link rel="icon" type="image/png" href="/static/icon.png" />

Go code reviews

{{range $tbl := .Tables}}

{{$tbl.Title}}

{{if .CLs}}

{{range $cl := .CLs}}

    {{$cl.DisplayOwner}}
    {{if $tbl.Assignable}}

    <select id="cl-rev-{{$cl.Number}}" {{if not $.UserIsReviewer}}disabled{{end}}>
      <option></option>
      {{range $.Reviewers}}
      <option {{selected . $cl.Reviewer}}>{{.}}</option>
      {{end}}
    </select>

    {{end}}

      {{.Number}}: {{.FirstLineHTML}}
      {{if and .LGTMs $tbl.Assignable}}
LGTMs: {{.LGTMHTML}}{{end}}
      {{if and .NotLGTMs $tbl.Assignable}}
NOT LGTMs: {{.NotLGTMHTML}}{{end}}

    {{.ModifiedAgo}}
    {{if $.IsAdmin}}⟳	{{end}}

{{end}}

{{else}}
none
{{end}}
{{end}}

<address>
You are {{.User}} · logout

datastore timing: {{range .Timing}} {{.}}{{end}}
</address>

`))
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package revocation

import (
	"database/sql"
	"encoding/hex"
	"fmt"
	"time"
)

type database interface {
	InsertCaveat(thirdPartyCaveatID string, revocationCaveatID []byte) error
	Revoke(thirdPartyCaveatID string) error
	IsRevoked(revocationCaveatID []byte) (bool, error)
	RevocationTime(thirdPartyCaveatID string) (*time.Time, error)
}

// Table with 3 columns:
// (1) ThirdPartyCaveatID= string thirdPartyCaveatID.
// (2) RevocationCaveatID= hex encoded revcationCaveatID.
// (3) RevocationTime= time (if any) that the Caveat was revoked.
type sqlDatabase struct {
	insertCaveatStmt, revokeStmt, isRevokedStmt, revocationTimeStmt *sql.Stmt
}

func (s *sqlDatabase) InsertCaveat(thirdPartyCaveatID string, revocationCaveatID []byte) error {
	_, err := s.insertCaveatStmt.Exec(thirdPartyCaveatID, hex.EncodeToString(revocationCaveatID))
	return err
}

func (s *sqlDatabase) Revoke(thirdPartyCaveatID string) error {
	_, err := s.revokeStmt.Exec(time.Now(), thirdPartyCaveatID)
	return err
}

func (s *sqlDatabase) IsRevoked(revocationCaveatID []byte) (bool, error) {
	rows, err := s.isRevokedStmt.Query(hex.EncodeToString(revocationCaveatID))
	if err != nil {
		return false, err
	}
	return rows.Next(), nil
}

func (s *sqlDatabase) RevocationTime(thirdPartyCaveatID string) (*time.Time, error) {
	rows, err := s.revocationTimeStmt.Query(thirdPartyCaveatID)
	if err != nil {
		return nil, err
	}
	if rows.Next() {
		var timestamp time.Time
		if err := rows.Scan(&timestamp); err != nil {
			return nil, err
		}
		return &timestamp, nil
	}
	return nil, fmt.Errorf("the caveat (%v) was not revoked", thirdPartyCaveatID)
}

func newSQLDatabase(db *sql.DB, table string) (database, error) {
	createStmt, err := db.Prepare(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s ( ThirdPartyCaveatID NVARCHAR(255), RevocationCaveatID NVARCHAR(255), RevocationTime DATETIME, PRIMARY KEY (ThirdPartyCaveatID), KEY (RevocationCaveatID) );", table))
	if err != nil {
		return nil, err
	}
	if _, err = createStmt.Exec(); err != nil {
		return nil, err
	}
	insertCaveatStmt, err := db.Prepare(fmt.Sprintf("INSERT INTO %s (ThirdPartyCaveatID, RevocationCaveatID, RevocationTime) VALUES (?, ?, NULL)", table))
	if err != nil {
		return nil, err
	}
	revokeStmt, err := db.Prepare(fmt.Sprintf("UPDATE %s SET RevocationTime=? WHERE ThirdPartyCaveatID=?", table))
	if err != nil {
		return nil, err
	}
	isRevokedStmt, err := db.Prepare(fmt.Sprintf("SELECT 1 FROM %s WHERE RevocationCaveatID=? AND RevocationTime IS NOT NULL", table))
	if err != nil {
		return nil, err
	}
	revocationTimeStmt, err := db.Prepare(fmt.Sprintf("SELECT RevocationTime FROM %s WHERE ThirdPartyCaveatID=?", table))
	return &sqlDatabase{insertCaveatStmt, revokeStmt, isRevokedStmt, revocationTimeStmt}, err
}
<commit_msg>services/identity: Close sql connections.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package revocation

import (
	"database/sql"
	"encoding/hex"
	"fmt"
	"time"
)

type database interface {
	InsertCaveat(thirdPartyCaveatID string, revocationCaveatID []byte) error
	Revoke(thirdPartyCaveatID string) error
	IsRevoked(revocationCaveatID []byte) (bool, error)
	RevocationTime(thirdPartyCaveatID string) (*time.Time, error)
}

// Table with 3 columns:
// (1) ThirdPartyCaveatID= string thirdPartyCaveatID.
// (2) RevocationCaveatID= hex encoded revcationCaveatID.
// (3) RevocationTime= time (if any) that the Caveat was revoked.
type sqlDatabase struct {
	insertCaveatStmt, revokeStmt, isRevokedStmt, revocationTimeStmt *sql.Stmt
}

func (s *sqlDatabase) InsertCaveat(thirdPartyCaveatID string, revocationCaveatID []byte) error {
	_, err := s.insertCaveatStmt.Exec(thirdPartyCaveatID, hex.EncodeToString(revocationCaveatID))
	return err
}

func (s *sqlDatabase) Revoke(thirdPartyCaveatID string) error {
	_, err := s.revokeStmt.Exec(time.Now(), thirdPartyCaveatID)
	return err
}

func (s *sqlDatabase) IsRevoked(revocationCaveatID []byte) (bool, error) {
	rows, err := s.isRevokedStmt.Query(hex.EncodeToString(revocationCaveatID))
	defer rows.Close()
	if err != nil {
		return false, err
	}
	return rows.Next(), nil
}

func (s *sqlDatabase) RevocationTime(thirdPartyCaveatID string) (*time.Time, error) {
	rows, err := s.revocationTimeStmt.Query(thirdPartyCaveatID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	if rows.Next() {
		var timestamp time.Time
		if err := rows.Scan(&timestamp); err != nil {
			return nil, err
		}
		return &timestamp, nil
	}
	return nil, fmt.Errorf("the caveat (%v) was not revoked", thirdPartyCaveatID)
}

func newSQLDatabase(db *sql.DB, table string) (database, error) {
	createStmt, err := db.Prepare(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s ( ThirdPartyCaveatID NVARCHAR(255), RevocationCaveatID NVARCHAR(255), RevocationTime DATETIME, PRIMARY KEY (ThirdPartyCaveatID), KEY (RevocationCaveatID) );", table))
	if err != nil {
		return nil, err
	}
	if _, err = createStmt.Exec(); err != nil {
		return nil, err
	}
	insertCaveatStmt, err := db.Prepare(fmt.Sprintf("INSERT INTO %s (ThirdPartyCaveatID, RevocationCaveatID, RevocationTime) VALUES (?, ?, NULL)", table))
	if err != nil {
		return nil, err
	}
	revokeStmt, err := db.Prepare(fmt.Sprintf("UPDATE %s SET RevocationTime=? WHERE ThirdPartyCaveatID=?", table))
	if err != nil {
		return nil, err
	}
	isRevokedStmt, err := db.Prepare(fmt.Sprintf("SELECT 1 FROM %s WHERE RevocationCaveatID=? AND RevocationTime IS NOT NULL", table))
	if err != nil {
		return nil, err
	}
	revocationTimeStmt, err := db.Prepare(fmt.Sprintf("SELECT RevocationTime FROM %s WHERE ThirdPartyCaveatID=?", table))
	return &sqlDatabase{insertCaveatStmt, revokeStmt, isRevokedStmt, revocationTimeStmt}, err
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"encoding/json"
	"errors"
	"log"
	"strings"
	"time"
)

// Handler contains reference to the database client
type Handler struct {
	*sql.DB
}

// Search returns search results for applications
func (handler Handler) Search(query string) (RebbleCards, error) {
	query = strings.Replace(query, "!", "!!", -1)
	query = strings.Replace(query, "%", "!%", -1)
	query = strings.Replace(query, "_", "!_", -1)
	query = strings.Replace(query, "[", "![", -1)
	query = "%" + query + "%"

	var cards RebbleCards
	rows, err := handler.Query(
		"SELECT id, name, type, thumbs_up, screenshots FROM apps WHERE name LIKE ? ESCAPE '!' ORDER BY thumbs_up DESC LIMIT 12",
		query,
	)
	if err != nil {
		return cards, err
	}
	cards.Cards = make([]RebbleCard, 0)
	for rows.Next() {
		card := RebbleCard{}
		var screenshots_b []byte
		var screenshots []RebbleScreenshotsPlatform
		err = rows.Scan(&card.Id, &card.Title, &card.Type, &card.ThumbsUp, &screenshots_b)
		if err != nil {
			return RebbleCards{}, err
		}
		err = json.Unmarshal(screenshots_b, &screenshots)
		if err != nil {
			return RebbleCards{}, err
		}
		if len(screenshots) != 0 && len(screenshots[0].Screenshots) != 0 {
			card.ImageUrl = screenshots[0].Screenshots[0]
		}
		cards.Cards = append(cards.Cards, card)
	}
	return cards, nil
}

// GetAppsForCollection returns list of apps for single collection
func (handler Handler) GetAppsForCollection(collectionID string) ([]RebbleApplication, error) {
	rows, err := handler.Query("SELECT apps FROM collections WHERE id=?", collectionID)
	if err != nil {
		return nil, err
	}
	if !rows.Next() {
		return nil, errors.New("Specified collection does not exist")
	}
	var appIdsB []byte
	var appIds []string
	err = rows.Scan(&appIdsB)
	if err != nil {
		return nil, err
	}
	json.Unmarshal(appIdsB, &appIds)

	apps := make([]RebbleApplication, 0)
	for _, id := range appIds {
		rows, err = handler.Query("SELECT id, name, type, thumbs_up, screenshots, published_date, supported_platforms FROM apps WHERE id=?", id)
		if err != nil {
			return nil, err
		}

		for rows.Next() {
			app := RebbleApplication{}
			var t int64
			var supported_platforms_b []byte
			var screenshots_b []byte
			err = rows.Scan(&app.Id, &app.Name, &app.Type, &app.ThumbsUp, &screenshots_b, &t, &supported_platforms_b)
			if err != nil {
				return []RebbleApplication{}, err
			}
			app.Published.Time = time.Unix(0, t)
			err = json.Unmarshal(supported_platforms_b, &app.SupportedPlatforms)
			if err != nil {
				return []RebbleApplication{}, err
			}
			err = json.Unmarshal(screenshots_b, &app.Assets.Screenshots)
			if err != nil {
				return []RebbleApplication{}, err
			}
			apps = append(apps, app)
		}
	}
	return apps, nil
}

// GetCollectionName returns the name of a collection
func (handler Handler) GetCollectionName(collectionID string) (string, error) {
	rows, err := handler.Query("SELECT name FROM collections WHERE id=?", collectionID)
	if err != nil {
		return "", err
	}
	if !rows.Next() {
		return "", errors.New("Specified collection does not exist")
	}
	var name string
	err = rows.Scan(&name)
	if err != nil {
		return "", err
	}

	return name, nil
}

// GetAllApps returns all available apps
func (handler Handler) GetAllApps(sortby string, ascending bool, offset int, limit int) ([]RebbleApplication, error) {
	order := "DESC"
	if ascending {
		order = "ASC"
	}

	var orderCol string
	switch sortby {
	case "popular":
		orderCol = "apps.thumbs_up"
	default:
		orderCol = "apps.published_date"
	}

	log.Printf("Sort by: %v\nOrder: %v\nLimit: %v\nOffset: %v\n", orderCol, order, limit, offset)

	rows, err := handler.Query(`
		SELECT apps.name, authors.name, apps.icon_url, apps.id, apps.thumbs_up, apps.published_date
		FROM apps
		JOIN authors ON apps.author_id = authors.id
		ORDER BY ?
		LIMIT ?
		OFFSET ?
	`, orderCol+" "+order, limit, offset)
	if err != nil {
		return nil, err
	}
	apps := make([]RebbleApplication, 0)
	for rows.Next() {
		app := RebbleApplication{}
		var t_published int64
		err = rows.Scan(&app.Name, &app.Author.Name, &app.Assets.Icon, &app.Id, &app.ThumbsUp, &t_published)
		app.Published.Time = time.Unix(0, t_published)

		apps = append(apps, app)
	}
	return apps, nil
}

// GetApp returns a specific app
func (handler Handler) GetApp(id string) (RebbleApplication, error) {
	rows, err := handler.Query("SELECT apps.id, apps.name, apps.author_id, authors.name, apps.tag_ids, apps.description, apps.thumbs_up, apps.type, apps.supported_platforms, apps.published_date, apps.pbw_url, apps.rebble_ready, apps.updated, apps.version, apps.support_url, apps.author_url, apps.source_url, apps.screenshots, apps.banner_url, apps.icon_url, apps.doomsday_backup FROM apps JOIN authors ON apps.author_id = authors.id WHERE apps.id=?", id)
	if err != nil {
		return RebbleApplication{}, err
	}

	exists := rows.Next()
	if !exists {
		return RebbleApplication{}, errors.New("No application with this ID")
	}

	app := RebbleApplication{}
	var supportedPlatforms_b []byte
	var t_published, t_updated int64
	var tagIds_b []byte
	var tagIds []string
	var screenshots_b []byte
	var screenshots *([]RebbleScreenshotsPlatform)
	err = rows.Scan(&app.Id, &app.Name, &app.Author.Id, &app.Author.Name, &tagIds_b, &app.Description, &app.ThumbsUp, &app.Type, &supportedPlatforms_b, &t_published, &app.AppInfo.PbwUrl, &app.AppInfo.RebbleReady, &t_updated, &app.AppInfo.Version, &app.AppInfo.SupportUrl, &app.AppInfo.AuthorUrl, &app.AppInfo.SourceUrl, &screenshots_b, &app.Assets.Banner, &app.Assets.Icon, &app.DoomsdayBackup)
	if err != nil {
		return RebbleApplication{}, err
	}
	json.Unmarshal(supportedPlatforms_b, &app.SupportedPlatforms)
	app.Published.Time = time.Unix(0, t_published)
	app.AppInfo.Updated.Time = time.Unix(0, t_updated)
	json.Unmarshal(tagIds_b, &tagIds)
	app.AppInfo.Tags = make([]RebbleCollection, len(tagIds))
	json.Unmarshal(screenshots_b, &screenshots)
	app.Assets.Screenshots = screenshots

	for i, tagID := range tagIds {
		rows, err := handler.Query("SELECT id, name, color FROM collections WHERE id=?", tagID)
		if err != nil {
			return RebbleApplication{}, err
		}

		rows.Next()
		err = rows.Scan(&app.AppInfo.Tags[i].Id, &app.AppInfo.Tags[i].Name, &app.AppInfo.Tags[i].Color)
		if err != nil {
			return RebbleApplication{}, err
		}
	}

	return app, nil
}

// GetAppTags returns the the list of tags of the application with the id `id`
func (handler Handler) GetAppTags(id string) ([]RebbleCollection, error) {
	rows, err := handler.Query("SELECT apps.tag_ids FROM apps WHERE id=?", id)
	if err != nil {
		return []RebbleCollection{}, err
	}
	exists := rows.Next()
	if !exists {
		return []RebbleCollection{}, err
	}

	var tagIds_b []byte
	var tagIds []string
	err = rows.Scan(&tagIds_b)
	if err != nil {
		return []RebbleCollection{}, err
	}
	json.Unmarshal(tagIds_b, &tagIds)
	collections := make([]RebbleCollection, len(tagIds))

	for i, tagId := range tagIds {
		rows, err := handler.Query("SELECT id, name, color FROM collections WHERE id=?", tagId)
		if err != nil {
			return []RebbleCollection{}, err
		}

		rows.Next()
		err = rows.Scan(&collections[i].Id, &collections[i].Name, &collections[i].Color)
		if err != nil {
			return []RebbleCollection{}, err
		}
	}

	return collections, nil
}

// GetAppVersions returns the the list of versions of the application with the id `id`
func (handler Handler) GetAppVersions(id string) ([]RebbleVersion, error) {
	rows, err := handler.Query("SELECT apps.versions FROM apps WHERE id=?", id)
	if err != nil {
		return []RebbleVersion{}, err
	}
	exists := rows.Next()
	if !exists {
		return []RebbleVersion{}, errors.New("No app with this ID")
	}

	var versions_b []byte
	var versions []RebbleVersion
	err = rows.Scan(&versions_b)
	if err != nil {
		return []RebbleVersion{}, err
	}
	json.Unmarshal(versions_b, &versions)

	return versions, nil
}
<commit_msg>Rework the single app query to query for only one, also prevents from having unclosed connections<commit_after>package db

import (
	"database/sql"
	"encoding/json"
	"errors"
	"log"
	"strings"
	"time"
)

// Handler contains reference to the database client
type Handler struct {
	*sql.DB
}

// Search returns search results for applications
func (handler Handler) Search(query string) (RebbleCards, error) {
	query = strings.Replace(query, "!", "!!", -1)
	query = strings.Replace(query, "%", "!%", -1)
	query = strings.Replace(query, "_", "!_", -1)
	query = strings.Replace(query, "[", "![", -1)
	query = "%" + query + "%"

	var cards RebbleCards
	rows, err := handler.Query(
		"SELECT id, name, type, thumbs_up, screenshots FROM apps WHERE name LIKE ? ESCAPE '!' ORDER BY thumbs_up DESC LIMIT 12",
		query,
	)
	if err != nil {
		return cards, err
	}
	cards.Cards = make([]RebbleCard, 0)
	for rows.Next() {
		card := RebbleCard{}
		var screenshots_b []byte
		var screenshots []RebbleScreenshotsPlatform
		err = rows.Scan(&card.Id, &card.Title, &card.Type, &card.ThumbsUp, &screenshots_b)
		if err != nil {
			return RebbleCards{}, err
		}
		err = json.Unmarshal(screenshots_b, &screenshots)
		if err != nil {
			return RebbleCards{}, err
		}
		if len(screenshots) != 0 && len(screenshots[0].Screenshots) != 0 {
			card.ImageUrl = screenshots[0].Screenshots[0]
		}
		cards.Cards = append(cards.Cards, card)
	}
	return cards, nil
}

// GetAppsForCollection returns list of apps for single collection
func (handler Handler) GetAppsForCollection(collectionID string) ([]RebbleApplication, error) {
	rows, err := handler.Query("SELECT apps FROM collections WHERE id=?", collectionID)
	if err != nil {
		return nil, err
	}
	if !rows.Next() {
		return nil, errors.New("Specified collection does not exist")
	}
	var appIdsB []byte
	var appIds []string
	err = rows.Scan(&appIdsB)
	if err != nil {
		return nil, err
	}
	json.Unmarshal(appIdsB, &appIds)

	apps := make([]RebbleApplication, 0)
	for _, id := range appIds {
		rows, err = handler.Query("SELECT id, name, type, thumbs_up, screenshots, published_date, supported_platforms FROM apps WHERE id=?", id)
		if err != nil {
			return nil, err
		}

		for rows.Next() {
			app := RebbleApplication{}
			var t int64
			var supported_platforms_b []byte
			var screenshots_b []byte
			err = rows.Scan(&app.Id, &app.Name, &app.Type, &app.ThumbsUp, &screenshots_b, &t, &supported_platforms_b)
			if err != nil {
				return []RebbleApplication{}, err
			}
			app.Published.Time = time.Unix(0, t)
			err = json.Unmarshal(supported_platforms_b, &app.SupportedPlatforms)
			if err != nil {
				return []RebbleApplication{}, err
			}
			err = json.Unmarshal(screenshots_b, &app.Assets.Screenshots)
			if err != nil {
				return []RebbleApplication{}, err
			}
			apps = append(apps, app)
		}
	}
	return apps, nil
}

// GetCollectionName returns the name of a collection
func (handler Handler) GetCollectionName(collectionID string) (string, error) {
	rows, err := handler.Query("SELECT name FROM collections WHERE id=?", collectionID)
	if err != nil {
		return "", err
	}
	if !rows.Next() {
		return "", errors.New("Specified collection does not exist")
	}
	var name string
	err = rows.Scan(&name)
	if err != nil {
		return "", err
	}

	return name, nil
}

// GetAllApps returns all available apps
func (handler Handler) GetAllApps(sortby string, ascending bool, offset int, limit int) ([]RebbleApplication, error) {
	order := "DESC"
	if ascending {
		order = "ASC"
	}

	var orderCol string
	switch sortby {
	case "popular":
		orderCol = "apps.thumbs_up"
	default:
		orderCol = "apps.published_date"
	}

	log.Printf("Sort by: %v\nOrder: %v\nLimit: %v\nOffset: %v\n", orderCol, order, limit, offset)

	rows, err := handler.Query(`
		SELECT apps.name, authors.name, apps.icon_url, apps.id, apps.thumbs_up, apps.published_date
		FROM apps
		JOIN authors ON apps.author_id = authors.id
		ORDER BY ?
		LIMIT ?
		OFFSET ?
	`, orderCol+" "+order, limit, offset)
	if err != nil {
		return nil, err
	}
	apps := make([]RebbleApplication, 0)
	for rows.Next() {
		app := RebbleApplication{}
		var t_published int64
		err = rows.Scan(&app.Name, &app.Author.Name, &app.Assets.Icon, &app.Id, &app.ThumbsUp, &t_published)
		app.Published.Time = time.Unix(0, t_published)

		apps = append(apps, app)
	}
	return apps, nil
}

// GetApp returns a specific app
func (handler Handler) GetApp(id string) (RebbleApplication, error) {
	row := handler.QueryRow("SELECT apps.id, apps.name, apps.author_id, authors.name, apps.tag_ids, apps.description, apps.thumbs_up, apps.type, apps.supported_platforms, apps.published_date, apps.pbw_url, apps.rebble_ready, apps.updated, apps.version, apps.support_url, apps.author_url, apps.source_url, apps.screenshots, apps.banner_url, apps.icon_url, apps.doomsday_backup FROM apps JOIN authors ON apps.author_id = authors.id WHERE apps.id=?", id)

	app := RebbleApplication{}
	var supportedPlatforms_b []byte
	var t_published, t_updated int64
	var tagIds_b []byte
	var tagIds []string
	var screenshots_b []byte
	var screenshots *([]RebbleScreenshotsPlatform)
	err := row.Scan(&app.Id, &app.Name, &app.Author.Id, &app.Author.Name, &tagIds_b, &app.Description, &app.ThumbsUp, &app.Type, &supportedPlatforms_b, &t_published, &app.AppInfo.PbwUrl, &app.AppInfo.RebbleReady, &t_updated, &app.AppInfo.Version, &app.AppInfo.SupportUrl, &app.AppInfo.AuthorUrl, &app.AppInfo.SourceUrl, &screenshots_b, &app.Assets.Banner, &app.Assets.Icon, &app.DoomsdayBackup)
	if err == sql.ErrNoRows {
		return RebbleApplication{}, errors.New("No application with this ID")
	} else if err != nil {
		return RebbleApplication{}, err
	}

	json.Unmarshal(supportedPlatforms_b, &app.SupportedPlatforms)
	app.Published.Time = time.Unix(0, t_published)
	app.AppInfo.Updated.Time = time.Unix(0, t_updated)
	json.Unmarshal(tagIds_b, &tagIds)
	app.AppInfo.Tags = make([]RebbleCollection, len(tagIds))
	json.Unmarshal(screenshots_b, &screenshots)
	app.Assets.Screenshots = screenshots

	for i, tagID := range tagIds {
		row := handler.QueryRow("SELECT id, name, color FROM collections WHERE id=?", tagID)

		err = row.Scan(&app.AppInfo.Tags[i].Id, &app.AppInfo.Tags[i].Name, &app.AppInfo.Tags[i].Color)
		if err != nil {
			return RebbleApplication{}, err
		}
	}

	return app, nil
}

// GetAppTags returns the the list of tags of the application with the id `id`
func (handler Handler) GetAppTags(id string) ([]RebbleCollection, error) {
	rows, err := handler.Query("SELECT apps.tag_ids FROM apps WHERE id=?", id)
	if err != nil {
		return []RebbleCollection{}, err
	}
	exists := rows.Next()
	if !exists {
		return []RebbleCollection{}, err
	}

	var tagIds_b []byte
	var tagIds []string
	err = rows.Scan(&tagIds_b)
	if err != nil {
		return []RebbleCollection{}, err
	}
	json.Unmarshal(tagIds_b, &tagIds)
	collections := make([]RebbleCollection, len(tagIds))

	for i, tagId := range tagIds {
		rows, err := handler.Query("SELECT id, name, color FROM collections WHERE id=?", tagId)
		if err != nil {
			return []RebbleCollection{}, err
		}

		rows.Next()
		err = rows.Scan(&collections[i].Id, &collections[i].Name, &collections[i].Color)
		if err != nil {
			return []RebbleCollection{}, err
		}
	}

	return collections, nil
}

// GetAppVersions returns the the list of versions of the application with the id `id`
func (handler Handler) GetAppVersions(id string) ([]RebbleVersion, error) {
	rows, err := handler.Query("SELECT apps.versions FROM apps WHERE id=?", id)
	if err != nil {
		return []RebbleVersion{}, err
	}
	exists := rows.Next()
	if !exists {
		return []RebbleVersion{}, errors.New("No app with this ID")
	}

	var versions_b []byte
	var versions []RebbleVersion
	err = rows.Scan(&versions_b)
	if err != nil {
		return []RebbleVersion{}, err
	}
	json.Unmarshal(versions_b, &versions)

	return versions, nil
}
<|endoftext|>


<commit_before>// Package flagsconfig enables to save user defined key-value pairs
// and flags into a configuration file written in json format.
package flagsconfig

import (
	"flag"
	"fmt"
	"os"
	"tojson"
)

// Config represents the configuration file overloaded with
// user defined flags.
// There can be no configuration file and just flags.
type Config struct {
	path   string
	flags  map[string]string
	filter map[string]struct{}
}

func (c *Config) saveFlags() error {
	// loop through used and user defined flags
	// except for the filtered ones and update
	// the flags map from the Config
	flag.VisitAll(func(flag *flag.Flag) {
		name := flag.Name
		if _, ok := c.filter[name]; !ok {
			c.flags[name] = flag.Value.String()
		}
	})
	// save the flags map into a configuration file if any
	if c.path != "" {
		return tojson.Save(c.path, &c.flags)
	}
	return nil
}

func (c *Config) updateFlags(path string) error {
	c.path = path
	c.flags = make(map[string]string)
	// load configuration file data if any into a flags map
	if path != "" {
		err := tojson.Load(c.path, &c.flags)
		if err != nil {
			return err
		}
	}
	// loop through used flags
	// and remove them from the flags map
	flag.Visit(func(flag *flag.Flag) {
		_, ok := c.flags[flag.Name]
		if ok {
			delete(c.flags, flag.Name)
		}
	})
	// loop through used flags and user defined flags
	// and update the flags map
	flag.VisitAll(func(flag *flag.Flag) {
		val, ok := c.flags[flag.Name]
		if ok {
			flag.Value.Set(val)
		}
	})
	return nil
}

// Parse parses the given configuration file and overloads it with the
// currently used flags.
func (c *Config) Parse(path string) error {
	err := c.updateFlags(path)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to load the config file: %v", err)
	}
	return c.saveFlags()
}

// NewConfig makes a Config given a config file path and a list
// of filtered flags not to appear in the config file.
func NewConfig(path string, filter ...string) (*Config, error) {
	c := &Config{
		filter: make(map[string]struct{}),
	}
	for _, v := range filter {
		c.filter[v] = struct{}{}
	}
	return c, c.Parse(path)
}

// Update updates a pair of key-value flags
// from the flags map of Config and update
// the configuration file if any.
func (c *Config) Update(key, value string) error {
	c.flags[key] = value
	if c.path != "" {
		return tojson.Save(c.path, &c.flags)
	}
	return nil
}

// Get gets a flag from a given key or
// return 0 value for a string if none.
func (c *Config) Get(key string) string {
	value, ok := c.flags[key]
	if ok {
		return value
	}
	return ""
}
<commit_msg>[flagsconfig] fix flag parsing and make the 'config' flag default and filtered<commit_after>// Package flagsconfig enables to save user defined key-value pairs
// and used flags at runtime into a configuration file written in json format.
package flagsconfig

import (
	"flag"
	"fmt"
	"os"
	"tojson"
)

// Config represents the configuration file overloaded with
// user defined flags.
// There can be no configuration file and just flags.
type Config struct {
	path   string
	flags  map[string]string
	filter map[string]struct{}
}

func (c *Config) saveFlags() error {
	// loop through used and user defined flags
	// except for the filtered ones and update
	// the flags map from the Config
	flag.VisitAll(func(flag *flag.Flag) {
		name := flag.Name
		if _, ok := c.filter[name]; !ok {
			c.flags[name] = flag.Value.String()
		}
	})
	// save the flags map into a configuration file if any
	if c.path != "" {
		return tojson.Save(c.path, &c.flags)
	}
	return nil
}

func (c *Config) updateFlags(path string) error {
	c.path = path
	c.flags = make(map[string]string)
	// load configuration file data if any into a flags map
	if path != "" {
		err := tojson.Load(c.path, &c.flags)
		if err != nil {
			return err
		}
	}
	// loop through used flags
	// and remove them from the flags map
	flag.Visit(func(flag *flag.Flag) {
		_, ok := c.flags[flag.Name]
		if ok {
			delete(c.flags, flag.Name)
		}
	})
	// loop through used flags and user defined flags
	// and update the flags map
	flag.VisitAll(func(flag *flag.Flag) {
		val, ok := c.flags[flag.Name]
		if ok {
			flag.Value.Set(val)
		}
	})
	return nil
}

// Parse parses the given configuration file and overloads it with the
// currently used flags.
func (c *Config) Parse(path string) error {
	err := c.updateFlags(path)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to load the config file: %v", err)
	}
	return c.saveFlags()
}

// NewConfig makes a Config given a default config file path and a list
// of filtered flags not to appear in the config file.
// Note that the 'config' flag is defined by this method and added
// to the list of filtered flags.
func NewConfig(path string, filter ...string) (*Config, error) {
	// print defaults flags when -h/--help is used
	flag.Usage = func() {
		flag.PrintDefaults()
	}
	// checks if the config flag is already defined
	// Note that in the case where the user define the 'config' flag
	// ahead of calling NewConfig, the user is forced to call flag.Parse()
	// also to retrieve the corresponding value and pass it to the first
	// argument of NewConfig.
	configKey := "config"
	configFile := path // this variable will be used if the 'config' flag is already defined
	if flag.Lookup(configKey) == nil {
		flag.StringVar(&configFile, configKey, configFile, "configuration filename")
	}
	// parses the flags
	flag.Parse()
	c := &Config{
		filter: make(map[string]struct{}),
	}
	// add the 'config' flag to the list of filtered flags by default
	c.filter[configKey] = struct{}{}
	for _, v := range filter {
		c.filter[v] = struct{}{}
	}
	return c, c.Parse(configFile)
}

// Update updates a pair of key-value flags
// from the flags map of Config and update
// the configuration file if any.
func (c *Config) Update(key, value string) error {
	c.flags[key] = value
	if c.path != "" {
		return tojson.Save(c.path, &c.flags)
	}
	return nil
}

// Get gets a flag from a given key or
// return 0 value for a string if none.
func (c *Config) Get(key string) string {
	value, ok := c.flags[key]
	if ok {
		return value
	}
	return ""
}
<|endoftext|>