<commit_before>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"os"
	"regexp"
	"runtime"
	"testing"

	"k8s.io/apimachinery/pkg/util/sets"
)

var (
	imageNames      = initImageNames()
	excludedTests   = initExcludedTests()
	imagesMappingRE map[*regexp.Regexp][]byte
)

const (
	// Busybox image with specific sha
	busyboxImage = iota
	// Registry image
	registryImage
	// kubectl image
	kanikoImage
	// dockerize image
	dockerizeImage
)

func init() {
	imagesMappingRE = getImagesMappingRE()
}

// getTestArch returns architecture of the cluster where test suites will be executed.
// default value is similar to build architecture, TEST_RUNTIME_ARCH is used when test target cluster has another architecture
func getTestArch() string {
	val, ok := os.LookupEnv("TEST_RUNTIME_ARCH")
	if ok {
		return val
	}
	return runtime.GOARCH
}

// initImageNames returns the map with arch dependent image names for e2e tests
func initImageNames() map[int]string {
	switch getTestArch() {
	case "s390x":
		return map[int]string{
			busyboxImage:   "busybox@sha256:4f47c01fa91355af2865ac10fef5bf6ec9c7f42ad2321377c21e844427972977",
			registryImage:  "ibmcom/registry:2.6.2.5",
			kanikoImage:    "gcr.io/kaniko-project/executor:s390x-9ed158c1f63a059cde4fd5f8b95af51d452d9aa7",
			dockerizeImage: "ibmcom/dockerize-s390x",
		}
	case "ppc64le":
		return map[int]string{
			busyboxImage:   "busybox@sha256:4f47c01fa91355af2865ac10fef5bf6ec9c7f42ad2321377c21e844427972977",
			registryImage:  "ppc64le/registry:2",
			kanikoImage:    "ibmcom/kaniko-project-executor-ppc64le:v0.17.1",
			dockerizeImage: "ibmcom/dockerize-ppc64le",
		}
	default:
		return map[int]string{
			busyboxImage:   "busybox@sha256:895ab622e92e18d6b461d671081757af7dbaa3b00e3e28e12505af7817f73649",
			registryImage:  "registry",
			kanikoImage:    "gcr.io/kaniko-project/executor:v1.3.0",
			dockerizeImage: "jwilder/dockerize",
		}
	}
}

// getImagesMappingRE generates the map ready to search and replace image names with regexp for examples files.
// search is done using "image: <name>" pattern.
func getImagesMappingRE() map[*regexp.Regexp][]byte {
	imageNamesMapping := imageNamesMapping()
	imageMappingRE := make(map[*regexp.Regexp][]byte, len(imageNamesMapping))

	for existingImage, archSpecificImage := range imageNamesMapping {
		imageMappingRE[regexp.MustCompile("(?im)image: "+existingImage+"$")] = []byte("image: " + archSpecificImage)
		imageMappingRE[regexp.MustCompile("(?im)default: "+existingImage+"$")] = []byte("default: " + archSpecificImage)
	}

	return imageMappingRE
}

// imageNamesMapping provides mapping between image name in the examples yaml files and desired image name for specific arch.
// by default empty map is returned.
func imageNamesMapping() map[string]string {

	switch getTestArch() {
	case "s390x":
		return map[string]string{
			"registry":                              getTestImage(registryImage),
			"node":                                  "node:alpine3.11",
			"gcr.io/cloud-builders/git":             "alpine/git:latest",
			"docker:dind":                           "ibmcom/docker-s390x:dind",
			"docker":                                "docker:18.06.3",
			"mikefarah/yq:3":                        "danielxlee/yq:2.4.0",
			"stedolan/jq":                           "ibmcom/jq-s390x:latest",
			"gcr.io/kaniko-project/executor:v1.3.0": getTestImage(kanikoImage),
		}
	case "ppc64le":
		return map[string]string{
			"registry":                              getTestImage(registryImage),
			"node":                                  "node:alpine3.11",
			"gcr.io/cloud-builders/git":             "alpine/git:latest",
			"docker:dind":                           "ibmcom/docker-ppc64le:19.03-dind",
			"docker":                                "docker:18.06.3",
			"mikefarah/yq:3":                        "danielxlee/yq:2.4.0",
			"stedolan/jq":                           "ibmcom/jq-ppc64le:latest",
			"gcr.io/kaniko-project/executor:v1.3.0": getTestImage(kanikoImage),
		}

	}

	return make(map[string]string)
}

// initExcludedTests provides list of excluded tests for e2e and exanples tests
func initExcludedTests() sets.String {

	switch getTestArch() {
	case "s390x":
		return sets.NewString(
			// examples
			"TestExamples/v1alpha1/taskruns/gcs-resource",
			"TestExamples/v1beta1/taskruns/gcs-resource",
		)
	case "ppc64le":
		return sets.NewString(
			// examples
			"TestExamples/v1alpha1/taskruns/gcs-resource",
			"TestExamples/v1beta1/taskruns/gcs-resource",
		)
	}

	return sets.NewString()
}

// getTestImage gets test image based on unique id
func getTestImage(image int) string {
	return imageNames[image]
}

// skipIfExcluded checks if test name is in the excluded list and skip it
func skipIfExcluded(t *testing.T) {
	if excludedTests.Has(t.Name()) {
		t.Skipf("skip for %s architecture", getTestArch())
	}
}
<commit_msg>Exclude creds-init-only-mounts-provided-credentials test for linux/s390x<commit_after>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"os"
	"regexp"
	"runtime"
	"testing"

	"k8s.io/apimachinery/pkg/util/sets"
)

var (
	imageNames      = initImageNames()
	excludedTests   = initExcludedTests()
	imagesMappingRE map[*regexp.Regexp][]byte
)

const (
	// Busybox image with specific sha
	busyboxImage = iota
	// Registry image
	registryImage
	// kubectl image
	kanikoImage
	// dockerize image
	dockerizeImage
)

func init() {
	imagesMappingRE = getImagesMappingRE()
}

// getTestArch returns architecture of the cluster where test suites will be executed.
// default value is similar to build architecture, TEST_RUNTIME_ARCH is used when test target cluster has another architecture
func getTestArch() string {
	val, ok := os.LookupEnv("TEST_RUNTIME_ARCH")
	if ok {
		return val
	}
	return runtime.GOARCH
}

// initImageNames returns the map with arch dependent image names for e2e tests
func initImageNames() map[int]string {
	switch getTestArch() {
	case "s390x":
		return map[int]string{
			busyboxImage:   "busybox@sha256:4f47c01fa91355af2865ac10fef5bf6ec9c7f42ad2321377c21e844427972977",
			registryImage:  "ibmcom/registry:2.6.2.5",
			kanikoImage:    "gcr.io/kaniko-project/executor:s390x-9ed158c1f63a059cde4fd5f8b95af51d452d9aa7",
			dockerizeImage: "ibmcom/dockerize-s390x",
		}
	case "ppc64le":
		return map[int]string{
			busyboxImage:   "busybox@sha256:4f47c01fa91355af2865ac10fef5bf6ec9c7f42ad2321377c21e844427972977",
			registryImage:  "ppc64le/registry:2",
			kanikoImage:    "ibmcom/kaniko-project-executor-ppc64le:v0.17.1",
			dockerizeImage: "ibmcom/dockerize-ppc64le",
		}
	default:
		return map[int]string{
			busyboxImage:   "busybox@sha256:895ab622e92e18d6b461d671081757af7dbaa3b00e3e28e12505af7817f73649",
			registryImage:  "registry",
			kanikoImage:    "gcr.io/kaniko-project/executor:v1.3.0",
			dockerizeImage: "jwilder/dockerize",
		}
	}
}

// getImagesMappingRE generates the map ready to search and replace image names with regexp for examples files.
// search is done using "image: <name>" pattern.
func getImagesMappingRE() map[*regexp.Regexp][]byte {
	imageNamesMapping := imageNamesMapping()
	imageMappingRE := make(map[*regexp.Regexp][]byte, len(imageNamesMapping))

	for existingImage, archSpecificImage := range imageNamesMapping {
		imageMappingRE[regexp.MustCompile("(?im)image: "+existingImage+"$")] = []byte("image: " + archSpecificImage)
		imageMappingRE[regexp.MustCompile("(?im)default: "+existingImage+"$")] = []byte("default: " + archSpecificImage)
	}

	return imageMappingRE
}

// imageNamesMapping provides mapping between image name in the examples yaml files and desired image name for specific arch.
// by default empty map is returned.
func imageNamesMapping() map[string]string {

	switch getTestArch() {
	case "s390x":
		return map[string]string{
			"registry":                              getTestImage(registryImage),
			"node":                                  "node:alpine3.11",
			"gcr.io/cloud-builders/git":             "alpine/git:latest",
			"docker:dind":                           "ibmcom/docker-s390x:dind",
			"docker":                                "docker:18.06.3",
			"mikefarah/yq:3":                        "danielxlee/yq:2.4.0",
			"stedolan/jq":                           "ibmcom/jq-s390x:latest",
			"gcr.io/kaniko-project/executor:v1.3.0": getTestImage(kanikoImage),
		}
	case "ppc64le":
		return map[string]string{
			"registry":                              getTestImage(registryImage),
			"node":                                  "node:alpine3.11",
			"gcr.io/cloud-builders/git":             "alpine/git:latest",
			"docker:dind":                           "ibmcom/docker-ppc64le:19.03-dind",
			"docker":                                "docker:18.06.3",
			"mikefarah/yq:3":                        "danielxlee/yq:2.4.0",
			"stedolan/jq":                           "ibmcom/jq-ppc64le:latest",
			"gcr.io/kaniko-project/executor:v1.3.0": getTestImage(kanikoImage),
		}

	}

	return make(map[string]string)
}

// initExcludedTests provides list of excluded tests for e2e and exanples tests
func initExcludedTests() sets.String {

	switch getTestArch() {
	case "s390x":
		return sets.NewString(
			// examples
			"TestExamples/v1alpha1/taskruns/gcs-resource",
			"TestExamples/v1beta1/taskruns/gcs-resource",
			"TestExamples/v1beta1/taskruns/creds-init-only-mounts-provided-credentials",
		)
	case "ppc64le":
		return sets.NewString(
			// examples
			"TestExamples/v1alpha1/taskruns/gcs-resource",
			"TestExamples/v1beta1/taskruns/gcs-resource",
		)
	}

	return sets.NewString()
}

// getTestImage gets test image based on unique id
func getTestImage(image int) string {
	return imageNames[image]
}

// skipIfExcluded checks if test name is in the excluded list and skip it
func skipIfExcluded(t *testing.T) {
	if excludedTests.Has(t.Name()) {
		t.Skipf("skip for %s architecture", getTestArch())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/cupcake/jsonschema"
	c "github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-check"
	"github.com/flynn/flynn/pkg/exec"
)

type ControllerSuite struct {
	schemaPaths []string
	schemaCache map[string]*jsonschema.Schema
	Helper
}

var _ = c.Suite(&ControllerSuite{})

func (s *ControllerSuite) SetUpSuite(t *c.C) {
	var schemaPaths []string
	walkFn := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() && filepath.Ext(path) == ".json" {
			schemaPaths = append(schemaPaths, path)
		}
		return nil
	}
	schemaRoot, err := filepath.Abs(filepath.Join("..", "schema"))
	t.Assert(err, c.IsNil)
	t.Assert(filepath.Walk(schemaRoot, walkFn), c.IsNil)

	s.schemaCache = make(map[string]*jsonschema.Schema, len(schemaPaths))
	for _, path := range schemaPaths {
		file, err := os.Open(path)
		t.Assert(err, c.IsNil)
		schema := &jsonschema.Schema{Cache: s.schemaCache}
		err = schema.ParseWithoutRefs(file)
		t.Assert(err, c.IsNil)
		cacheKey := "https://flynn.io/schema" + strings.TrimSuffix(strings.TrimPrefix(path, schemaRoot), ".json")
		s.schemaCache[cacheKey] = schema
		file.Close()
	}
	for _, schema := range s.schemaCache {
		schema.ResolveRefs(false)
	}
}

type controllerExampleRequest struct {
	Method  string            `json:"method,omitempty"`
	URL     string            `json:"url,omitempty"`
	Headers map[string]string `json:"headers,omitempty"`
	Body    interface{}       `json:"body,omitempty"`
}

type controllerExampleResponse struct {
	Headers map[string]string `json:"headers,omitempty"`
	Body    interface{}       `json:"body,omitempty"`
}

type controllerExample struct {
	Request  controllerExampleRequest  `json:"request,omitempty"`
	Response controllerExampleResponse `json:"response,omitempty"`
}

var jsonContentTypePattern = regexp.MustCompile(`\bjson`)

func unmarshalControllerExample(data []byte) (map[string]interface{}, error) {
	var example controllerExample
	if err := json.Unmarshal(data, &example); err != nil {
		return nil, err
	}

	if jsonContentTypePattern.MatchString(example.Request.Headers["Content-Type"]) {
		if body, ok := example.Request.Body.(string); ok {
			var reqBody interface{}
			if err := json.Unmarshal([]byte(body), &reqBody); err != nil {
				return nil, err
			}
			example.Request.Body = reqBody
		}
	}
	if jsonContentTypePattern.MatchString(example.Response.Headers["Content-Type"]) {
		if body, ok := example.Response.Body.(string); ok {
			var resBody interface{}
			if err := json.Unmarshal([]byte(body), &resBody); err != nil {
				return nil, err
			}
			example.Response.Body = resBody
		}
	}

	rawData, err := json.Marshal(example)
	if err != nil {
		return nil, err
	}
	var out map[string]interface{}
	decoder := json.NewDecoder(bytes.NewReader(rawData))
	decoder.UseNumber()
	return out, decoder.Decode(&out)
}

func (s *ControllerSuite) generateControllerExamples(t *c.C) map[string]interface{} {
	cmd := exec.Command(exec.DockerImage(imageURIs["controller-examples"]), "/bin/flynn-controller-examples")
	cmd.Env = map[string]string{"CONTROLLER_KEY": s.clusterConf(t).Key}

	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	t.Logf("stdout: %q", stdout.String())
	t.Logf("stderr: %q", stderr.String())
	t.Assert(err, c.IsNil)

	var controllerExamples map[string]json.RawMessage
	t.Assert(json.Unmarshal(stdout.Bytes(), &controllerExamples), c.IsNil)

	examples := make(map[string]interface{}, len(controllerExamples))
	for key, data := range controllerExamples {
		example, err := unmarshalControllerExample(data)
		t.Assert(err, c.IsNil)
		examples[key] = example
	}
	return examples
}

func (s *ControllerSuite) TestExampleOutput(t *c.C) {
	examples := s.generateControllerExamples(t)
	exampleKeys := make([]string, 0, len(examples))
	for key := range examples {
		exampleKeys = append(exampleKeys, key)
	}
	sort.Strings(exampleKeys)
	for _, key := range exampleKeys {
		cacheKey := "https://flynn.io/schema/examples/controller/" + key
		schema := s.schemaCache[cacheKey]
		if schema == nil {
			continue
		}
		data := examples[key]
		errs := schema.Validate(nil, data)
		var jsonData []byte
		if len(errs) > 0 {
			jsonData, _ = json.MarshalIndent(data, "", "\t")
		}
		t.Assert(errs, c.HasLen, 0, c.Commentf("%s validation errors: %v\ndata: %v\n", cacheKey, errs, string(jsonData)))
	}
}
<commit_msg>test: Add test for controller key rotation<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/cupcake/jsonschema"
	c "github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-check"
	"github.com/flynn/flynn/cli/config"
	"github.com/flynn/flynn/pkg/exec"
	"github.com/flynn/flynn/pkg/random"
)

type ControllerSuite struct {
	schemaPaths []string
	schemaCache map[string]*jsonschema.Schema
	Helper
}

var _ = c.Suite(&ControllerSuite{})

func (s *ControllerSuite) SetUpSuite(t *c.C) {
	var schemaPaths []string
	walkFn := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() && filepath.Ext(path) == ".json" {
			schemaPaths = append(schemaPaths, path)
		}
		return nil
	}
	schemaRoot, err := filepath.Abs(filepath.Join("..", "schema"))
	t.Assert(err, c.IsNil)
	t.Assert(filepath.Walk(schemaRoot, walkFn), c.IsNil)

	s.schemaCache = make(map[string]*jsonschema.Schema, len(schemaPaths))
	for _, path := range schemaPaths {
		file, err := os.Open(path)
		t.Assert(err, c.IsNil)
		schema := &jsonschema.Schema{Cache: s.schemaCache}
		err = schema.ParseWithoutRefs(file)
		t.Assert(err, c.IsNil)
		cacheKey := "https://flynn.io/schema" + strings.TrimSuffix(strings.TrimPrefix(path, schemaRoot), ".json")
		s.schemaCache[cacheKey] = schema
		file.Close()
	}
	for _, schema := range s.schemaCache {
		schema.ResolveRefs(false)
	}
}

type controllerExampleRequest struct {
	Method  string            `json:"method,omitempty"`
	URL     string            `json:"url,omitempty"`
	Headers map[string]string `json:"headers,omitempty"`
	Body    interface{}       `json:"body,omitempty"`
}

type controllerExampleResponse struct {
	Headers map[string]string `json:"headers,omitempty"`
	Body    interface{}       `json:"body,omitempty"`
}

type controllerExample struct {
	Request  controllerExampleRequest  `json:"request,omitempty"`
	Response controllerExampleResponse `json:"response,omitempty"`
}

var jsonContentTypePattern = regexp.MustCompile(`\bjson`)

func unmarshalControllerExample(data []byte) (map[string]interface{}, error) {
	var example controllerExample
	if err := json.Unmarshal(data, &example); err != nil {
		return nil, err
	}

	if jsonContentTypePattern.MatchString(example.Request.Headers["Content-Type"]) {
		if body, ok := example.Request.Body.(string); ok {
			var reqBody interface{}
			if err := json.Unmarshal([]byte(body), &reqBody); err != nil {
				return nil, err
			}
			example.Request.Body = reqBody
		}
	}
	if jsonContentTypePattern.MatchString(example.Response.Headers["Content-Type"]) {
		if body, ok := example.Response.Body.(string); ok {
			var resBody interface{}
			if err := json.Unmarshal([]byte(body), &resBody); err != nil {
				return nil, err
			}
			example.Response.Body = resBody
		}
	}

	rawData, err := json.Marshal(example)
	if err != nil {
		return nil, err
	}
	var out map[string]interface{}
	decoder := json.NewDecoder(bytes.NewReader(rawData))
	decoder.UseNumber()
	return out, decoder.Decode(&out)
}

func (s *ControllerSuite) generateControllerExamples(t *c.C) map[string]interface{} {
	cmd := exec.Command(exec.DockerImage(imageURIs["controller-examples"]), "/bin/flynn-controller-examples")
	cmd.Env = map[string]string{"CONTROLLER_KEY": s.clusterConf(t).Key}

	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	t.Logf("stdout: %q", stdout.String())
	t.Logf("stderr: %q", stderr.String())
	t.Assert(err, c.IsNil)

	var controllerExamples map[string]json.RawMessage
	t.Assert(json.Unmarshal(stdout.Bytes(), &controllerExamples), c.IsNil)

	examples := make(map[string]interface{}, len(controllerExamples))
	for key, data := range controllerExamples {
		example, err := unmarshalControllerExample(data)
		t.Assert(err, c.IsNil)
		examples[key] = example
	}
	return examples
}

func (s *ControllerSuite) TestExampleOutput(t *c.C) {
	examples := s.generateControllerExamples(t)
	exampleKeys := make([]string, 0, len(examples))
	for key := range examples {
		exampleKeys = append(exampleKeys, key)
	}
	sort.Strings(exampleKeys)
	for _, key := range exampleKeys {
		cacheKey := "https://flynn.io/schema/examples/controller/" + key
		schema := s.schemaCache[cacheKey]
		if schema == nil {
			continue
		}
		data := examples[key]
		errs := schema.Validate(nil, data)
		var jsonData []byte
		if len(errs) > 0 {
			jsonData, _ = json.MarshalIndent(data, "", "\t")
		}
		t.Assert(errs, c.HasLen, 0, c.Commentf("%s validation errors: %v\ndata: %v\n", cacheKey, errs, string(jsonData)))
	}
}

func (s *ControllerSuite) TestKeyRotation(t *c.C) {
	cc := s.clusterConf(t)
	oldKey := cc.Key
	newKey := random.Hex(16)

	// allow auth to API with old and new keys
	set := flynn(t, "/", "-a", "controller", "env", "set", "-t", "web", fmt.Sprintf("AUTH_KEY=%s,%s", newKey, oldKey))
	t.Assert(set, Succeeds)

	// reconfigure components to use new key
	for _, app := range []string{"gitreceive", "taffy", "dashboard"} {
		set := flynn(t, "/", "-a", app, "env", "set", "CONTROLLER_KEY="+newKey)
		t.Assert(set, Succeeds)
	}

	// write a new flynnrc
	cc.Key = newKey
	conf := &config.Config{}
	err := conf.Add(cc, true)
	t.Assert(err, c.IsNil)
	err = conf.SaveTo(flynnrc)
	t.Assert(err, c.IsNil)

	// clear any cached configs
	s.Helper.config = nil
	s.Helper.controller = nil

	// use new key for deployer+controller
	set = flynn(t, "/", "-a", "controller", "env", "set", "AUTH_KEY="+newKey)
	t.Assert(set, Succeeds)

	// remove old key from API
	set = flynn(t, "/", "-a", "controller", "env", "unset", "-t", "web", "AUTH_KEY")
	t.Assert(set, Succeeds)
}
<|endoftext|>


<commit_before>package corepxe

import (
	"log"
	"net/http"

	"github.com/elazarl/goproxy"
)

func reponseHandler(r *http.Response, ctx *goproxy.ProxyCtx, corpxeChan chan []int) *http.Response {
	r.Header.Set("X-COREPXE", "corepxe")
	return r
}

func proxySetup(corpxeChan chan []int) {
	proxy := goproxy.NewProxyHttpServer()
	proxy.OnRequest(UrlIs("public.update.core-os.net/v1/update/")).HandleConnect(goproxy.AlwaysMitm)
	proxy.OnResponse().DoFunc(reponseHandler(corpxeChan))
	proxy.Verbose = true
	log.Fatal(http.ListenAndServe(":8080", proxy))
}
<commit_msg>fix typo<commit_after>package corepxe

import (
	"log"
	"net/http"

	"github.com/elazarl/goproxy"
)

func cpRespHandler(r *http.Response, ctx *goproxy.ProxyCtx, corpxeChan chan []int) *http.Response {
	r.Header.Set("X-COREPXE", "corepxe")
	return r
}

func proxySetup(corpxeChan chan []int) {
	proxy := goproxy.NewProxyHttpServer()
	proxy.OnRequest(UrlIs("public.update.core-os.net/v1/update/")).HandleConnect(goproxy.AlwaysMitm)
	proxy.OnResponse().DoFunc(reponseHandler(corpxeChan))
	proxy.Verbose = true
	log.Fatal(http.ListenAndServe(":8080", proxy))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"
	"sync"
	"time"

	"github.com/abbot/go-http-auth"
	"path/filepath"
)

const (
	browserStarted int = iota
	browserFailed
	seleniumError
)

const (
	pingPath  string = "/ping"
	errPath   string = "/err"
	routePath string = "/wd/hub/session"
	proxyPath string = routePath + "/"
	head      int    = len(proxyPath)
	tail      int    = head + 32
	sessPart  int    = 4 // /wd/hub/session/{various length session}
)

var (
	port     int
	quotaDir string
	users    string
	listen   string
	quota    map[string]Browsers = make(map[string]Browsers)
	routes   map[string]Routes   = make(map[string]Routes)
	num      uint64
	numLock  sync.Mutex
	confLock sync.RWMutex
)

type Routes map[string]*Host

type caps map[string]interface{}

func (c *caps) capability(k string) string {
	dc := (*c)["desiredCapabilities"]
	switch dc.(type) {
	case map[string]interface{}:
		v := dc.(map[string]interface{})
		switch v[k].(type) {
		case string:
			return v[k].(string)
		}
	}
	return ""
}

func (c *caps) browser() string {
	return c.capability("browserName")
}

func (c *caps) version() string {
	return c.capability("version")
}

func (h *Host) url() string {
	return fmt.Sprintf("http://%s%s", h.net(), routePath)
}

func (h *Host) session(c caps) (map[string]interface{}, int) {
	b, _ := json.Marshal(c)
	resp, err := http.Post(h.url(), "application/json", bytes.NewReader(b))
	if err != nil {
		return nil, seleniumError
	}
	var reply map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&reply)
	if err != nil {
		return nil, seleniumError
	}
	if resp.StatusCode != http.StatusOK {
		return reply, browserFailed
	}
	return reply, browserStarted
}

func reply(w http.ResponseWriter, msg map[string]interface{}) {
	reply, _ := json.Marshal(msg)
	w.Header().Set("Content-Type", "application/json")
	w.Write(reply)
}

func serial() uint64 {
	numLock.Lock()
	defer numLock.Unlock()
	id := num
	num++
	return id
}

func info(r *http.Request) (user, remote string) {
	user = "unknown"
	if u, _, ok := r.BasicAuth(); ok {
		user = u
	}
	remote = r.Header.Get("X-Forwarded-For")
	if remote != "" {
		return
	}
	remote, _, _ = net.SplitHostPort(r.RemoteAddr)
	return
}

func fmtBrowser(browser, version string) string {
	if version != "" {
		return fmt.Sprintf("%s-%s", browser, version)
	}
	return browser
}

func browserErrMsg(js map[string]interface{}) string {
	if js == nil {
		return ""
	}
	val, ok := js["value"].(map[string]interface{})
	if !ok {
		return ""
	}
	msg, ok := val["message"].(string)
	if !ok {
		return ""
	}
	return msg
}

func jsonErrMsg(msg string) string {
	message := make(map[string]string)
	message["message"] = msg
	value := make(map[string]interface{})
	value["value"] = message
	value["status"] = 13
	result, _ := json.Marshal(value)
	return string(result)
}

func route(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	id := serial()
	user, remote := info(r)
	var c caps
	err := json.NewDecoder(r.Body).Decode(&c)
	if err != nil {
		http.Error(w, fmt.Sprintf("bad json format: %s", err.Error()), http.StatusBadRequest)
		log.Printf("[%d] [BAD_JSON] [%s] [%s] [%v]\n", id, user, remote, err)
		return
	}
	browser, version := c.browser(), c.version()
	if browser == "" {
		http.Error(w, "browser not set", http.StatusBadRequest)
		log.Printf("[%d] [BROWSER_NOT_SET] [%s] [%s]\n", id, user, remote)
		return
	}
	count := 0
loop:
	for {
		confLock.RLock()
		browsers := quota[user]
		hosts := browsers.find(browser, version)
		confLock.RUnlock()
		if len(hosts) == 0 {
			http.Error(w, fmt.Sprintf("unsupported browser: %s", fmtBrowser(browser, version)), http.StatusNotFound)
			log.Printf("[%d] [UNSUPPORTED_BROWSER] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
			return
		}
		for h, i := hosts.choose(); ; h, i = hosts.choose() {
			count++
			if h == nil {
				break loop
			}
			log.Printf("[%d] [SESSION_ATTEMPTED] [%s] [%s] [%s] [%s] [%d]\n", id, user, remote, fmtBrowser(browser, version), h.net(), count)
			excludes := make([]string, 0)
			resp, status := h.session(c)
			switch status {
			case browserStarted:
				sess := resp["sessionId"].(string)
				resp["sessionId"] = h.sum() + sess
				reply(w, resp)
				log.Printf("[%d] [%.2fs] [SESSION_CREATED] [%s] [%s] [%s] [%s] [%s] [%d]\n", id, float64(time.Now().Sub(start).Seconds()), user, remote, fmtBrowser(browser, version), h.net(), sess, count)
				return
			case browserFailed:
				hosts = append(hosts[:i], hosts[i+1:]...)
			case seleniumError:
				excludes = append(excludes, h.region)
				hosts = browsers.find(browser, version, excludes...)
			}
			log.Printf("[%d] [SESSION_FAILED] [%s] [%s] [%s] [%s] %s\n", id, user, remote, fmtBrowser(browser, version), h.net(), browserErrMsg(resp))
			if len(hosts) == 0 {
				break loop
			}
		}
	}
	http.Error(w, jsonErrMsg(fmt.Sprintf("cannot create session %s on any hosts after %d attempt(s)", fmtBrowser(browser, version), count)), http.StatusInternalServerError)
	log.Printf("[%d] [SESSION_NOT_CREATED] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
}

func proxy(r *http.Request) {
	user, remote := info(r)
	r.URL.Scheme = "http"
	if len(r.URL.Path) > tail {
		sum := r.URL.Path[head:tail]
		proxyPath := r.URL.Path[:head] + r.URL.Path[tail:]
		userRoutes := routes[user]
		if h, ok := userRoutes[sum]; ok {
			if body, err := ioutil.ReadAll(r.Body); err == nil {
				r.Body.Close()
				var msg map[string]interface{}
				if err := json.Unmarshal(body, &msg); err == nil {
					delete(msg, "sessionId")
					body, _ = json.Marshal(msg)
					r.ContentLength = int64(len(body))
				}
				r.Body = ioutil.NopCloser(bytes.NewReader(body))
			}
			r.URL.Host = h.net()
			r.URL.Path = proxyPath
			if r.Method == "DELETE" {
				sess := strings.Split(proxyPath, "/")[sessPart]
				log.Printf("[SESSION_DELETED] [%s] [%s] [%s] [%s]\n", user, remote, h.net(), sess)
			}
			return
		}
	}
	r.URL.Host = listen
	r.URL.Path = errPath
}

func ping(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("Ok\n"))
}

func err(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "route not found", http.StatusNotFound)
}

func postOnly(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}
		handler(w, r)
	}
}

func readConfig(fn string, browsers *Browsers) error {
	file, err := ioutil.ReadFile(fn)
	if err != nil {
		return errors.New(fmt.Sprintf("error reading configuration file %s: %v", fn, err))
	}
	if err := xml.Unmarshal(file, browsers); err != nil {
		return errors.New(fmt.Sprintf("error parsing configuration file %s: %v", fn, err))
	}
	return nil
}

func createRoutes(config *Browsers) Routes {
	routes := make(Routes)
	for _, b := range config.Browsers {
		for _, v := range b.Versions {
			for _, r := range v.Regions {
				for i, h := range r.Hosts {
					r.Hosts[i].region = r.Name
					routes[h.sum()] = &r.Hosts[i]
				}
			}
		}
	}
	return routes
}

func parseArgs() {
	flag.IntVar(&port, "port", 4444, "port to bind to")
	flag.StringVar(&quotaDir, "quotaDir", "quota", "quota directory")
	flag.StringVar(&users, "users", ".htpasswd", "htpasswd auth file path")
	flag.Parse()
	listen = fmt.Sprintf(":%d", port)
}

func loadConfig() {
	log.Printf("Users file is [%s]\n", users)
	err := loadQuotaFiles(quotaDir)
	if err != nil {
		log.Fatalf("%v\n", err)
	}
}

func loadQuotaFiles(quotaDir string) error {
	log.Printf("Loading configuration files from [%s]\n", quotaDir)

	confLock.Lock()
	defer confLock.Unlock()
	glob := fmt.Sprintf("%s%c%s", quotaDir, filepath.Separator, "*.xml")
	files, _ := filepath.Glob(glob)
	if len(files) == 0 {
		return errors.New(fmt.Sprintf("no quota XML files found in [%s] - exiting\n", quotaDir))
	}

	for _, file := range files {
		loadQuotaFile(file)
	}
	return nil
}

func loadQuotaFile(file string) {
	fileName := filepath.Base(file)
	quotaName := strings.TrimSuffix(fileName, filepath.Ext(fileName))
	var browsers Browsers
	err := readConfig(file, &browsers)
	if err != nil {
		log.Printf("Failed to load configuration from [%s]: %v", fileName, err)
		return
	}
	quota[quotaName] = browsers
	routes[quotaName] = createRoutes(&browsers)
	log.Printf("Loaded configuration from [%s]:\n%v\n", file, browsers)
}

func requireBasicAuth(authenticator *auth.BasicAuth, handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return authenticator.Wrap(func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		handler(w, &r.Request)
	})
}

func mux() http.Handler {
	mux := http.NewServeMux()
	authenticator := auth.NewBasicAuthenticator(
		"Selenium Grid Router",
		auth.HtpasswdFileProvider(users),
	)

	mux.HandleFunc(pingPath, ping)
	mux.HandleFunc(errPath, err)
	mux.HandleFunc(routePath, requireBasicAuth(authenticator, postOnly(route)))
	mux.Handle(proxyPath, &httputil.ReverseProxy{Director: proxy})
	return mux
}

func init() {
	parseArgs()
	loadConfig()
}

func main() {
	log.Println("listening on", listen)
	log.Print(http.ListenAndServe(listen, mux()))
}
<commit_msg>Moved log.Fatalf() to init()<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"
	"sync"
	"time"

	"github.com/abbot/go-http-auth"
	"path/filepath"
)

const (
	browserStarted int = iota
	browserFailed
	seleniumError
)

const (
	pingPath  string = "/ping"
	errPath   string = "/err"
	routePath string = "/wd/hub/session"
	proxyPath string = routePath + "/"
	head      int    = len(proxyPath)
	tail      int    = head + 32
	sessPart  int    = 4 // /wd/hub/session/{various length session}
)

var (
	port     int
	quotaDir string
	users    string
	listen   string
	quota    map[string]Browsers = make(map[string]Browsers)
	routes   map[string]Routes   = make(map[string]Routes)
	num      uint64
	numLock  sync.Mutex
	confLock sync.RWMutex
)

type Routes map[string]*Host

type caps map[string]interface{}

func (c *caps) capability(k string) string {
	dc := (*c)["desiredCapabilities"]
	switch dc.(type) {
	case map[string]interface{}:
		v := dc.(map[string]interface{})
		switch v[k].(type) {
		case string:
			return v[k].(string)
		}
	}
	return ""
}

func (c *caps) browser() string {
	return c.capability("browserName")
}

func (c *caps) version() string {
	return c.capability("version")
}

func (h *Host) url() string {
	return fmt.Sprintf("http://%s%s", h.net(), routePath)
}

func (h *Host) session(c caps) (map[string]interface{}, int) {
	b, _ := json.Marshal(c)
	resp, err := http.Post(h.url(), "application/json", bytes.NewReader(b))
	if err != nil {
		return nil, seleniumError
	}
	var reply map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&reply)
	if err != nil {
		return nil, seleniumError
	}
	if resp.StatusCode != http.StatusOK {
		return reply, browserFailed
	}
	return reply, browserStarted
}

func reply(w http.ResponseWriter, msg map[string]interface{}) {
	reply, _ := json.Marshal(msg)
	w.Header().Set("Content-Type", "application/json")
	w.Write(reply)
}

func serial() uint64 {
	numLock.Lock()
	defer numLock.Unlock()
	id := num
	num++
	return id
}

func info(r *http.Request) (user, remote string) {
	user = "unknown"
	if u, _, ok := r.BasicAuth(); ok {
		user = u
	}
	remote = r.Header.Get("X-Forwarded-For")
	if remote != "" {
		return
	}
	remote, _, _ = net.SplitHostPort(r.RemoteAddr)
	return
}

func fmtBrowser(browser, version string) string {
	if version != "" {
		return fmt.Sprintf("%s-%s", browser, version)
	}
	return browser
}

func browserErrMsg(js map[string]interface{}) string {
	if js == nil {
		return ""
	}
	val, ok := js["value"].(map[string]interface{})
	if !ok {
		return ""
	}
	msg, ok := val["message"].(string)
	if !ok {
		return ""
	}
	return msg
}

func jsonErrMsg(msg string) string {
	message := make(map[string]string)
	message["message"] = msg
	value := make(map[string]interface{})
	value["value"] = message
	value["status"] = 13
	result, _ := json.Marshal(value)
	return string(result)
}

func route(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	id := serial()
	user, remote := info(r)
	var c caps
	err := json.NewDecoder(r.Body).Decode(&c)
	if err != nil {
		http.Error(w, fmt.Sprintf("bad json format: %s", err.Error()), http.StatusBadRequest)
		log.Printf("[%d] [BAD_JSON] [%s] [%s] [%v]\n", id, user, remote, err)
		return
	}
	browser, version := c.browser(), c.version()
	if browser == "" {
		http.Error(w, "browser not set", http.StatusBadRequest)
		log.Printf("[%d] [BROWSER_NOT_SET] [%s] [%s]\n", id, user, remote)
		return
	}
	count := 0
loop:
	for {
		confLock.RLock()
		browsers := quota[user]
		hosts := browsers.find(browser, version)
		confLock.RUnlock()
		if len(hosts) == 0 {
			http.Error(w, fmt.Sprintf("unsupported browser: %s", fmtBrowser(browser, version)), http.StatusNotFound)
			log.Printf("[%d] [UNSUPPORTED_BROWSER] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
			return
		}
		for h, i := hosts.choose(); ; h, i = hosts.choose() {
			count++
			if h == nil {
				break loop
			}
			log.Printf("[%d] [SESSION_ATTEMPTED] [%s] [%s] [%s] [%s] [%d]\n", id, user, remote, fmtBrowser(browser, version), h.net(), count)
			excludes := make([]string, 0)
			resp, status := h.session(c)
			switch status {
			case browserStarted:
				sess := resp["sessionId"].(string)
				resp["sessionId"] = h.sum() + sess
				reply(w, resp)
				log.Printf("[%d] [%.2fs] [SESSION_CREATED] [%s] [%s] [%s] [%s] [%s] [%d]\n", id, float64(time.Now().Sub(start).Seconds()), user, remote, fmtBrowser(browser, version), h.net(), sess, count)
				return
			case browserFailed:
				hosts = append(hosts[:i], hosts[i+1:]...)
			case seleniumError:
				excludes = append(excludes, h.region)
				hosts = browsers.find(browser, version, excludes...)
			}
			log.Printf("[%d] [SESSION_FAILED] [%s] [%s] [%s] [%s] %s\n", id, user, remote, fmtBrowser(browser, version), h.net(), browserErrMsg(resp))
			if len(hosts) == 0 {
				break loop
			}
		}
	}
	http.Error(w, jsonErrMsg(fmt.Sprintf("cannot create session %s on any hosts after %d attempt(s)", fmtBrowser(browser, version), count)), http.StatusInternalServerError)
	log.Printf("[%d] [SESSION_NOT_CREATED] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
}

func proxy(r *http.Request) {
	user, remote := info(r)
	r.URL.Scheme = "http"
	if len(r.URL.Path) > tail {
		sum := r.URL.Path[head:tail]
		proxyPath := r.URL.Path[:head] + r.URL.Path[tail:]
		userRoutes := routes[user]
		if h, ok := userRoutes[sum]; ok {
			if body, err := ioutil.ReadAll(r.Body); err == nil {
				r.Body.Close()
				var msg map[string]interface{}
				if err := json.Unmarshal(body, &msg); err == nil {
					delete(msg, "sessionId")
					body, _ = json.Marshal(msg)
					r.ContentLength = int64(len(body))
				}
				r.Body = ioutil.NopCloser(bytes.NewReader(body))
			}
			r.URL.Host = h.net()
			r.URL.Path = proxyPath
			if r.Method == "DELETE" {
				sess := strings.Split(proxyPath, "/")[sessPart]
				log.Printf("[SESSION_DELETED] [%s] [%s] [%s] [%s]\n", user, remote, h.net(), sess)
			}
			return
		}
	}
	r.URL.Host = listen
	r.URL.Path = errPath
}

func ping(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("Ok\n"))
}

func err(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "route not found", http.StatusNotFound)
}

func postOnly(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}
		handler(w, r)
	}
}

func readConfig(fn string, browsers *Browsers) error {
	file, err := ioutil.ReadFile(fn)
	if err != nil {
		return errors.New(fmt.Sprintf("error reading configuration file %s: %v", fn, err))
	}
	if err := xml.Unmarshal(file, browsers); err != nil {
		return errors.New(fmt.Sprintf("error parsing configuration file %s: %v", fn, err))
	}
	return nil
}

func createRoutes(config *Browsers) Routes {
	routes := make(Routes)
	for _, b := range config.Browsers {
		for _, v := range b.Versions {
			for _, r := range v.Regions {
				for i, h := range r.Hosts {
					r.Hosts[i].region = r.Name
					routes[h.sum()] = &r.Hosts[i]
				}
			}
		}
	}
	return routes
}

func parseArgs() {
	flag.IntVar(&port, "port", 4444, "port to bind to")
	flag.StringVar(&quotaDir, "quotaDir", "quota", "quota directory")
	flag.StringVar(&users, "users", ".htpasswd", "htpasswd auth file path")
	flag.Parse()
	listen = fmt.Sprintf(":%d", port)
}

func loadConfig() error {
	log.Printf("Users file is [%s]\n", users)
	return loadQuotaFiles(quotaDir)
}

func loadQuotaFiles(quotaDir string) error {
	log.Printf("Loading configuration files from [%s]\n", quotaDir)

	confLock.Lock()
	defer confLock.Unlock()
	glob := fmt.Sprintf("%s%c%s", quotaDir, filepath.Separator, "*.xml")
	files, _ := filepath.Glob(glob)
	if len(files) == 0 {
		return errors.New(fmt.Sprintf("no quota XML files found in [%s] - exiting\n", quotaDir))
	}

	for _, file := range files {
		loadQuotaFile(file)
	}
	return nil
}

func loadQuotaFile(file string) {
	fileName := filepath.Base(file)
	quotaName := strings.TrimSuffix(fileName, filepath.Ext(fileName))
	var browsers Browsers
	err := readConfig(file, &browsers)
	if err != nil {
		log.Printf("Failed to load configuration from [%s]: %v", fileName, err)
		return
	}
	quota[quotaName] = browsers
	routes[quotaName] = createRoutes(&browsers)
	log.Printf("Loaded configuration from [%s]:\n%v\n", file, browsers)
}

func requireBasicAuth(authenticator *auth.BasicAuth, handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return authenticator.Wrap(func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		handler(w, &r.Request)
	})
}

func mux() http.Handler {
	mux := http.NewServeMux()
	authenticator := auth.NewBasicAuthenticator(
		"Selenium Grid Router",
		auth.HtpasswdFileProvider(users),
	)

	mux.HandleFunc(pingPath, ping)
	mux.HandleFunc(errPath, err)
	mux.HandleFunc(routePath, requireBasicAuth(authenticator, postOnly(route)))
	mux.Handle(proxyPath, &httputil.ReverseProxy{Director: proxy})
	return mux
}

func init() {
	parseArgs()
	err := loadConfig()
	if err != nil {
		log.Fatalf("%v\n", err)
	}
}

func main() {
	log.Println("listening on", listen)
	log.Print(http.ListenAndServe(listen, mux()))
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
)

type Proxy struct {
	config           Config
	contentTypeRegex *regexp.Regexp
}

func newProxy(config Config) *Proxy {
	// Todo, add config error checking

	contentTypeRegex, err := regexp.Compile(config.allowedContentTypes)
	if err != nil {
		log.Panic(err)
	}

	return &Proxy{
		config:           config,
		contentTypeRegex: contentTypeRegex,
	}
}

func (p Proxy) handler(w http.ResponseWriter, r *http.Request) {
	params := r.URL.Query()

	if params.Get("url") != "" {
		p.proxyRequest(w, params)
	} else {
		log.Println("No request url to proxy")
		http.Error(w, "No request url to proxy", http.StatusBadRequest)
	}
}

func (p Proxy) proxyRequest(w http.ResponseWriter, params url.Values) {
	url := params.Get("url")
	resp, err := http.Get(url) // http.Get follows up to 10 redirects
	if err != nil {
		log.Print(err)
		// Todo, handle specific errors
		http.Error(w, "Could not proxy", http.StatusInternalServerError)
	}

	if p.contentTypeRegex.MatchString(resp.Header.Get("Content-Type")) {
		p.writeResponse(w, resp)
	} else {
		log.Println("Upstream content doesn't match configured allowedContentTypes")
		http.Error(w, "Upstream content doesn't match configured allowedContentTypes", http.StatusBadRequest)
	}
}

func (p Proxy) writeResponse(w http.ResponseWriter, resp *http.Response) {
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	io.Copy(w, resp.Body)
	resp.Body.Close()
}
<commit_msg>Pull out request checks into validRequest helper function :lipstick:<commit_after>package main

import (
	"errors"
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
)

type Proxy struct {
	config           Config
	contentTypeRegex *regexp.Regexp
}

func newProxy(config Config) *Proxy {
	// Todo, add config error checking

	contentTypeRegex, err := regexp.Compile(config.allowedContentTypes)
	if err != nil {
		log.Panic(err)
	}

	return &Proxy{
		config:           config,
		contentTypeRegex: contentTypeRegex,
	}
}

func (p Proxy) handler(w http.ResponseWriter, r *http.Request) {
	params := r.URL.Query()

	if err := p.validRequest(params); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
	} else {
		p.proxyRequest(w, params)
	}
}

func (p Proxy) validRequest(params url.Values) error {
	url := params.Get("url")
	if url == "" {
		return errors.New("No request url to proxy")
	}

	return nil
}

func (p Proxy) proxyRequest(w http.ResponseWriter, params url.Values) {
	url := params.Get("url")
	resp, err := http.Get(url) // http.Get follows up to 10 redirects
	if err != nil {
		log.Print(err)
		// Todo, handle specific errors
		http.Error(w, "Could not proxy", http.StatusInternalServerError)
	}

	if p.contentTypeRegex.MatchString(resp.Header.Get("Content-Type")) {
		p.writeResponse(w, resp)
	} else {
		log.Println("Upstream content doesn't match configured allowedContentTypes")
		http.Error(w, "Upstream content doesn't match configured allowedContentTypes", http.StatusBadRequest)
	}
}

func (p Proxy) writeResponse(w http.ResponseWriter, resp *http.Response) {
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	io.Copy(w, resp.Body)
	resp.Body.Close()
}
<|endoftext|>


<commit_before>package backend

import (
	"fmt"
	"time"
)

// CreateProxy creates a new proxy
func CreateProxy(ip string, port int, t ProxyType, p ProxyProtocol) *Proxy {
	proxy := &Proxy{
		IP:       ip,
		Port:     port,
		Type:     t,
		Protocol: p,
		Joined:   time.Now().Unix(),
	}
	proxy.GenerateIdentifier()
	return proxy
}

// GenerateIdentifier generates the unique identifier for the proxy
func (p *Proxy) GenerateIdentifier() {
	p.Identifier = fmt.Sprintf("%s:%d", p.IP, p.Port)
}
<commit_msg>Added HasKey function to check for key duplication<commit_after>package backend

import (
	"fmt"
	"time"
)

// CreateProxy creates a new proxy
func CreateProxy(ip string, port int, t ProxyType, p ProxyProtocol) *Proxy {
	proxy := &Proxy{
		IP:       ip,
		Port:     port,
		Type:     t,
		Protocol: p,
		Joined:   time.Now().Unix(),
	}
	proxy.GenerateIdentifier()
	return proxy
}

// GenerateIdentifier generates the unique identifier for the proxy
func (p *Proxy) GenerateIdentifier() {
	p.Identifier = fmt.Sprintf("%s:%d", p.IP, p.Port)
}

// HasKey checks if a check key is already present
func (p *Proxy) HasKey(key string) bool {
	for i := 0; i < len(p.Checks); i++ {
		if p.Checks[i].Key == key {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"sync"
	"time"

	"net"
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/hagen1778/chproxy/config"
	"github.com/hagen1778/chproxy/log"
	"github.com/prometheus/client_golang/prometheus"
)

// Creates new reverseProxy with provided config
func NewReverseProxy(cfg *config.Config) (*reverseProxy, error) {
	rp := &reverseProxy{}
	rp.ReverseProxy = &httputil.ReverseProxy{
		Director: func(*http.Request) {},
		ErrorLog: log.ErrorLogger,
		Transport: &observableTransport{
			http.Transport{
				DialContext: (&net.Dialer{
					KeepAlive: 30 * time.Second,
					DualStack: true,
				}).DialContext,
				MaxIdleConns:          100,
				IdleConnTimeout:       90 * time.Second,
				TLSHandshakeTimeout:   10 * time.Second,
				ExpectContinueTimeout: 1 * time.Second,
			},
		},
	}
	err := rp.ApplyConfig(cfg)

	return rp, err
}

func (rp *reverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Debugf("Accepting request from %s: %s", req.RemoteAddr, req.URL.String())
	s, err := rp.getRequestScope(req)
	if err != nil {
		respondWithErr(rw, err)
		return
	}
	log.Debugf("Request scope %s", s)

	if err = s.inc(); err != nil {
		respondWithErr(rw, err)
		return
	}
	defer s.dec()

	label := prometheus.Labels{
		"user":         s.user.name,
		"cluster_user": s.clusterUser.name,
		"host":         s.host.addr.Host,
	}
	requestSum.With(label).Inc()

	req.URL.Scheme = s.host.addr.Scheme
	req.URL.Host = s.host.addr.Host
	// set custom User-Agent for proper handling of killQuery func
	ua := fmt.Sprintf("ClickHouseProxy: %s", s.user.name)
	req.Header.Set("User-Agent", ua)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	req = req.WithContext(ctx)

	done := make(chan struct{})
	go func() {
		rp.ReverseProxy.ServeHTTP(rw, req)
		done <- struct{}{}
	}()

	select {
	case <-s.user.timeout():
		cancel()
		<-done

		userTimeouts.With(prometheus.Labels{
			"host": s.host.addr.Host,
			"user": s.user.name,
		}).Inc()
		condition := fmt.Sprintf("http_user_agent = '%s'", ua)
		s.cluster.killQueries(condition, s.user.maxExecutionTime.Seconds())
		message := fmt.Sprintf("timeout for user %q exceeded: %v", s.user.name, s.user.maxExecutionTime)
		rw.Write([]byte(message))
	case <-s.clusterUser.timeout():
		cancel()
		<-done

		clusterTimeouts.With(prometheus.Labels{
			"host":         s.host.addr.Host,
			"cluster_user": s.clusterUser.name,
		}).Inc()
		condition := fmt.Sprintf("user = '%s'", s.clusterUser.name)
		s.cluster.killQueries(condition, s.clusterUser.maxExecutionTime.Seconds())
		message := fmt.Sprintf("timeout for cluster user %q exceeded: %v", s.clusterUser.name, s.clusterUser.maxExecutionTime)
		rw.Write([]byte(message))
	case <-done:
		requestSuccess.With(label).Inc()
	}

	log.Debugf("Request scope %s successfully proxied", s)
}

// Applies provided config to reverseProxy
// New config will be applied only if non-nil error returned
func (rp *reverseProxy) ApplyConfig(cfg *config.Config) error {
	clusters := make(map[string]*cluster, len(cfg.Clusters))
	for _, c := range cfg.Clusters {
		hosts := make([]*host, len(c.Nodes))
		for i, node := range c.Nodes {
			addr, err := url.Parse(fmt.Sprintf("%s://%s", c.Scheme, node))
			if err != nil {
				return err
			}

			hosts[i] = &host{
				addr: addr,
			}
		}

		clusterUsers := make(map[string]*clusterUser, len(c.ClusterUsers))
		for _, u := range c.ClusterUsers {
			clusterUsers[u.Name] = &clusterUser{
				name:                 u.Name,
				password:             u.Password,
				maxConcurrentQueries: u.MaxConcurrentQueries,
				maxExecutionTime:     u.MaxExecutionTime,
			}
		}

		clusters[c.Name] = newCluster(hosts, clusterUsers)
	}

	users := make(map[string]*user, len(cfg.Users))
	for _, u := range cfg.Users {
		c, ok := clusters[u.ToCluster]
		if !ok {
			return fmt.Errorf("error while mapping user %q to cluster %q: no such cluster", u.Name, u.ToCluster)
		}

		if _, ok := c.users[u.ToUser]; !ok {
			return fmt.Errorf("error while mapping user %q to cluster's %q user %q: no such user", u.Name, u.ToCluster, u.ToUser)
		}

		users[u.Name] = &user{
			clusterUser: clusterUser{
				name:                 u.Name,
				password:             u.Password,
				maxConcurrentQueries: u.MaxConcurrentQueries,
				maxExecutionTime:     u.MaxExecutionTime,
			},
			toCluster:       u.ToCluster,
			toUser:          u.ToUser,
			allowedNetworks: u.AllowedNetworks,
		}
	}

	rp.Lock()
	rp.clusters = clusters
	rp.users = users
	rp.Unlock()

	return nil
}

type reverseProxy struct {
	*httputil.ReverseProxy

	sync.Mutex
	users    map[string]*user
	clusters map[string]*cluster
}

func (rp *reverseProxy) getRequestScope(req *http.Request) (*scope, error) {
	name, password := basicAuth(req)

	rp.Lock()
	defer rp.Unlock()

	u, ok := rp.users[name]
	if !ok {
		return nil, fmt.Errorf("invalid username or password for user %q", name)
	}

	if u.password != password {
		return nil, fmt.Errorf("invalid username or password for user %q", name)
	}

	if !isAllowedAddr(req.RemoteAddr, u.allowedNetworks) {
		return nil, fmt.Errorf("user %q is not allowed to access from %s", name, req.RemoteAddr)
	}

	c, ok := rp.clusters[u.toCluster]
	if !ok {
		return nil, fmt.Errorf("BUG: user %q matches to unknown cluster %q", u.name, u.toCluster)
	}

	cu, ok := c.users[u.toUser]
	if !ok {
		return nil, fmt.Errorf("BUG: user %q matches to unknown user %q at cluster %q", u.name, u.toUser, u.toCluster)
	}

	return newScope(u, cu, c), nil
}

type observableTransport struct {
	http.Transport
}

func (pt *observableTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	response, err := pt.Transport.RoundTrip(r)
	if response != nil {
		statusCodes.With(
			prometheus.Labels{"host": r.URL.Host, "code": response.Status},
		).Inc()
	}

	if err != nil {
		errors.With(
			prometheus.Labels{"host": r.URL.Host, "message": err.Error()},
		).Inc()
	}

	return response, err
}
<commit_msg>validate duplicating of users and clusters<commit_after>package main

import (
	"context"
	"fmt"
	"sync"
	"time"

	"net"
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/hagen1778/chproxy/config"
	"github.com/hagen1778/chproxy/log"
	"github.com/prometheus/client_golang/prometheus"
)

// Creates new reverseProxy with provided config
func NewReverseProxy(cfg *config.Config) (*reverseProxy, error) {
	rp := &reverseProxy{}
	rp.ReverseProxy = &httputil.ReverseProxy{
		Director: func(*http.Request) {},
		ErrorLog: log.ErrorLogger,
		Transport: &observableTransport{
			http.Transport{
				DialContext: (&net.Dialer{
					KeepAlive: 30 * time.Second,
					DualStack: true,
				}).DialContext,
				MaxIdleConns:          100,
				IdleConnTimeout:       90 * time.Second,
				TLSHandshakeTimeout:   10 * time.Second,
				ExpectContinueTimeout: 1 * time.Second,
			},
		},
	}
	err := rp.ApplyConfig(cfg)

	return rp, err
}

func (rp *reverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Debugf("Accepting request from %s: %s", req.RemoteAddr, req.URL.String())
	s, err := rp.getRequestScope(req)
	if err != nil {
		respondWithErr(rw, err)
		return
	}
	log.Debugf("Request scope %s", s)

	if err = s.inc(); err != nil {
		respondWithErr(rw, err)
		return
	}
	defer s.dec()

	label := prometheus.Labels{
		"user":         s.user.name,
		"cluster_user": s.clusterUser.name,
		"host":         s.host.addr.Host,
	}
	requestSum.With(label).Inc()

	req.URL.Scheme = s.host.addr.Scheme
	req.URL.Host = s.host.addr.Host
	// set custom User-Agent for proper handling of killQuery func
	ua := fmt.Sprintf("ClickHouseProxy: %s", s.user.name)
	req.Header.Set("User-Agent", ua)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	req = req.WithContext(ctx)

	done := make(chan struct{})
	go func() {
		rp.ReverseProxy.ServeHTTP(rw, req)
		done <- struct{}{}
	}()

	select {
	case <-s.user.timeout():
		cancel()
		<-done

		userTimeouts.With(prometheus.Labels{
			"host": s.host.addr.Host,
			"user": s.user.name,
		}).Inc()
		condition := fmt.Sprintf("http_user_agent = '%s'", ua)
		s.cluster.killQueries(condition, s.user.maxExecutionTime.Seconds())
		message := fmt.Sprintf("timeout for user %q exceeded: %v", s.user.name, s.user.maxExecutionTime)
		rw.Write([]byte(message))
	case <-s.clusterUser.timeout():
		cancel()
		<-done

		clusterTimeouts.With(prometheus.Labels{
			"host":         s.host.addr.Host,
			"cluster_user": s.clusterUser.name,
		}).Inc()
		condition := fmt.Sprintf("user = '%s'", s.clusterUser.name)
		s.cluster.killQueries(condition, s.clusterUser.maxExecutionTime.Seconds())
		message := fmt.Sprintf("timeout for cluster user %q exceeded: %v", s.clusterUser.name, s.clusterUser.maxExecutionTime)
		rw.Write([]byte(message))
	case <-done:
		requestSuccess.With(label).Inc()
	}

	log.Debugf("Request scope %s successfully proxied", s)
}

// Applies provided config to reverseProxy
// New config will be applied only if non-nil error returned
func (rp *reverseProxy) ApplyConfig(cfg *config.Config) error {
	clusters := make(map[string]*cluster, len(cfg.Clusters))
	for _, c := range cfg.Clusters {
		hosts := make([]*host, len(c.Nodes))
		for i, node := range c.Nodes {
			addr, err := url.Parse(fmt.Sprintf("%s://%s", c.Scheme, node))
			if err != nil {
				return err
			}

			hosts[i] = &host{
				addr: addr,
			}
		}

		clusterUsers := make(map[string]*clusterUser, len(c.ClusterUsers))
		for _, u := range c.ClusterUsers {
			if _, ok := clusterUsers[u.Name]; ok {
				return fmt.Errorf("cluster user %q already exists", u.Name)
			}

			clusterUsers[u.Name] = &clusterUser{
				name:                 u.Name,
				password:             u.Password,
				maxConcurrentQueries: u.MaxConcurrentQueries,
				maxExecutionTime:     u.MaxExecutionTime,
			}
		}

		if _, ok := clusters[c.Name]; ok {
			return fmt.Errorf("cluster %q already exists", c.Name)
		}
		clusters[c.Name] = newCluster(hosts, clusterUsers)
	}

	users := make(map[string]*user, len(cfg.Users))
	for _, u := range cfg.Users {
		c, ok := clusters[u.ToCluster]
		if !ok {
			return fmt.Errorf("error while mapping user %q to cluster %q: no such cluster", u.Name, u.ToCluster)
		}

		if _, ok := c.users[u.ToUser]; !ok {
			return fmt.Errorf("error while mapping user %q to cluster's %q user %q: no such user", u.Name, u.ToCluster, u.ToUser)
		}

		if _, ok := users[u.Name]; ok {
			return fmt.Errorf("user %q already exists", u.Name)
		}

		users[u.Name] = &user{
			clusterUser: clusterUser{
				name:                 u.Name,
				password:             u.Password,
				maxConcurrentQueries: u.MaxConcurrentQueries,
				maxExecutionTime:     u.MaxExecutionTime,
			},
			toCluster:       u.ToCluster,
			toUser:          u.ToUser,
			allowedNetworks: u.AllowedNetworks,
		}
	}

	rp.Lock()
	rp.clusters = clusters
	rp.users = users
	rp.Unlock()

	return nil
}

type reverseProxy struct {
	*httputil.ReverseProxy

	sync.Mutex
	users    map[string]*user
	clusters map[string]*cluster
}

func (rp *reverseProxy) getRequestScope(req *http.Request) (*scope, error) {
	name, password := basicAuth(req)

	rp.Lock()
	defer rp.Unlock()

	u, ok := rp.users[name]
	if !ok {
		return nil, fmt.Errorf("invalid username or password for user %q", name)
	}

	if u.password != password {
		return nil, fmt.Errorf("invalid username or password for user %q", name)
	}

	if !isAllowedAddr(req.RemoteAddr, u.allowedNetworks) {
		return nil, fmt.Errorf("user %q is not allowed to access from %s", name, req.RemoteAddr)
	}

	c, ok := rp.clusters[u.toCluster]
	if !ok {
		return nil, fmt.Errorf("BUG: user %q matches to unknown cluster %q", u.name, u.toCluster)
	}

	cu, ok := c.users[u.toUser]
	if !ok {
		return nil, fmt.Errorf("BUG: user %q matches to unknown user %q at cluster %q", u.name, u.toUser, u.toCluster)
	}

	return newScope(u, cu, c), nil
}

type observableTransport struct {
	http.Transport
}

func (pt *observableTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	response, err := pt.Transport.RoundTrip(r)
	if response != nil {
		statusCodes.With(
			prometheus.Labels{"host": r.URL.Host, "code": response.Status},
		).Inc()
	}

	if err != nil {
		errors.With(
			prometheus.Labels{"host": r.URL.Host, "message": err.Error()},
		).Inc()
	}

	return response, err
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	defaultPagination := &piazza.JsonPagination{
		PerPage: 10,
		Page:    0,
		Order:   piazza.PaginationOrderDescending,
		SortBy:  "createdOn",
	}
	pagination, err := piazza.NewJsonPagination(params, defaultPagination)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, pagination)
	} else {
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, dsl)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			return service.newInternalErrorResponse(err)
		}

		// just in case
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	pagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams,
) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.AsString("service", nil)
	if err != nil {
		return "", err
	}

	contains, err := params.AsString("contains", nil)
	if err != nil {
		return "", err
	}

	before, err := params.AsTime("before", nil)
	if err != nil {
		return "", err
	}

	after, err := params.AsTime("after", nil)
	if err != nil {
		return "", err
	}

	if service != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": *service,
			},
		})
	}

	if contains != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  contains,
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if after != nil || before != nil {
		rangeParams := map[string]time.Time{}

		if after != nil {
			rangeParams["gte"] = *after
		}

		if before != nil {
			rangeParams["lte"] = *before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"stamp": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}
<commit_msg>Add logging, update DSL<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	defaultPagination := &piazza.JsonPagination{
		PerPage: 10,
		Page:    0,
		Order:   piazza.PaginationOrderDescending,
		SortBy:  "createdOn",
	}
	pagination, err := piazza.NewJsonPagination(params, defaultPagination)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	log.Printf("dsl: %s", dsl)

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, pagination)
	} else {
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, dsl)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			return service.newInternalErrorResponse(err)
		}

		// just in case
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	pagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.AsString("service", nil)
	if err != nil {
		return "", err
	}

	contains, err := params.AsString("contains", nil)
	if err != nil {
		return "", err
	}

	before, err := params.AsTime("before", nil)
	if err != nil {
		return "", err
	}

	after, err := params.AsTime("after", nil)
	if err != nil {
		return "", err
	}

	if service != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": *service,
			},
		})
	}

	if contains != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  contains,
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if after != nil || before != nil {
		rangeParams := map[string]time.Time{}

		if after != nil {
			rangeParams["gte"] = *after
		}

		if before != nil {
			rangeParams["lte"] = *before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"createdOn": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}
<|endoftext|>


<commit_before>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"fmt"
	"os"
	"path"
	"runtime"
	"sync/atomic"
	"time"
)

type log struct {
	level      Level
	seqid      uint64
	pathname   string
	filename   string
	module     string
	lineno     int
	funcName   string
	thread     int
	threadName string
	process    int
	message    string
	time       time.Time
}

type field func(*logging, *log) interface{}

var fields = map[string]field{
	"name":            (*logging).lname,
	"seqid":           (*logging).nextSeqid,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
	"timestamp":       (*logging).timestamp,
}

const errString = "???"

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

// generate the runtime information, including pathname, function name,
// filename, line number.
func genRuntime(l *log) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// generate short function name
		fname := runtime.FuncForPC(pc).Name()
		fshort := fname
		for i := len(fname) - 1; i > 0; i-- {
			if fname[i] == '.' {
				fshort = fname[i+1:]
				break
			}
		}

		l.pathname = file
		l.funcName = fshort
		l.filename = path.Base(file)
		l.lineno = line
	} else {
		l.pathname = errString
		l.funcName = errString
		l.filename = errString
		l.lineno = -1
	}
}

func (logger *logging) lname(l *log) interface{} {
	return logger.name
}

func (logger *logging) nextSeqid(l *log) interface{} {
	if l.seqid == 0 {
		l.seqid = atomic.AddUint64(&(logger.seqid), 1)
	}
	return l.seqid
}

func (logger *logging) levelno(l *log) interface{} {
	return int(l.level)
}

func (logger *logging) levelname(l *log) interface{} {
	return levelNames[l.level]
}

func (logger *logging) pathname(l *log) interface{} {
	if l.pathname == "" {
		genRuntime(l)
	}
	return l.pathname
}

func (logger *logging) filename(l *log) interface{} {
	if l.filename == "" {
		genRuntime(l)
	}
	return l.filename
}

func (logger *logging) module(l *log) interface{} {
	return ""
}

func (logger *logging) lineno(l *log) interface{} {
	if l.lineno == 0 {
		genRuntime(l)
	}
	return l.lineno
}

func (logger *logging) funcName(l *log) interface{} {
	if l.funcName == "" {
		genRuntime(l)
	}
	return l.funcName
}

func (logger *logging) created(l *log) interface{} {
	return logger.startTime.UnixNano()
}

func (logger *logging) asctime(l *log) interface{} {
	if l.time.IsZero() {
		l.time = time.Now()
	}
	return l.time.String()
}

func (logger *logging) msecs(l *log) interface{} {
	return logger.startTime.Nanosecond()
}

func (logger *logging) timestamp(l *log) interface{} {
	if l.time.IsZero() {
		l.time = time.Now()
	}
	return l.time.UnixNano()
}

func (logger *logging) relativeCreated(l *log) interface{} {
	if l.time.IsZero() {
		l.time = time.Now()
	}
	return l.time.Sub(logger.startTime).Nanoseconds()
}

func (logger *logging) thread(l *log) interface{} {
	if l.thread == 0 {
		l.thread = int(GetGoId())
	}
	return l.thread
}

func (logger *logging) threadName(l *log) interface{} {
	if l.threadName == "" {
		l.threadName = fmt.Sprintf("Thread-%d", GetGoId())
	}
	return l.threadName
}

// Process ID
func (logger *logging) process(l *log) interface{} {
	if l.process == 0 {
		l.process = os.Getpid()
	}
	return l.process
}

func (logger *logging) message(l *log) interface{} {
	return l.message
}
<commit_msg>change time string format<commit_after>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"fmt"
	"os"
	"path"
	"runtime"
	"sync/atomic"
	"time"
)

type log struct {
	level      Level
	seqid      uint64
	pathname   string
	filename   string
	module     string
	lineno     int
	funcName   string
	thread     int
	threadName string
	process    int
	message    string
	time       time.Time
}

type field func(*logging, *log) interface{}

var fields = map[string]field{
	"name":            (*logging).lname,
	"seqid":           (*logging).nextSeqid,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
	"timestamp":       (*logging).timestamp,
}

const errString = "???"

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

// generate the runtime information, including pathname, function name,
// filename, line number.
func genRuntime(l *log) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// generate short function name
		fname := runtime.FuncForPC(pc).Name()
		fshort := fname
		for i := len(fname) - 1; i > 0; i-- {
			if fname[i] == '.' {
				fshort = fname[i+1:]
				break
			}
		}

		l.pathname = file
		l.funcName = fshort
		l.filename = path.Base(file)
		l.lineno = line
	} else {
		l.pathname = errString
		l.funcName = errString
		l.filename = errString
		l.lineno = -1
	}
}

func (logger *logging) lname(l *log) interface{} {
	return logger.name
}

func (logger *logging) nextSeqid(l *log) interface{} {
	if l.seqid == 0 {
		l.seqid = atomic.AddUint64(&(logger.seqid), 1)
	}
	return l.seqid
}

func (logger *logging) levelno(l *log) interface{} {
	return int(l.level)
}

func (logger *logging) levelname(l *log) interface{} {
	return levelNames[l.level]
}

func (logger *logging) pathname(l *log) interface{} {
	if l.pathname == "" {
		genRuntime(l)
	}
	return l.pathname
}

func (logger *logging) filename(l *log) interface{} {
	if l.filename == "" {
		genRuntime(l)
	}
	return l.filename
}

func (logger *logging) module(l *log) interface{} {
	return ""
}

func (logger *logging) lineno(l *log) interface{} {
	if l.lineno == 0 {
		genRuntime(l)
	}
	return l.lineno
}

func (logger *logging) funcName(l *log) interface{} {
	if l.funcName == "" {
		genRuntime(l)
	}
	return l.funcName
}

func (logger *logging) created(l *log) interface{} {
	return logger.startTime.UnixNano()
}

func (logger *logging) asctime(l *log) interface{} {
	if l.time.IsZero() {
		l.time = time.Now()
	}
	return l.time.Format("2006-01-02 15:04:05.999999999")
}

func (logger *logging) msecs(l *log) interface{} {
	return logger.startTime.Nanosecond()
}

func (logger *logging) timestamp(l *log) interface{} {
	if l.time.IsZero() {
		l.time = time.Now()
	}
	return l.time.UnixNano()
}

func (logger *logging) relativeCreated(l *log) interface{} {
	if l.time.IsZero() {
		l.time = time.Now()
	}
	return l.time.Sub(logger.startTime).Nanoseconds()
}

func (logger *logging) thread(l *log) interface{} {
	if l.thread == 0 {
		l.thread = int(GetGoId())
	}
	return l.thread
}

func (logger *logging) threadName(l *log) interface{} {
	if l.threadName == "" {
		l.threadName = fmt.Sprintf("Thread-%d", GetGoId())
	}
	return l.threadName
}

// Process ID
func (logger *logging) process(l *log) interface{} {
	if l.process == 0 {
		l.process = os.Getpid()
	}
	return l.process
}

func (logger *logging) message(l *log) interface{} {
	return l.message
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"

	"github.com/mdevilliers/redishappy"
	"github.com/mdevilliers/redishappy/configuration"
	"github.com/mdevilliers/redishappy/services/logger"
	"github.com/zenazn/goji/web"
)

var configFile string
var logPath string

func init() {
	flag.StringVar(&configFile, "config", "config.json", "Full path of the configuration JSON file.")
	flag.StringVar(&logPath, "log", "log", "Folder for the logging folder.")
}

func main() {

	flag.Parse()
	logger.InitLogging(logPath)

	config, err := configuration.LoadFromFile(configFile)

	if err != nil {
		logger.Error.Panicf("Error opening config file : %s", err.Error())
	}

	sane, errors := config.GetCurrentConfiguration().SanityCheckConfiguration(&configuration.ConfigContainsRequiredSections{})

	if !sane {

		for _, errorAsStr := range errors {
			logger.Error.Print(errorAsStr)
		}

		logger.Error.Panic("Configuration fails checks")
	}

	flipper := NewNoOpFlipper()
	engine := redishappy.NewRedisHappyEngine(flipper, config)
	engine.ConfigureHandlersAndServe(AddHandlers)
}

// example handler
func AddHandlers(mux *web.Mux) {
	logger.Info.Print("muxed!")
	mux.Get("/api/xxxx", hello)
}
func hello(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Hello!")
}
<commit_msg>removed extra logging<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"

	"github.com/mdevilliers/redishappy"
	"github.com/mdevilliers/redishappy/configuration"
	"github.com/mdevilliers/redishappy/services/logger"
	"github.com/zenazn/goji/web"
)

var configFile string
var logPath string

func init() {
	flag.StringVar(&configFile, "config", "config.json", "Full path of the configuration JSON file.")
	flag.StringVar(&logPath, "log", "log", "Folder for the logging folder.")
}

func main() {

	flag.Parse()
	logger.InitLogging(logPath)

	config, err := configuration.LoadFromFile(configFile)

	if err != nil {
		logger.Error.Panicf("Error opening config file : %s", err.Error())
	}

	sane, errors := config.GetCurrentConfiguration().SanityCheckConfiguration(&configuration.ConfigContainsRequiredSections{})

	if !sane {

		for _, errorAsStr := range errors {
			logger.Error.Print(errorAsStr)
		}

		logger.Error.Panic("Configuration fails checks")
	}

	flipper := NewNoOpFlipper()
	engine := redishappy.NewRedisHappyEngine(flipper, config)
	engine.ConfigureHandlersAndServe(AddHandlers)
}

// example handler
func AddHandlers(mux *web.Mux) {
	mux.Get("/api/xxxx", hello)
}
func hello(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Hello!")
}
<|endoftext|>


<commit_before>package manifest

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// "library" is the default "library directory"
// returns the parsed version of (in order):
//   if "repo" is a URL, the remote contents of that URL
//   if "repo" is a relative path like "./repo", that file
//   the file "library/repo"
// (repoName, tagName, man, err)
func Fetch(library, repo string) (string, string, *Manifest2822, error) {
	repoName := filepath.Base(repo)
	tagName := ""
	if tagIndex := strings.IndexRune(repoName, ':'); tagIndex > 0 {
		tagName = repoName[tagIndex+1:]
		repoName = repoName[:tagIndex]
		repo = strings.TrimSuffix(repo, ":"+tagName)
	}

	u, err := url.Parse(repo)
	if err == nil && u.IsAbs() {
		// must be remote URL!
		resp, err := http.Get(repo)
		if err != nil {
			return repoName, tagName, nil, err
		}
		defer resp.Body.Close()
		man, err := Parse(resp.Body)
		if tagName != "" && man.GetTag(tagName) == nil {
			return repoName, tagName, man, fmt.Errorf("tag not found in manifest for %q: %q", repoName, tagName)
		}
		return repoName, tagName, man, err
	}

	// try file paths
	filePaths := []string{}
	if filepath.IsAbs(repo) || strings.IndexRune(repo, filepath.Separator) >= 0 || strings.IndexRune(repo, '/') >= 0 {
		filePaths = append(filePaths, repo)
	}
	if !filepath.IsAbs(repo) {
		filePaths = append(filePaths, filepath.Join(library, repo))
	}
	for _, fileName := range filePaths {
		f, err := os.Open(fileName)
		if err != nil && !os.IsNotExist(err) {
			return repoName, tagName, nil, err
		}
		if err == nil {
			defer f.Close()
			man, err := Parse(f)
			if tagName != "" && man.GetTag(tagName) == nil {
				return repoName, tagName, man, fmt.Errorf("tag not found in manifest for %q: %q", repoName, tagName)
			}
			return repoName, tagName, man, err
		}
	}

	return repoName, tagName, nil, fmt.Errorf("unable to find a manifest named %q (in %q or as a remote URL)", repo, library)
}
<commit_msg>Allow only "http" and "https" URLs<commit_after>package manifest

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// "library" is the default "library directory"
// returns the parsed version of (in order):
//   if "repo" is a URL, the remote contents of that URL
//   if "repo" is a relative path like "./repo", that file
//   the file "library/repo"
// (repoName, tagName, man, err)
func Fetch(library, repo string) (string, string, *Manifest2822, error) {
	repoName := filepath.Base(repo)
	tagName := ""
	if tagIndex := strings.IndexRune(repoName, ':'); tagIndex > 0 {
		tagName = repoName[tagIndex+1:]
		repoName = repoName[:tagIndex]
		repo = strings.TrimSuffix(repo, ":"+tagName)
	}

	u, err := url.Parse(repo)
	if err == nil && u.IsAbs() && (u.Scheme == "http" || u.Scheme == "https") {
		// must be remote URL!
		resp, err := http.Get(repo)
		if err != nil {
			return repoName, tagName, nil, err
		}
		defer resp.Body.Close()
		man, err := Parse(resp.Body)
		if tagName != "" && man.GetTag(tagName) == nil {
			return repoName, tagName, man, fmt.Errorf("tag not found in manifest for %q: %q", repoName, tagName)
		}
		return repoName, tagName, man, err
	}

	// try file paths
	filePaths := []string{}
	if filepath.IsAbs(repo) || strings.IndexRune(repo, filepath.Separator) >= 0 || strings.IndexRune(repo, '/') >= 0 {
		filePaths = append(filePaths, repo)
	}
	if !filepath.IsAbs(repo) {
		filePaths = append(filePaths, filepath.Join(library, repo))
	}
	for _, fileName := range filePaths {
		f, err := os.Open(fileName)
		if err != nil && !os.IsNotExist(err) {
			return repoName, tagName, nil, err
		}
		if err == nil {
			defer f.Close()
			man, err := Parse(f)
			if tagName != "" && man.GetTag(tagName) == nil {
				return repoName, tagName, man, fmt.Errorf("tag not found in manifest for %q: %q", repoName, tagName)
			}
			return repoName, tagName, man, err
		}
	}

	return repoName, tagName, nil, fmt.Errorf("unable to find a manifest named %q (in %q or as a remote URL)", repo, library)
}
<|endoftext|>