<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/ikravets/errs"
	"github.com/jessevdk/go-flags"

	"my/ev/efh"
)

type cmdEfhSuite struct {
	Exchange    string   `long:"exch"  short:"e" value-name:"EXCHANGE" default:"nasdaq" description:"nasdaq, bats, etc."`
	Suites      []string `long:"suite" short:"s" value-name:"SUITE"`
	Tests       []string `long:"test"  short:"t" value-name:"TEST"`
	Speed       int      `long:"speed" value-name:"NUM" default:"50000"`
	Limit       int      `long:"limit" value-name:"NUM"`
	TestEfh     string   `long:"test-efh (default: system-installed version)"`
	Local       bool     `long:"local"`
	EfhLoglevel int      `long:"efh-loglevel" default:"6"`
	EfhProf     bool     `long:"efh-prof"`

	shouldExecute bool
	topOutDirName string
	testRunsTotal int
	testRunsOk    int
}

func (c *cmdEfhSuite) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhSuite) ConfigParser(parser *flags.Parser) {
	parser.AddCommand("efh_suite", "run test_efh test suite", "", c)
}

func (c *cmdEfhSuite) ParsingFinished() (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
	})
	if !c.shouldExecute {
		return
	}
	c.topOutDirName = time.Now().Format("efh_regression.2006-01-02-15:04:05")
	suitesDirName := fmt.Sprintf("/local/dumps/%s/regression", c.Exchange)

	if len(c.Suites) == 0 || c.Suites[0] == "?" {
		suites, err2 := listDirs(suitesDirName)
		errs.CheckE(err2)
		if len(c.Suites) > 0 {
			fmt.Printf("suites: %s\n", suites)
			return
		}
		if len(suites) == 0 {
			log.Println("Warning: no suites found")
			return
		}
		c.Suites = suites
	}

	for _, suite := range c.Suites {
		var suiteDirName string
		if strings.HasPrefix(suite, "/") {
			suiteDirName = suite
		} else if strings.HasPrefix(suite, "./") {
			cwd, err := os.Getwd()
			errs.CheckE(err)
			suiteDirName = filepath.Join(cwd, suite)
		} else {
			suiteDirName = filepath.Join(suitesDirName, suite)
		}
		var tests []string
		if len(c.Tests) == 0 || c.Tests[0] == "?" {
			tests, err = listDirs(suiteDirName)
			errs.CheckE(err)
			if len(c.Tests) > 0 {
				fmt.Printf("suite %s tests: %v\n", suiteDirName, tests)
				continue
			}
		} else {
			tests = c.Tests
		}
		if len(tests) == 0 {
			log.Printf("Warning: no tests found in suite %s\n", suite)
			continue
		}

		for _, testName := range tests {
			testDirName := filepath.Join(suiteDirName, testName)
			subscriptionFileNames, err := filepath.Glob(filepath.Join(testDirName, "subscription*"))
			errs.CheckE(err)
			if len(subscriptionFileNames) == 0 {
				c.RunTest(testDirName, nil)
			} else {
				for _, sfn := range subscriptionFileNames {
					a := strings.SplitAfter(sfn, "/subscription")
					suf := ""
					if len(a) > 0 {
						suf = a[len(a)-1]
					}
					c.RunTest(testDirName, &suf)
				}
			}
		}
	}
	log.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	fmt.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	return
}
func (c *cmdEfhSuite) RunTest(testDirName string, suffix *string) (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
		err = ce
	})
	c.testRunsTotal++
	testRunName := time.Now().Format("2006-01-02-15:04:05.")
	tdnDir, tdnFile := filepath.Split(testDirName)
	tdnDir = filepath.Base(tdnDir)
	testRunName += tdnDir + "-" + tdnFile
	if suffix != nil {
		testRunName += *suffix
	}
	fmt.Printf("run %s\n", testRunName)
	outDirName := filepath.Join(c.topOutDirName, testRunName)
	errs.CheckE(os.MkdirAll(outDirName, 0777))
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "fail"), nil, 0666))
	expoutName := filepath.Join(testDirName, "expout-efh-orders")
	if suffix != nil {
		expoutName += *suffix
	}
	inputPcapName := filepath.Join(testDirName, "dump.pcap")
	_, err = os.Stat(expoutName)
	errs.CheckE(err)
	_, err = os.Stat(inputPcapName)
	errs.CheckE(err)
	efhDumpName := filepath.Join(outDirName, "expout_orders")
	errs.CheckE(os.Symlink(expoutName, efhDumpName))
	errs.CheckE(os.Symlink(testDirName, filepath.Join(outDirName, "dump_dir")))
	errs.CheckE(os.Symlink(inputPcapName, filepath.Join(outDirName, "dump.pcap")))
	conf := efh.ReplayConfig{
		InputFileName:   inputPcapName,
		OutputInterface: "eth1",
		Pps:             c.Speed,
		Limit:           c.Limit,
		Loop:            1,
		EfhLoglevel:     c.EfhLoglevel,
		EfhIgnoreGap:    true,
		EfhDump:         "expout_orders",
		EfhChannel:      c.genEfhChannels(testDirName),
		EfhProf:         c.EfhProf,
		TestEfh:         c.TestEfh,
		Local:           c.Local,
	}
	if suffix != nil {
		subscr := "subscription" + *suffix
		errs.CheckE(os.Symlink(filepath.Join(testDirName, subscr), filepath.Join(outDirName, subscr)))
		conf.EfhSubscribe = []string{subscr}
	}
	origWd, err := os.Getwd()
	errs.CheckE(err)
	errs.CheckE(os.Chdir(outDirName))
	er := efh.NewEfhReplay(conf)
	efhReplayErr := er.Run()
	errs.CheckE(os.Chdir(origWd))
	errs.CheckE(efhReplayErr)
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "ok"), nil, 0666))
	errs.CheckE(os.Remove(filepath.Join(outDirName, "fail")))
	c.testRunsOk++
	return
}
func (c *cmdEfhSuite) genEfhChannels(testDirName string) (channels []string) {
	inChannels := []string{c.Exchange}
	if contents, err := ioutil.ReadFile(filepath.Join(testDirName, "channels")); err == nil {
		errs.Check(len(contents) > 0, "channels file must not be empty")
		inChannels = strings.Fields(string(contents))
	}
	for _, c := range inChannels {
		switch c {
		case "nasdaq":
			for i := 0; i < 4; i++ {
				channels = append(channels, fmt.Sprintf("233.54.12.%d:%d", 1+i, 18001+i))
			}
		case "bats":
			for i := 0; i < 32; i++ {
				channels = append(channels, fmt.Sprintf("224.0.131.%d:%d", i/4, 30101+i))
			}
		case "bats-b":
			for i := 0; i < 32; i++ {
				channels = append(channels, fmt.Sprintf("233.130.124.%d:%d", i/4, 30101+i))
			}
		case "miax":
			for i := 0; i < 24; i++ {
				channels = append(channels, fmt.Sprintf("224.0.105.%d:%d", 1+i, 51001+i))
			}
		default:
			_, err := net.ResolveUDPAddr("udp", c)
			errs.CheckE(err)
			channels = append(channels, c)
		}
	}
	return
}

func listDirs(parent string) (children []string, err error) {
	defer errs.PassE(&err)
	matches, err := filepath.Glob(parent + "/*")
	errs.CheckE(err)
	for _, f := range matches {
		fi, err := os.Stat(f)
		errs.CheckE(err)
		if fi.IsDir() {
			children = append(children, fi.Name())
		}
	}
	return
}

func init() {
	var c cmdEfhSuite
	Registry.Register(&c)
}
<commit_msg>cmd:efh_suite: use channels.Config<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/ikravets/errs"
	"github.com/jessevdk/go-flags"

	"my/ev/channels"
	"my/ev/efh"
)

type cmdEfhSuite struct {
	Exchange    string   `long:"exch"  short:"e" value-name:"EXCHANGE" default:"nasdaq" description:"nasdaq, bats, etc."`
	Suites      []string `long:"suite" short:"s" value-name:"SUITE"`
	Tests       []string `long:"test"  short:"t" value-name:"TEST"`
	Speed       int      `long:"speed" value-name:"NUM" default:"50000"`
	Limit       int      `long:"limit" value-name:"NUM"`
	TestEfh     string   `long:"test-efh (default: system-installed version)"`
	Local       bool     `long:"local"`
	EfhLoglevel int      `long:"efh-loglevel" default:"6"`
	EfhProf     bool     `long:"efh-prof"`

	shouldExecute bool
	topOutDirName string
	testRunsTotal int
	testRunsOk    int
}

func (c *cmdEfhSuite) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhSuite) ConfigParser(parser *flags.Parser) {
	parser.AddCommand("efh_suite", "run test_efh test suite", "", c)
}

func (c *cmdEfhSuite) ParsingFinished() (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
	})
	if !c.shouldExecute {
		return
	}
	c.topOutDirName = time.Now().Format("efh_regression.2006-01-02-15:04:05")
	suitesDirName := fmt.Sprintf("/local/dumps/%s/regression", c.Exchange)

	if len(c.Suites) == 0 || c.Suites[0] == "?" {
		suites, err2 := listDirs(suitesDirName)
		errs.CheckE(err2)
		if len(c.Suites) > 0 {
			fmt.Printf("suites: %s\n", suites)
			return
		}
		if len(suites) == 0 {
			log.Println("Warning: no suites found")
			return
		}
		c.Suites = suites
	}

	for _, suite := range c.Suites {
		var suiteDirName string
		if strings.HasPrefix(suite, "/") {
			suiteDirName = suite
		} else if strings.HasPrefix(suite, "./") {
			cwd, err := os.Getwd()
			errs.CheckE(err)
			suiteDirName = filepath.Join(cwd, suite)
		} else {
			suiteDirName = filepath.Join(suitesDirName, suite)
		}
		var tests []string
		if len(c.Tests) == 0 || c.Tests[0] == "?" {
			tests, err = listDirs(suiteDirName)
			errs.CheckE(err)
			if len(c.Tests) > 0 {
				fmt.Printf("suite %s tests: %v\n", suiteDirName, tests)
				continue
			}
		} else {
			tests = c.Tests
		}
		if len(tests) == 0 {
			log.Printf("Warning: no tests found in suite %s\n", suite)
			continue
		}

		for _, testName := range tests {
			testDirName := filepath.Join(suiteDirName, testName)
			subscriptionFileNames, err := filepath.Glob(filepath.Join(testDirName, "subscription*"))
			errs.CheckE(err)
			if len(subscriptionFileNames) == 0 {
				c.RunTest(testDirName, nil)
			} else {
				for _, sfn := range subscriptionFileNames {
					a := strings.SplitAfter(sfn, "/subscription")
					suf := ""
					if len(a) > 0 {
						suf = a[len(a)-1]
					}
					c.RunTest(testDirName, &suf)
				}
			}
		}
	}
	log.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	fmt.Printf("Tests OK/Total: %d/%d\n", c.testRunsOk, c.testRunsTotal)
	return
}
func (c *cmdEfhSuite) RunTest(testDirName string, suffix *string) (err error) {
	defer errs.Catch(func(ce errs.CheckerError) {
		log.Printf("caught %s\n", ce)
		err = ce
	})
	c.testRunsTotal++
	testRunName := time.Now().Format("2006-01-02-15:04:05.")
	tdnDir, tdnFile := filepath.Split(testDirName)
	tdnDir = filepath.Base(tdnDir)
	testRunName += tdnDir + "-" + tdnFile
	if suffix != nil {
		testRunName += *suffix
	}
	fmt.Printf("run %s\n", testRunName)
	outDirName := filepath.Join(c.topOutDirName, testRunName)
	errs.CheckE(os.MkdirAll(outDirName, 0777))
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "fail"), nil, 0666))
	expoutName := filepath.Join(testDirName, "expout-efh-orders")
	if suffix != nil {
		expoutName += *suffix
	}
	inputPcapName := filepath.Join(testDirName, "dump.pcap")
	_, err = os.Stat(expoutName)
	errs.CheckE(err)
	_, err = os.Stat(inputPcapName)
	errs.CheckE(err)
	efhDumpName := filepath.Join(outDirName, "expout_orders")
	errs.CheckE(os.Symlink(expoutName, efhDumpName))
	errs.CheckE(os.Symlink(testDirName, filepath.Join(outDirName, "dump_dir")))
	errs.CheckE(os.Symlink(inputPcapName, filepath.Join(outDirName, "dump.pcap")))
	conf := efh.ReplayConfig{
		InputFileName:   inputPcapName,
		OutputInterface: "eth1",
		Pps:             c.Speed,
		Limit:           c.Limit,
		Loop:            1,
		EfhLoglevel:     c.EfhLoglevel,
		EfhIgnoreGap:    true,
		EfhDump:         "expout_orders",
		EfhChannel:      c.genEfhChannels(testDirName),
		EfhProf:         c.EfhProf,
		TestEfh:         c.TestEfh,
		Local:           c.Local,
	}
	if suffix != nil {
		subscr := "subscription" + *suffix
		errs.CheckE(os.Symlink(filepath.Join(testDirName, subscr), filepath.Join(outDirName, subscr)))
		conf.EfhSubscribe = []string{subscr}
	}
	origWd, err := os.Getwd()
	errs.CheckE(err)
	errs.CheckE(os.Chdir(outDirName))
	er := efh.NewEfhReplay(conf)
	efhReplayErr := er.Run()
	errs.CheckE(os.Chdir(origWd))
	errs.CheckE(efhReplayErr)
	errs.CheckE(ioutil.WriteFile(filepath.Join(outDirName, "ok"), nil, 0666))
	errs.CheckE(os.Remove(filepath.Join(outDirName, "fail")))
	c.testRunsOk++
	return
}
func (c *cmdEfhSuite) genEfhChannels(testDirName string) []string {
	cc := channels.NewConfig()
	if file, err := os.Open(filepath.Join(testDirName, "channels")); err == nil {
		defer file.Close()
		errs.CheckE(cc.LoadFromReader(file))
	} else {
		errs.CheckE(cc.LoadFromStr(c.Exchange))
	}
	return cc.Addrs()
}

func listDirs(parent string) (children []string, err error) {
	defer errs.PassE(&err)
	matches, err := filepath.Glob(parent + "/*")
	errs.CheckE(err)
	for _, f := range matches {
		fi, err := os.Stat(f)
		errs.CheckE(err)
		if fi.IsDir() {
			children = append(children, fi.Name())
		}
	}
	return
}

func init() {
	var c cmdEfhSuite
	Registry.Register(&c)
}
<|endoftext|>


<commit_before>/* Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"strings"
	"bytes"
	"regexp"
	"strconv"
	"sort"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"go/build"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

const (
	projectDirName = "github.com/emepyc/guiHive"
)

var (
	port string
	isVersion = regexp.MustCompile(`^[0-9]+$`)
)

func init() {
	flag.StringVar(&port, "port", "8080", "Port to listen (defaults to 8080)")
	flag.Parse()
}

func checkError(s string, err error, ss ...string) {
	if err != nil {
		log.Fatal(s, err, ss)
	}
}

// Sortable os.fileInfos by name (-> num)
type sortableFiles []os.FileInfo
func (s sortableFiles) Len () int {
	return len(s)
}
func (s sortableFiles) Less (i, j int) bool {
	iVer, err := strconv.Atoi(s[i].Name())
	checkError(fmt.Sprintf("Dir name %s can't be converted to int", s[i].Name()), err)
	jVer, err := strconv.Atoi(s[j].Name())
	checkError(fmt.Sprintf("Dir name %s can't be converted to int", s[j].Name()), err)
	return iVer < jVer;
}
func (s sortableFiles) Swap (i, j int) {
	s[i], s[j] = s[j], s[i]
}

func version(r *http.Request) string {
	parts := strings.SplitN(r.URL.Path, "/", 4)
	version := parts[2]
	if (isVersion.MatchString(version)) {
		return version
	} else {
		path := os.Getenv("GUIHIVE_BASEDIR") + "/versions/"
		dir, err := os.Open(path)
		checkError("Can't open dir " + path, err)
		files, err := dir.Readdir(-1)
		checkError("Can't read dir " + path, err)
		sort.Sort(sortableFiles(files))
		version = files[len(files)-1].Name()
		return version
	}
	return ""
}

func unknown(w http.ResponseWriter, r *http.Request) {
	version := version(r)
	fmt.Fprintln(w, r.URL)
	fmt.Fprintf(w, "version %s is currently not supported by guiHive\n", version)
}

func scriptHandler(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	defer r.Body.Close()
	checkError("Can't parse Form: ", err)

	debug("METHOD: %s", r.Method)
	debug("URL: %s", r.URL)

	var outMsg bytes.Buffer
	var errMsg bytes.Buffer

	fname := os.Getenv("GUIHIVE_BASEDIR") + r.URL.Path
	args, err := json.Marshal(r.Form)
	checkError("Can't Marshal JSON:", err)

	debug("EXECUTING SCRIPT: %s", fname)
	debug("ARGS: %s", args)
	version := version(r);
	debug("VERSION: %s", version)

	versionRootDir := os.Getenv("GUIHIVE_BASEDIR") + "/versions/" + version;
	ehiveRootDir := versionRootDir + "/ensembl-hive"
	ehiveRootLib := ehiveRootDir + "/modules"
	guihiveRootLib := versionRootDir + "/scripts/lib"
	newPerl5Lib  := addPerl5Lib(ehiveRootLib + ":" + guihiveRootLib)

	debug("EHIVE_ROOT_DIR: %s", ehiveRootDir)
	debug("NEW_PERL5LIB: %s", newPerl5Lib)
	cmd := exec.Command(fname, string(args))
	cmd.Env = make([]string,0)
	cmd.Env = append(cmd.Env, "PERL5LIB=" + newPerl5Lib)
	cmd.Env = append(cmd.Env, "EHIVE_ROOT_DIR=" + ehiveRootDir)
	cmd.Env = append(cmd.Env, "GUIHIVE_BASEDIR=" + os.Getenv("GUIHIVE_BASEDIR"))
	cmd.Env = append(cmd.Env, "PATH=" + os.Getenv("PATH"))

	cmd.Stdout = &outMsg
	cmd.Stderr = &errMsg

	if err := cmd.Start(); err != nil {
		log.Println("Error Starting Command: ", err)
	}
	if err := cmd.Wait(); err != nil {
		log.Println("Error Executing Command: ", err)
	}

	debug("OUTMSG: %s", outMsg.Bytes())
	debug("ERRMSG: %s", errMsg.Bytes())
	fmt.Fprintln(w, string(outMsg.Bytes()))

}

func pathExists(name string) bool {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false
	}
	return err == nil
}

func guessProjectDir() (string, error) {
	// First, we try to find the project dir in the working directory
	serverPath := os.Args[0]
	serverDir := filepath.Dir(serverPath)
	pathToIndex := serverDir + "/../index.html"
	absPathToIndex, err := filepath.Abs(pathToIndex)
	if err != nil {
		debug("ABSPATHTOINDEX: %s\n", absPathToIndex)
		return "", err
	}
	if pathExists(absPathToIndex) {
		return path.Clean(absPathToIndex + "/.."), nil
	}
	for _, srcdir := range build.Default.SrcDirs() {
		dirName := path.Join(srcdir, projectDirName)
		fmt.Println("DIRNAME: ", dirName)
		if pathExists(dirName) {
			return dirName, nil
		}
	}
	return "", errors.New("Project directory not found")
}

func setEnvVar() error {
	projectDirectory, err := guessProjectDir()
	if err != nil {
		return err
	}
	debug("PROJECT_DIRECTORY: %s\n", projectDirectory)

	// PER5LIB
	newPerl5Lib := addPerl5Lib(path.Clean(projectDirectory + "/scripts/lib"))
	err = setPerl5Lib(newPerl5Lib)
	if (err != nil) {
		return err
	}

	//GUIHIVE_BASEDIR
	if err := os.Setenv("GUIHIVE_BASEDIR", projectDirectory+"/"); err != nil {
		return err
	}
	debug("GUIHIVE_BASEDIR: %s", os.Getenv("GUIHIVE_BASEDIR"))

	// ENSEMBL_CVS_ROOT_DIR
	ensembl_cvs_root_dir := os.Getenv("ENSEMBL_CVS_ROOT_DIR")
	if ensembl_cvs_root_dir == "" {
		return errors.New("ENSEMBL_CVS_ROOT_DIR has to be set")
	}
	debug("ENSEMBL_CVS_ROOT_DIR: %s", ensembl_cvs_root_dir)

	return nil
}

func addPerl5Lib (newDir string) string {
	perl5lib := os.Getenv("PERL5LIB")
	if perl5lib == "" {
		perl5lib = newDir
	} else {
		perl5lib = newDir + ":" + perl5lib
	}
	return perl5lib
}

func setPerl5Lib (perl5lib string) error {
	err := os.Setenv("PERL5LIB", perl5lib)
	if err != nil {
		return err
	}
	debug("PERL5LIB: %s\n", os.Getenv("PERL5LIB"))

	return nil
}

func main() {

	//  Fix environmental variables
	errV := setEnvVar()
	checkError("Problem setting environmental variables: ", errV)

	relPath := os.Getenv("GUIHIVE_BASEDIR")

	http.Handle("/", http.FileServer(http.Dir(relPath)))

	http.HandleFunc("/versions/", unknown)
	http.Handle("/versions/56/", http.FileServer(http.Dir(relPath)))
	http.Handle("/versions/62/", http.FileServer(http.Dir(relPath)))

	http.Handle("/styles/", http.FileServer(http.Dir(relPath)))
	http.Handle("/javascript/", http.FileServer(http.Dir(relPath)))
	http.Handle("/versions/56/javascript/", http.FileServer(http.Dir(relPath)))
	http.Handle("/versions/62/javascript/", http.FileServer(http.Dir(relPath)))

	http.Handle("/images/", http.FileServer(http.Dir(relPath)))
	http.HandleFunc("/scripts/", scriptHandler)
	http.HandleFunc("/versions/56/scripts/", scriptHandler)
	http.HandleFunc("/versions/62/scripts/", scriptHandler)
	debug("Listening to port: %s", port)
	err := http.ListenAndServe(":"+port, nil)
	checkError("ListenAndServe ", err)
}
<commit_msg>guiHive should not depend on ENSEMBL_CVS_ROOT_DIR<commit_after>/* Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"strings"
	"bytes"
	"regexp"
	"strconv"
	"sort"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"go/build"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

const (
	projectDirName = "github.com/emepyc/guiHive"
)

var (
	port string
	isVersion = regexp.MustCompile(`^[0-9]+$`)
)

func init() {
	flag.StringVar(&port, "port", "8080", "Port to listen (defaults to 8080)")
	flag.Parse()
}

func checkError(s string, err error, ss ...string) {
	if err != nil {
		log.Fatal(s, err, ss)
	}
}

// Sortable os.fileInfos by name (-> num)
type sortableFiles []os.FileInfo
func (s sortableFiles) Len () int {
	return len(s)
}
func (s sortableFiles) Less (i, j int) bool {
	iVer, err := strconv.Atoi(s[i].Name())
	checkError(fmt.Sprintf("Dir name %s can't be converted to int", s[i].Name()), err)
	jVer, err := strconv.Atoi(s[j].Name())
	checkError(fmt.Sprintf("Dir name %s can't be converted to int", s[j].Name()), err)
	return iVer < jVer;
}
func (s sortableFiles) Swap (i, j int) {
	s[i], s[j] = s[j], s[i]
}

func version(r *http.Request) string {
	parts := strings.SplitN(r.URL.Path, "/", 4)
	version := parts[2]
	if (isVersion.MatchString(version)) {
		return version
	} else {
		path := os.Getenv("GUIHIVE_BASEDIR") + "/versions/"
		dir, err := os.Open(path)
		checkError("Can't open dir " + path, err)
		files, err := dir.Readdir(-1)
		checkError("Can't read dir " + path, err)
		sort.Sort(sortableFiles(files))
		version = files[len(files)-1].Name()
		return version
	}
	return ""
}

func unknown(w http.ResponseWriter, r *http.Request) {
	version := version(r)
	fmt.Fprintln(w, r.URL)
	fmt.Fprintf(w, "version %s is currently not supported by guiHive\n", version)
}

func scriptHandler(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	defer r.Body.Close()
	checkError("Can't parse Form: ", err)

	debug("METHOD: %s", r.Method)
	debug("URL: %s", r.URL)

	var outMsg bytes.Buffer
	var errMsg bytes.Buffer

	fname := os.Getenv("GUIHIVE_BASEDIR") + r.URL.Path
	args, err := json.Marshal(r.Form)
	checkError("Can't Marshal JSON:", err)

	debug("EXECUTING SCRIPT: %s", fname)
	debug("ARGS: %s", args)
	version := version(r);
	debug("VERSION: %s", version)

	versionRootDir := os.Getenv("GUIHIVE_BASEDIR") + "/versions/" + version;
	ehiveRootDir := versionRootDir + "/ensembl-hive"
	ehiveRootLib := ehiveRootDir + "/modules"
	guihiveRootLib := versionRootDir + "/scripts/lib"
	newPerl5Lib  := addPerl5Lib(ehiveRootLib + ":" + guihiveRootLib)

	debug("EHIVE_ROOT_DIR: %s", ehiveRootDir)
	debug("NEW_PERL5LIB: %s", newPerl5Lib)
	cmd := exec.Command(fname, string(args))
	cmd.Env = make([]string,0)
	cmd.Env = append(cmd.Env, "PERL5LIB=" + newPerl5Lib)
	cmd.Env = append(cmd.Env, "EHIVE_ROOT_DIR=" + ehiveRootDir)
	cmd.Env = append(cmd.Env, "GUIHIVE_BASEDIR=" + os.Getenv("GUIHIVE_BASEDIR"))
	cmd.Env = append(cmd.Env, "PATH=" + os.Getenv("PATH"))

	cmd.Stdout = &outMsg
	cmd.Stderr = &errMsg

	if err := cmd.Start(); err != nil {
		log.Println("Error Starting Command: ", err)
	}
	if err := cmd.Wait(); err != nil {
		log.Println("Error Executing Command: ", err)
	}

	debug("OUTMSG: %s", outMsg.Bytes())
	debug("ERRMSG: %s", errMsg.Bytes())
	fmt.Fprintln(w, string(outMsg.Bytes()))

}

func pathExists(name string) bool {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false
	}
	return err == nil
}

func guessProjectDir() (string, error) {
	// First, we try to find the project dir in the working directory
	serverPath := os.Args[0]
	serverDir := filepath.Dir(serverPath)
	pathToIndex := serverDir + "/../index.html"
	absPathToIndex, err := filepath.Abs(pathToIndex)
	if err != nil {
		debug("ABSPATHTOINDEX: %s\n", absPathToIndex)
		return "", err
	}
	if pathExists(absPathToIndex) {
		return path.Clean(absPathToIndex + "/.."), nil
	}
	for _, srcdir := range build.Default.SrcDirs() {
		dirName := path.Join(srcdir, projectDirName)
		fmt.Println("DIRNAME: ", dirName)
		if pathExists(dirName) {
			return dirName, nil
		}
	}
	return "", errors.New("Project directory not found")
}

func setEnvVar() error {
	projectDirectory, err := guessProjectDir()
	if err != nil {
		return err
	}
	debug("PROJECT_DIRECTORY: %s\n", projectDirectory)

	// PER5LIB
	newPerl5Lib := addPerl5Lib(path.Clean(projectDirectory + "/scripts/lib"))
	err = setPerl5Lib(newPerl5Lib)
	if (err != nil) {
		return err
	}

	//GUIHIVE_BASEDIR
	if err := os.Setenv("GUIHIVE_BASEDIR", projectDirectory+"/"); err != nil {
		return err
	}
	debug("GUIHIVE_BASEDIR: %s", os.Getenv("GUIHIVE_BASEDIR"))

	return nil
}

func addPerl5Lib (newDir string) string {
	perl5lib := os.Getenv("PERL5LIB")
	if perl5lib == "" {
		perl5lib = newDir
	} else {
		perl5lib = newDir + ":" + perl5lib
	}
	return perl5lib
}

func setPerl5Lib (perl5lib string) error {
	err := os.Setenv("PERL5LIB", perl5lib)
	if err != nil {
		return err
	}
	debug("PERL5LIB: %s\n", os.Getenv("PERL5LIB"))

	return nil
}

func main() {

	//  Fix environmental variables
	errV := setEnvVar()
	checkError("Problem setting environmental variables: ", errV)

	relPath := os.Getenv("GUIHIVE_BASEDIR")

	http.Handle("/", http.FileServer(http.Dir(relPath)))

	http.HandleFunc("/versions/", unknown)
	http.Handle("/versions/56/", http.FileServer(http.Dir(relPath)))
	http.Handle("/versions/62/", http.FileServer(http.Dir(relPath)))

	http.Handle("/styles/", http.FileServer(http.Dir(relPath)))
	http.Handle("/javascript/", http.FileServer(http.Dir(relPath)))
	http.Handle("/versions/56/javascript/", http.FileServer(http.Dir(relPath)))
	http.Handle("/versions/62/javascript/", http.FileServer(http.Dir(relPath)))

	http.Handle("/images/", http.FileServer(http.Dir(relPath)))
	http.HandleFunc("/scripts/", scriptHandler)
	http.HandleFunc("/versions/56/scripts/", scriptHandler)
	http.HandleFunc("/versions/62/scripts/", scriptHandler)
	debug("Listening to port: %s", port)
	err := http.ListenAndServe(":"+port, nil)
	checkError("ListenAndServe ", err)
}
<|endoftext|>


<commit_before>// package server contains the `pilosa server` subcommand which runs Pilosa
// itself. The purpose of this package is to define an easily tested Command
// object which handles interpreting configuration and setting up all the
// objects that Pilosa needs.
package server

import (
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/gossip"
	"github.com/pilosa/pilosa/httpbroadcast"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

const (
	// DefaultDataDir is the default data directory.
	DefaultDataDir = "~/.pilosa"
)

// Command represents the state of the pilosa server command.
type Command struct {
	Server *pilosa.Server

	// Configuration.
	Config *pilosa.Config

	// Profiling options.
	CPUProfile string
	CPUTime    time.Duration

	// Standard input/output
	*pilosa.CmdIO

	// running will be closed once Command.Run is finished.
	Started chan struct{}
	// Done will be closed when Command.Close() is called
	Done chan struct{}
}

// NewMain returns a new instance of Main.
func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
	return &Command{
		Server: pilosa.NewServer(),
		Config: pilosa.NewConfig(),

		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),

		Started: make(chan struct{}),
		Done:    make(chan struct{}),
	}
}

// Run executes the pilosa server.
func (m *Command) Run(args ...string) (err error) {
	defer close(m.Started)
	prefix := "~" + string(filepath.Separator)
	if strings.HasPrefix(m.Config.DataDir, prefix) {
		HomeDir := os.Getenv("HOME")
		if HomeDir == "" {
			return errors.New("data directory not specified and no home dir available")
		}
		m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix))
	}

	// SetupServer
	err = m.SetupServer()
	if err != nil {
		return err
	}

	// Initialize server.
	if err = m.Server.Open(); err != nil {
		return fmt.Errorf("server.Open: %v", err)
	}
	fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host)
	return nil
}

func (m *Command) SetupServer() error {
	cluster := pilosa.NewCluster()
	cluster.ReplicaN = m.Config.Cluster.ReplicaN

	for _, hostport := range m.Config.Cluster.Nodes {
		cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport})
	}
	m.Server.Cluster = cluster

	// Setup logging output.
	if m.Config.LogPath == "" {
		m.Server.LogOutput = m.Stderr
	} else {
		logFile, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			return err
		}
		m.Server.LogOutput = logFile
	}

	// Configure index.
	fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir)
	m.Server.Index.Path = m.Config.DataDir
	m.Server.Index.Stats = pilosa.NewExpvarStatsClient()

	var err error
	m.Server.Host, err = normalizeHost(m.Config.Host)
	if err != nil {
		return err
	}

	switch m.Config.Cluster.Type { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership
	case "http":
		m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort)
		m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr)
		m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet()
		err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes)
		if err != nil {
			return err
		}
	case "gossip":
		gossipPortStr := pilosa.DefaultGossipPort
		if m.Config.Cluster.InternalPort != "" {
			gossipPortStr = m.Config.Cluster.InternalPort
		}
		gossipPort, err := strconv.Atoi(gossipPortStr)
		if err != nil {
			return err
		}
		gossipSeed := pilosa.DefaultHost
		if m.Config.Cluster.GossipSeed != "" {
			gossipSeed = m.Config.Cluster.GossipSeed
		}
		// get the host portion of addr to use for binding
		gossipHost, _, err := net.SplitHostPort(m.Config.Host)
		if err != nil {
			gossipHost = m.Config.Host
		}
		gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server)
		m.Server.Cluster.NodeSet = gossipNodeSet
		m.Server.Broadcaster = gossipNodeSet
		m.Server.BroadcastReceiver = gossipNodeSet
	case "static", "":
		m.Server.Broadcaster = pilosa.NopBroadcaster
		m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet()
		m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
	default:
		return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type)
	}

	// Set configuration options.
	m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
	return nil
}

func normalizeHost(host string) (string, error) {
	if !strings.Contains(host, ":") {
		host = host + ":"
	} else if strings.Contains(host, "://") {
		if strings.HasPrefix(host, "http://") {
			host = host[7:]
		} else {
			return "", fmt.Errorf("invalid scheme or host: '%s'. use the format [http://]<host>:<port>", host)
		}
	}
	return host, nil
}

// Close shuts down the server.
func (m *Command) Close() error {
	var logErr error
	serveErr := m.Server.Close()
	logOutput := m.Server.LogOutput
	if closer, ok := logOutput.(io.Closer); ok {
		logErr = closer.Close()
	}
	close(m.Done)
	if serveErr != nil && logErr != nil {
		return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr)
	} else if logErr != nil {
		return logErr
	}
	return serveErr
}
<commit_msg>remove done TODO<commit_after>// package server contains the `pilosa server` subcommand which runs Pilosa
// itself. The purpose of this package is to define an easily tested Command
// object which handles interpreting configuration and setting up all the
// objects that Pilosa needs.
package server

import (
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/gossip"
	"github.com/pilosa/pilosa/httpbroadcast"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

const (
	// DefaultDataDir is the default data directory.
	DefaultDataDir = "~/.pilosa"
)

// Command represents the state of the pilosa server command.
type Command struct {
	Server *pilosa.Server

	// Configuration.
	Config *pilosa.Config

	// Profiling options.
	CPUProfile string
	CPUTime    time.Duration

	// Standard input/output
	*pilosa.CmdIO

	// running will be closed once Command.Run is finished.
	Started chan struct{}
	// Done will be closed when Command.Close() is called
	Done chan struct{}
}

// NewMain returns a new instance of Main.
func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
	return &Command{
		Server: pilosa.NewServer(),
		Config: pilosa.NewConfig(),

		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),

		Started: make(chan struct{}),
		Done:    make(chan struct{}),
	}
}

// Run executes the pilosa server.
func (m *Command) Run(args ...string) (err error) {
	defer close(m.Started)
	prefix := "~" + string(filepath.Separator)
	if strings.HasPrefix(m.Config.DataDir, prefix) {
		HomeDir := os.Getenv("HOME")
		if HomeDir == "" {
			return errors.New("data directory not specified and no home dir available")
		}
		m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix))
	}

	// SetupServer
	err = m.SetupServer()
	if err != nil {
		return err
	}

	// Initialize server.
	if err = m.Server.Open(); err != nil {
		return fmt.Errorf("server.Open: %v", err)
	}
	fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host)
	return nil
}

func (m *Command) SetupServer() error {
	cluster := pilosa.NewCluster()
	cluster.ReplicaN = m.Config.Cluster.ReplicaN

	for _, hostport := range m.Config.Cluster.Nodes {
		cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport})
	}
	m.Server.Cluster = cluster

	// Setup logging output.
	if m.Config.LogPath == "" {
		m.Server.LogOutput = m.Stderr
	} else {
		logFile, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			return err
		}
		m.Server.LogOutput = logFile
	}

	// Configure index.
	fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir)
	m.Server.Index.Path = m.Config.DataDir
	m.Server.Index.Stats = pilosa.NewExpvarStatsClient()

	var err error
	m.Server.Host, err = normalizeHost(m.Config.Host)
	if err != nil {
		return err
	}

	switch m.Config.Cluster.Type {
	case "http":
		m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort)
		m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr)
		m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet()
		err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes)
		if err != nil {
			return err
		}
	case "gossip":
		gossipPortStr := pilosa.DefaultGossipPort
		if m.Config.Cluster.InternalPort != "" {
			gossipPortStr = m.Config.Cluster.InternalPort
		}
		gossipPort, err := strconv.Atoi(gossipPortStr)
		if err != nil {
			return err
		}
		gossipSeed := pilosa.DefaultHost
		if m.Config.Cluster.GossipSeed != "" {
			gossipSeed = m.Config.Cluster.GossipSeed
		}
		// get the host portion of addr to use for binding
		gossipHost, _, err := net.SplitHostPort(m.Config.Host)
		if err != nil {
			gossipHost = m.Config.Host
		}
		gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server)
		m.Server.Cluster.NodeSet = gossipNodeSet
		m.Server.Broadcaster = gossipNodeSet
		m.Server.BroadcastReceiver = gossipNodeSet
	case "static", "":
		m.Server.Broadcaster = pilosa.NopBroadcaster
		m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet()
		m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
	default:
		return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type)
	}

	// Set configuration options.
	m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
	return nil
}

func normalizeHost(host string) (string, error) {
	if !strings.Contains(host, ":") {
		host = host + ":"
	} else if strings.Contains(host, "://") {
		if strings.HasPrefix(host, "http://") {
			host = host[7:]
		} else {
			return "", fmt.Errorf("invalid scheme or host: '%s'. use the format [http://]<host>:<port>", host)
		}
	}
	return host, nil
}

// Close shuts down the server.
func (m *Command) Close() error {
	var logErr error
	serveErr := m.Server.Close()
	logOutput := m.Server.LogOutput
	if closer, ok := logOutput.(io.Closer); ok {
		logErr = closer.Close()
	}
	close(m.Done)
	if serveErr != nil && logErr != nil {
		return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr)
	} else if logErr != nil {
		return logErr
	}
	return serveErr
}
<|endoftext|>


<commit_before>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the server loop. Deals with incoming requests and delegates
// them to the event store.
package server

import (
	"bytes"
	"errors"
	"log"
	"container/list"
	"time"
	"sync"
	zmq "github.com/alecthomas/gozmq"
	es "github.com/JensRantil/gorewind/eventstore"
)

type InitParams struct {
	Store *es.EventStore
	CommandSocketZPath *string
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context

	runningMutex sync.Mutex
	running bool
	stopChan chan bool
}

// IsRunning returns true if the server is running, false otherwise.
func (v *Server) IsRunning() bool {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	return v.running
}

// Stop stops the server.
func (v* Server) Stop() error {
	if v.IsRunning() {
		return errors.New("Not running.")
	}

	select {
	case v.stopChan <- true:
	default:
		return errors.New("Stop already signalled.")
	}
	<-v.stopChan
	// v.running is modified by Server.Run(...)

	if v.IsRunning() {
		return errors.New("Signalled stopped, but never stopped.")
	}

	return nil
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
		running: false,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.closeZmq()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) closeZmq() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

func (v *Server) setRunningState(newState bool) {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	v.running = newState
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	v.setRunningState(true)
	defer v.setRunningState(false)
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock, v.stopChan)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems, stop chan bool) {
	for {
		timeout := time.Duration(1)*time.Second
		count, err := zmq.Poll(items, int64(timeout))
		if count > 0 || err != nil {
			notifier <- zmqPollResult{err}
		}

		select {
		case <-stop:
			stop <- true
			return
		default:
		}
	}
}

func stopPoller(cancelChan chan bool) {
	cancelChan <- true
	<-cancelChan
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
//
// TODO: Make this a type function of `Server` to remove a lot of
// parameters.
func loopServer(estore *es.EventStore, evpubsock, frontend zmq.Socket,
stop chan bool) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan es.StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan zMsg)

	pollCancel := make(chan bool)
	defer stopPoller(pollCancel)

	go asyncPoll(pollchan, toPoll, pollCancel)
	for {
		select {
		case res := <-pollchan:
			if res.err != nil {
				log.Print("Could not poll:", res.err)
			}
			if res.err == nil && toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				zmsg := zMsg(msg)
				go handleRequest(respchan, estore, zmsg)
			}
			go asyncPoll(pollchan, toPoll, pollCancel)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				log.Println(err)
			}
		case <- stop:
			stop <- true
			return
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan es.StoredEvent, evpub zmq.Socket) {
	msg := make(zMsg, 3)
	for {
		stored := <-toPublish

		msg[0] = stored.Event.Stream
		msg[1] = stored.Id
		msg[2] = stored.Event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			log.Println(err)
		}
	}
}

// A single frame in a ZeroMQ message.
type zFrame []byte

// A ZeroMQ message.
//
// I wish it could have been `[]zFrame`, but that would make conversion
// from `[][]byte` pretty messy[1].
//
// [1] http://stackoverflow.com/a/15650327/260805
type zMsg [][]byte

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan zMsg, estore *es.EventStore, msg zMsg) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	resptemplate := list.New()
	emptyFrame := zFrame("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.(zFrame), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.(zFrame))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for PUBLISH."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := es.Event{
				estream.(es.StreamName),
				data.(zFrame),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack(zFrame("PUBLISHED"))
				response.PushBack(zFrame(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for QUERY."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			req := es.QueryRequest{
				Stream: estream.(zFrame),
				FromId: fromid.(zFrame),
				ToId: toid.(zFrame),
			}
			events, err := estore.Query(req)

			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				for eventdata := range(events) {
					response := copyList(resptemplate)
					response.PushBack([]byte("EVENT"))
					response.PushBack(eventdata.Id)
					response.PushBack(eventdata.Data)

					respchan <- listToFrames(response)
				}
				response := copyList(resptemplate)
				response.PushBack(zFrame("END"))
				respchan <- listToFrames(response)
			}
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		// TODO: Constantify this error message
		errstr := "Unknown request type."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) zMsg {
	frames := make(zMsg, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.(zFrame)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<commit_msg>Documentation improvements<commit_after>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the ZeroMQ server loop. Deals with incoming requests and
// delegates them to the event store. Also publishes newly stored events
// using a PUB socket.
//
// See README file for an up-to-date documentation of the ZeroMQ wire
// format.
package server

import (
	"bytes"
	"errors"
	"log"
	"container/list"
	"time"
	"sync"
	zmq "github.com/alecthomas/gozmq"
	es "github.com/JensRantil/gorewind/eventstore"
)

// StartParams are parameters required for starting the server.
type InitParams struct {
	// The event store to use as backend.
	Store *es.EventStore
	// The ZeroMQ path that the command receiving socket will bind
	// to.
	CommandSocketZPath *string
	// The ZeroMQ path that the event publishing socket will bind
	// to.
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context

	runningMutex sync.Mutex
	running bool
	stopChan chan bool
}

// IsRunning returns true if the server is running, false otherwise.
func (v *Server) IsRunning() bool {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	return v.running
}

// Stop stops a running server. Blocks until the server is stopped. If
// the server is not running, an error is returned.
func (v* Server) Stop() error {
	if v.IsRunning() {
		return errors.New("Not running.")
	}

	select {
	case v.stopChan <- true:
	default:
		return errors.New("Stop already signalled.")
	}
	<-v.stopChan
	// v.running is modified by Server.Run(...)

	if v.IsRunning() {
		return errors.New("Signalled stopped, but never stopped.")
	}

	return nil
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
		running: false,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.closeZmq()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) closeZmq() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

func (v *Server) setRunningState(newState bool) {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	v.running = newState
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	v.setRunningState(true)
	defer v.setRunningState(false)
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock, v.stopChan)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems, stop chan bool) {
	for {
		timeout := time.Duration(1)*time.Second
		count, err := zmq.Poll(items, int64(timeout))
		if count > 0 || err != nil {
			notifier <- zmqPollResult{err}
		}

		select {
		case <-stop:
			stop <- true
			return
		default:
		}
	}
}

func stopPoller(cancelChan chan bool) {
	cancelChan <- true
	<-cancelChan
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
//
// TODO: Make this a type function of `Server` to remove a lot of
// parameters.
func loopServer(estore *es.EventStore, evpubsock, frontend zmq.Socket,
stop chan bool) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan es.StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan zMsg)

	pollCancel := make(chan bool)
	defer stopPoller(pollCancel)

	go asyncPoll(pollchan, toPoll, pollCancel)
	for {
		select {
		case res := <-pollchan:
			if res.err != nil {
				log.Print("Could not poll:", res.err)
			}
			if res.err == nil && toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				zmsg := zMsg(msg)
				go handleRequest(respchan, estore, zmsg)
			}
			go asyncPoll(pollchan, toPoll, pollCancel)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				log.Println(err)
			}
		case <- stop:
			stop <- true
			return
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan es.StoredEvent, evpub zmq.Socket) {
	msg := make(zMsg, 3)
	for {
		stored := <-toPublish

		msg[0] = stored.Event.Stream
		msg[1] = stored.Id
		msg[2] = stored.Event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			log.Println(err)
		}
	}
}

// A single frame in a ZeroMQ message.
type zFrame []byte

// A ZeroMQ message.
//
// I wish it could have been `[]zFrame`, but that would make conversion
// from `[][]byte` pretty messy[1].
//
// [1] http://stackoverflow.com/a/15650327/260805
type zMsg [][]byte

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan zMsg, estore *es.EventStore, msg zMsg) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	resptemplate := list.New()
	emptyFrame := zFrame("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.(zFrame), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.(zFrame))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for PUBLISH."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := es.Event{
				estream.(es.StreamName),
				data.(zFrame),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack(zFrame("PUBLISHED"))
				response.PushBack(zFrame(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for QUERY."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			req := es.QueryRequest{
				Stream: estream.(zFrame),
				FromId: fromid.(zFrame),
				ToId: toid.(zFrame),
			}
			events, err := estore.Query(req)

			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				for eventdata := range(events) {
					response := copyList(resptemplate)
					response.PushBack([]byte("EVENT"))
					response.PushBack(eventdata.Id)
					response.PushBack(eventdata.Data)

					respchan <- listToFrames(response)
				}
				response := copyList(resptemplate)
				response.PushBack(zFrame("END"))
				respchan <- listToFrames(response)
			}
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		// TODO: Constantify this error message
		errstr := "Unknown request type."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) zMsg {
	frames := make(zMsg, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.(zFrame)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<|endoftext|>


<commit_before>package server

import (
	"net/http"

	"github.com/goware/heartbeat"
	"github.com/goware/httpcoala"
	"github.com/goware/lg"
	"github.com/pressly/chainstore"
	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
	"github.com/pressly/consistentrd"
	"github.com/pressly/imgry"
	"github.com/pressly/imgry/imagick"
	"github.com/rs/cors"
	"golang.org/x/net/context"
)

var (
	app     *Server
	respond = NewResponder()
)

type Server struct {
	Config      *Config
	DB          *DB
	Chainstore  chainstore.Store
	Fetcher     *Fetcher
	ImageEngine imgry.Engine

	Ctx        context.Context
	CancelFunc context.CancelFunc
}

func New(conf *Config) *Server {
	ctx, cancel := context.WithCancel(context.Background())
	app = &Server{Ctx: ctx, CancelFunc: cancel, Config: conf}
	return app
}

func (srv *Server) Configure() (err error) {
	if err := srv.Config.Apply(); err != nil {
		return err
	}

	srv.DB, err = srv.Config.GetDB()
	if err != nil {
		return err
	}

	srv.Chainstore, err = srv.Config.GetChainstore()
	if err != nil {
		return err
	}

	if err := srv.Config.SetupLibrato(); err != nil {
		return err
	}

	srv.Fetcher = NewFetcher()

	tmpDir := srv.Config.TmpDir
	srv.ImageEngine = imagick.Engine{}
	if err := srv.ImageEngine.Initialize(tmpDir); err != nil {
		return err
	}

	return nil
}

// Close signals to the server that should deny new requests
// and finish up requests in progress.
func (srv *Server) Close() {
	lg.Info("closing server..")
	srv.CancelFunc()
}

// Shutdown will release other resources and halt the server.
func (srv *Server) Shutdown() {
	srv.ImageEngine.Terminate()
	srv.DB.Close()
	srv.Chainstore.Close()
	lg.Info("server shutdown.")
}

func (srv *Server) NewRouter() http.Handler {
	cf := srv.Config

	conrd, err := consistentrd.New(cf.Cluster.LocalNode, cf.Cluster.Nodes)
	if err != nil {
		panic(err)
	}

	r := chi.NewRouter()

	r.Use(ParentContext(srv.Ctx))
	r.Use(middleware.RequestID)
	r.Use(middleware.RealIP)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)

	r.Use(middleware.CloseNotify)
	r.Use(middleware.Timeout(cf.Limits.RequestTimeout))

	cors := cors.New(cors.Options{
		AllowedOrigins:   []string{"*"},
		AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
		AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
		ExposedHeaders:   []string{"Link"},
		AllowCredentials: true,
		MaxAge:           300, // Maximum value not ignored by any of major browsers
	})
	r.Use(cors.Handler)

	r.Use(httpcoala.Route("HEAD", "GET"))

	r.Use(heartbeat.Route("/ping"))
	r.Use(heartbeat.Route("/favicon.ico"))

	if cf.Airbrake.ApiKey != "" {
		r.Use(AirbrakeRecoverer(cf.Airbrake.ApiKey))
	}

	if srv.Config.Profiler {
		r.Mount("/debug", middleware.NoCache, Profiler())
	}

	r.Get("/", trackRoute("root"), func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("."))
	})

	r.Get("/info", trackRoute("imageInfo"), GetImageInfo)

	r.Route("/:bucket", func(r chi.Router) {
		r.Post("/", BucketImageUpload)
		r.Get("/", conrd.RouteWithParams("url"), trackRoute("bucketV1GetItem"), BucketGetIndex)
		r.Get("/fetch", conrd.RouteWithParams("url"), trackRoute("bucketV1GetItem"), BucketFetchItem)

		// TODO: review
		r.Get("/add", trackRoute("bucketAddItems"), BucketAddItems)
		r.Get("/:key", conrd.Route(), BucketGetItem)
		r.Delete("/:key", conrd.Route(), BucketDeleteItem)
	})

	// DEPRECATED
	r.Get("/fetch", trackRoute("bucketV0GetItem"), BucketV0FetchItem) // for Pressilla v2 apps
	r.Get("/:bucket//fetch", conrd.RouteWithParams("url"), trackRoute("bucketV1GetItem"), BucketFetchItem)
	// --

	return r
}
<commit_msg>Restricts CORS middleware to routes that are not accessed via reeler<commit_after>package server

import (
	"net/http"

	"github.com/goware/heartbeat"
	"github.com/goware/httpcoala"
	"github.com/goware/lg"
	"github.com/pressly/chainstore"
	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
	"github.com/pressly/consistentrd"
	"github.com/pressly/imgry"
	"github.com/pressly/imgry/imagick"
	"github.com/rs/cors"
	"golang.org/x/net/context"
)

var (
	app     *Server
	respond = NewResponder()
)

type Server struct {
	Config      *Config
	DB          *DB
	Chainstore  chainstore.Store
	Fetcher     *Fetcher
	ImageEngine imgry.Engine

	Ctx        context.Context
	CancelFunc context.CancelFunc
}

func New(conf *Config) *Server {
	ctx, cancel := context.WithCancel(context.Background())
	app = &Server{Ctx: ctx, CancelFunc: cancel, Config: conf}
	return app
}

func (srv *Server) Configure() (err error) {
	if err := srv.Config.Apply(); err != nil {
		return err
	}

	srv.DB, err = srv.Config.GetDB()
	if err != nil {
		return err
	}

	srv.Chainstore, err = srv.Config.GetChainstore()
	if err != nil {
		return err
	}

	if err := srv.Config.SetupLibrato(); err != nil {
		return err
	}

	srv.Fetcher = NewFetcher()

	tmpDir := srv.Config.TmpDir
	srv.ImageEngine = imagick.Engine{}
	if err := srv.ImageEngine.Initialize(tmpDir); err != nil {
		return err
	}

	return nil
}

// Close signals to the server that should deny new requests
// and finish up requests in progress.
func (srv *Server) Close() {
	lg.Info("closing server..")
	srv.CancelFunc()
}

// Shutdown will release other resources and halt the server.
func (srv *Server) Shutdown() {
	srv.ImageEngine.Terminate()
	srv.DB.Close()
	srv.Chainstore.Close()
	lg.Info("server shutdown.")
}

func (srv *Server) NewRouter() http.Handler {
	cf := srv.Config

	conrd, err := consistentrd.New(cf.Cluster.LocalNode, cf.Cluster.Nodes)
	if err != nil {
		panic(err)
	}

	r := chi.NewRouter()

	r.Use(ParentContext(srv.Ctx))
	r.Use(middleware.RequestID)
	r.Use(middleware.RealIP)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)

	r.Use(middleware.CloseNotify)
	r.Use(middleware.Timeout(cf.Limits.RequestTimeout))
	r.Use(httpcoala.Route("HEAD", "GET"))

	r.Use(heartbeat.Route("/ping"))
	r.Use(heartbeat.Route("/favicon.ico"))

	if cf.Airbrake.ApiKey != "" {
		r.Use(AirbrakeRecoverer(cf.Airbrake.ApiKey))
	}

	if srv.Config.Profiler {
		r.Mount("/debug", middleware.NoCache, Profiler())
	}

	r.Get("/", trackRoute("root"), func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("."))
	})

	r.Get("/info", trackRoute("imageInfo"), GetImageInfo)

	r.Route("/:bucket", func(r chi.Router) {
		r.Post("/", BucketImageUpload)

		r.Group(func(r chi.Router) {
			cors := cors.New(cors.Options{
				AllowedOrigins:   []string{"*"},
				AllowedMethods:   []string{"GET", "OPTIONS"},
				AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
				ExposedHeaders:   []string{"Link"},
				AllowCredentials: true,
				MaxAge:           300, // Maximum value not ignored by any of major browsers
			})
			r.Use(cors.Handler)

			r.Get("/", conrd.RouteWithParams("url"), trackRoute("bucketV1GetItem"), BucketGetIndex)
			r.Get("/fetch", conrd.RouteWithParams("url"), trackRoute("bucketV1GetItem"), BucketFetchItem)
		})

		// TODO: review
		r.Get("/add", trackRoute("bucketAddItems"), BucketAddItems)
		r.Get("/:key", conrd.Route(), BucketGetItem)
		r.Delete("/:key", conrd.Route(), BucketDeleteItem)
	})

	// DEPRECATED
	r.Get("/fetch", trackRoute("bucketV0GetItem"), BucketV0FetchItem) // for Pressilla v2 apps
	r.Get("/:bucket//fetch", conrd.RouteWithParams("url"), trackRoute("bucketV1GetItem"), BucketFetchItem)
	// --

	return r
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"net/http"
)

var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"}

func worker(data *announceData) []string {
	if RedisGetBoolKeyVal(data.redisClient, data.info_hash, data) {
		x := RedisGetKeyVal(data.redisClient, data.info_hash, data)

		RedisSetKeyVal(data.redisClient,
			concatenateKeyMember(data.info_hash, "ip"),
			createIpPortPair(data))
		RedisSetKeyVal(data.redisClient,
			concatena

		return x

	} else {
		CreateNewTorrentKey(data.redisClient, data.info_hash)
		return worker(data)
	}
}

func requestHandler(w http.ResponseWriter, req *http.Request) {
	data := new(announceData)
	err := data.parseAnnounceData(req)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Event: %s from host %s on port %v\n", data.event, data.ip, data.port)

	switch data.event {

	case "started":
		data.StartedEventHandler()

	case "stopped":
		data.StoppedEventHandler()

	case "completed":
		data.CompletedEventHandler()
	default:
		panic(fmt.Errorf("We're somehow getting this strange error..."))
	}

	if data.event == "started" || data.event == "completed" {
		worker(data)
		x := RedisGetKeyVal(data.redisClient, data.info_hash, data)
		// TODO(ian): Move this into a seperate function.
		// TODO(ian): Remove this magic number and use data.numwant, but limit it
		// to 30 max, as that's the bittorrent protocol suggested limit.
		if len(x) >= 30 {
			x = x[0:30]
		} else {
			x = x[0:len(x)]
		}

		if len(x) > 0 {
			w.Header().Set("Content-Type", "text/plain")
			response := formatResponseData(x, data)

			w.Write([]byte(response))

		} else {
			failMsg := fmt.Sprintf("No peers for torrent %s\n", data.info_hash)
			w.Write([]byte(createFailureMessage(failMsg)))
		}
	}
}

func RunServer() {
	mux := http.NewServeMux()

	mux.HandleFunc("/announce", requestHandler)
	http.ListenAndServe(":3000", mux)
}
<commit_msg>Update server.go<commit_after>package server

import (
	"fmt"
	"net/http"
)

var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"}

func worker(data *announceData) []string {
	if RedisGetBoolKeyVal(data.redisClient, data.info_hash, data) {
		x := RedisGetKeyVal(data.redisClient, data.info_hash, data)

		RedisSetKeyVal(data.redisClient,
			concatenateKeyMember(data.info_hash, "ip"),
			createIpPortPair(data))

		return x

	} else {
		CreateNewTorrentKey(data.redisClient, data.info_hash)
		return worker(data)
	}
}

func requestHandler(w http.ResponseWriter, req *http.Request) {
	data := new(announceData)
	err := data.parseAnnounceData(req)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Event: %s from host %s on port %v\n", data.event, data.ip, data.port)

	switch data.event {

	case "started":
		data.StartedEventHandler()

	case "stopped":
		data.StoppedEventHandler()

	case "completed":
		data.CompletedEventHandler()
	default:
		panic(fmt.Errorf("We're somehow getting this strange error..."))
	}

	if data.event == "started" || data.event == "completed" {
		worker(data)
		x := RedisGetKeyVal(data.redisClient, data.info_hash, data)
		// TODO(ian): Move this into a seperate function.
		// TODO(ian): Remove this magic number and use data.numwant, but limit it
		// to 30 max, as that's the bittorrent protocol suggested limit.
		if len(x) >= 30 {
			x = x[0:30]
		} else {
			x = x[0:len(x)]
		}

		if len(x) > 0 {
			w.Header().Set("Content-Type", "text/plain")
			response := formatResponseData(x, data)

			w.Write([]byte(response))

		} else {
			failMsg := fmt.Sprintf("No peers for torrent %s\n", data.info_hash)
			w.Write([]byte(createFailureMessage(failMsg)))
		}
	}
}

func RunServer() {
	mux := http.NewServeMux()

	mux.HandleFunc("/announce", requestHandler)
	http.ListenAndServe(":3000", mux)
}
<|endoftext|>


<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package server contains the `pilosa server` subcommand which runs Pilosa
// itself. The purpose of this package is to define an easily tested Command
// object which handles interpreting configuration and setting up all the
// objects that Pilosa needs.

package server

import (
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"crypto/tls"

	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/boltdb"
	"github.com/pilosa/pilosa/gcnotify"
	"github.com/pilosa/pilosa/gopsutil"
	"github.com/pilosa/pilosa/gossip"
	"github.com/pilosa/pilosa/statik"
	"github.com/pilosa/pilosa/statsd"
	"github.com/pkg/errors"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

type loggerLogger interface {
	pilosa.Logger
	Logger() *log.Logger
}

// Command represents the state of the pilosa server command.
type Command struct {
	Server *pilosa.Server

	// Configuration.
	Config *Config

	// Gossip transport
	GossipTransport *gossip.Transport

	// Standard input/output
	*pilosa.CmdIO

	// Started will be closed once Command.Run is finished.
	Started chan struct{}
	// Done will be closed when Command.Close() is called
	Done chan struct{}

	// Passed to the Gossip implementation.
	logOutput io.Writer
	logger    loggerLogger
}

// NewCommand returns a new instance of Main.
func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
	return &Command{
		Config: NewConfig(),

		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),

		Started: make(chan struct{}),
		Done:    make(chan struct{}),
	}
}

// Start starts the pilosa server - it returns once the server is running.
func (m *Command) Start() (err error) {
	defer close(m.Started)

	// SetupServer
	err = m.SetupServer()
	if err != nil {
		return errors.Wrap(err, "setting up server")
	}

	// SetupNetworking
	err = m.SetupNetworking()
	if err != nil {
		return errors.Wrap(err, "setting up networking")
	}

	// Initialize server.
	if err = m.Server.Open(); err != nil {
		return errors.Wrap(err, "opening server")
	}

	m.logger.Printf("Listening as %s\n", m.Server.URI)

	return nil
}

// Wait waits for the server to be closed or interrupted.
func (m *Command) Wait() error {
	// First SIGKILL causes server to shut down gracefully.
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	select {
	case sig := <-c:
		m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String())

		// Second signal causes a hard shutdown.
		go func() { <-c; os.Exit(1) }()
		return errors.Wrap(m.Close(), "closing command")
	case <-m.Done:
		m.logger.Printf("Server closed externally")
		return nil
	}
}

// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
	var err error
	if m.Config.LogPath == "" {
		m.logOutput = m.Stderr
	} else {
		m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			return errors.Wrap(err, "opening file")
		}
	}

	if m.Config.Verbose {
		m.logger = pilosa.NewVerboseLogger(m.logOutput)
	} else {
		m.logger = pilosa.NewStandardLogger(m.logOutput)
	}
	return nil
}

// SetupServer uses the cluster configuration to set up this server.
func (m *Command) SetupServer() error {
	err := m.setupLogger()
	if err != nil {
		return errors.Wrap(err, "setting up logger")
	}

	handler := pilosa.NewHandler()
	handler.Logger = m.logger
	handler.FileSystem = &statik.FileSystem{}
	handler.API = pilosa.NewAPI()
	handler.API.Logger = m.logger

	uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
	if err != nil {
		return errors.Wrap(err, "processing bind address")
	}

	// Setup TLS
	var TLSConfig *tls.Config
	if uri.Scheme() == "https" {
		if m.Config.TLS.CertificatePath == "" {
			return errors.New("certificate path is required for TLS sockets")
		}
		if m.Config.TLS.CertificateKeyPath == "" {
			return errors.New("certificate key path is required for TLS sockets")
		}
		cert, err := tls.LoadX509KeyPair(m.Config.TLS.CertificatePath, m.Config.TLS.CertificateKeyPath)
		if err != nil {
			return errors.Wrap(err, "load x509 key pair")
		}
		TLSConfig = &tls.Config{
			Certificates:       []tls.Certificate{cert},
			InsecureSkipVerify: m.Config.TLS.SkipVerify,
		}
	}

	diagnosticsInterval := time.Duration(0)
	if m.Config.Metric.Diagnostics {
		diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval)
	}

	statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
	if err != nil {
		return errors.Wrap(err, "new stats client")
	}

	ln, err := getListener(*uri, TLSConfig)
	if err != nil {
		return errors.Wrap(err, "getting listener")
	}

	c := GetHTTPClient(TLSConfig)
	handler.API.RemoteClient = c

	m.Server, err = pilosa.NewServer(
		pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)),
		pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)),
		pilosa.OptServerDataDir(m.Config.DataDir),
		pilosa.OptServerReplicaN(m.Config.Cluster.ReplicaN),
		pilosa.OptServerMaxWritesPerRequest(m.Config.MaxWritesPerRequest),
		pilosa.OptServerMetricInterval(time.Duration(m.Config.Metric.PollInterval)),
		pilosa.OptServerDiagnosticsInterval(diagnosticsInterval),

		pilosa.OptServerLogger(m.logger),
		pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
		pilosa.OptServerHandler(handler),
		pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
		pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
		pilosa.OptServerStatsClient(statsClient),
		pilosa.OptServerListener(ln),
		pilosa.OptServerURI(uri),
		pilosa.OptServerRemoteClient(c),
	)

	return errors.Wrap(err, "new server")
}

func GetHTTPClient(t *tls.Config) *http.Client {
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          1000,
		MaxIdleConnsPerHost:   200,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}
	if t != nil {
		transport.TLSClientConfig = t
	}
	return &http.Client{Transport: transport}
}

// SetupNetworking sets up internode communication based on the configuration.
func (m *Command) SetupNetworking() error {

	m.Server.NodeID = m.Server.LoadNodeID()

	if m.Config.Cluster.Disabled {
		m.Server.Cluster.Static = true
		m.Server.Cluster.Coordinator = m.Server.NodeID
		for _, address := range m.Config.Cluster.Hosts {
			uri, err := pilosa.NewURIFromAddress(address)
			if err != nil {
				return errors.Wrap(err, "getting URI")
			}
			m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{
				URI: *uri,
			})
		}

		m.Server.Broadcaster = pilosa.NopBroadcaster
		m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes)
		m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
		m.Server.Gossiper = pilosa.NopGossiper
		return nil
	}

	gossipPort, err := strconv.Atoi(m.Config.Gossip.Port)
	if err != nil {
		return errors.Wrap(err, "parsing port")
	}

	// get the host portion of addr to use for binding
	gossipHost := m.Server.URI.Host()
	var transport *gossip.Transport
	if m.GossipTransport != nil {
		transport = m.GossipTransport
	} else {
		transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
		if err != nil {
			return errors.Wrap(err, "getting transport")
		}
	}

	// Set Coordinator.
	if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 {
		m.Server.Cluster.Coordinator = m.Server.NodeID
		m.Server.Cluster.Node.IsCoordinator = true
	}

	gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger)
	m.Server.Cluster.EventReceiver = gossipEventReceiver
	gossipMemberSet, err := gossip.NewGossipMemberSet(
		m.Server.NodeID,
		m.Server.URI.Host(),
		m.Config.Gossip,
		gossipEventReceiver,
		m.Server,
		gossip.WithLogger(m.logger.Logger()),
		gossip.WithTransport(transport),
	)
	if err != nil {
		return errors.Wrap(err, "getting memberset")
	}
	gossipMemberSet.Logger = m.logger
	m.Server.Cluster.MemberSet = gossipMemberSet
	m.Server.Broadcaster = m.Server
	m.Server.BroadcastReceiver = gossipMemberSet
	m.Server.Gossiper = gossipMemberSet
	return nil
}

// Close shuts down the server.
func (m *Command) Close() error {
	var logErr error
	serveErr := m.Server.Close()
	if closer, ok := m.logOutput.(io.Closer); ok {
		logErr = closer.Close()
	}
	close(m.Done)
	if serveErr != nil && logErr != nil {
		return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr)
	} else if logErr != nil {
		return logErr
	}
	return serveErr
}

// NewStatsClient creates a stats client from the config
func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
	switch name {
	case "expvar":
		return pilosa.NewExpvarStatsClient(), nil
	case "statsd":
		return statsd.NewStatsClient(host)
	case "nop", "none":
		return pilosa.NopStatsClient, nil
	default:
		return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].")
	}
}

// getListener gets a net.Listener based on the config.
func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
	// If bind URI has the https scheme, enable TLS
	if uri.Scheme() == "https" && tlsconf != nil {
		ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
		if err != nil {
			return nil, errors.Wrap(err, "tls.Listener")
		}
	} else if uri.Scheme() == "http" {
		// Open HTTP listener to determine port (if specified as :0).
		ln, err = net.Listen("tcp", uri.HostPort())
		if err != nil {
			return nil, errors.Wrap(err, "net.Listen")
		}
	} else {
		return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme())
	}

	return ln, nil
}
<commit_msg>replace "Pilosa starting..." log line<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package server contains the `pilosa server` subcommand which runs Pilosa
// itself. The purpose of this package is to define an easily tested Command
// object which handles interpreting configuration and setting up all the
// objects that Pilosa needs.

package server

import (
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"crypto/tls"

	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/boltdb"
	"github.com/pilosa/pilosa/gcnotify"
	"github.com/pilosa/pilosa/gopsutil"
	"github.com/pilosa/pilosa/gossip"
	"github.com/pilosa/pilosa/statik"
	"github.com/pilosa/pilosa/statsd"
	"github.com/pkg/errors"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

type loggerLogger interface {
	pilosa.Logger
	Logger() *log.Logger
}

// Command represents the state of the pilosa server command.
type Command struct {
	Server *pilosa.Server

	// Configuration.
	Config *Config

	// Gossip transport
	GossipTransport *gossip.Transport

	// Standard input/output
	*pilosa.CmdIO

	// Started will be closed once Command.Run is finished.
	Started chan struct{}
	// Done will be closed when Command.Close() is called
	Done chan struct{}

	// Passed to the Gossip implementation.
	logOutput io.Writer
	logger    loggerLogger
}

// NewCommand returns a new instance of Main.
func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
	return &Command{
		Config: NewConfig(),

		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),

		Started: make(chan struct{}),
		Done:    make(chan struct{}),
	}
}

// Start starts the pilosa server - it returns once the server is running.
func (m *Command) Start() (err error) {
	defer close(m.Started)

	// SetupServer
	err = m.SetupServer()
	if err != nil {
		return errors.Wrap(err, "setting up server")
	}

	// SetupNetworking
	err = m.SetupNetworking()
	if err != nil {
		return errors.Wrap(err, "setting up networking")
	}

	// Initialize server.
	if err = m.Server.Open(); err != nil {
		return errors.Wrap(err, "opening server")
	}

	m.logger.Printf("Listening as %s\n", m.Server.URI)

	return nil
}

// Wait waits for the server to be closed or interrupted.
func (m *Command) Wait() error {
	// First SIGKILL causes server to shut down gracefully.
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	select {
	case sig := <-c:
		m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String())

		// Second signal causes a hard shutdown.
		go func() { <-c; os.Exit(1) }()
		return errors.Wrap(m.Close(), "closing command")
	case <-m.Done:
		m.logger.Printf("Server closed externally")
		return nil
	}
}

// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
	var err error
	if m.Config.LogPath == "" {
		m.logOutput = m.Stderr
	} else {
		m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			return errors.Wrap(err, "opening file")
		}
	}

	if m.Config.Verbose {
		m.logger = pilosa.NewVerboseLogger(m.logOutput)
	} else {
		m.logger = pilosa.NewStandardLogger(m.logOutput)
	}
	return nil
}

// SetupServer uses the cluster configuration to set up this server.
func (m *Command) SetupServer() error {
	err := m.setupLogger()
	if err != nil {
		return errors.Wrap(err, "setting up logger")
	}

	productName := "Pilosa"
	if pilosa.EnterpriseEnabled {
		productName += " Enterprise"
	}
	m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime)

	handler := pilosa.NewHandler()
	handler.Logger = m.logger
	handler.FileSystem = &statik.FileSystem{}
	handler.API = pilosa.NewAPI()
	handler.API.Logger = m.logger

	uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
	if err != nil {
		return errors.Wrap(err, "processing bind address")
	}

	// Setup TLS
	var TLSConfig *tls.Config
	if uri.Scheme() == "https" {
		if m.Config.TLS.CertificatePath == "" {
			return errors.New("certificate path is required for TLS sockets")
		}
		if m.Config.TLS.CertificateKeyPath == "" {
			return errors.New("certificate key path is required for TLS sockets")
		}
		cert, err := tls.LoadX509KeyPair(m.Config.TLS.CertificatePath, m.Config.TLS.CertificateKeyPath)
		if err != nil {
			return errors.Wrap(err, "load x509 key pair")
		}
		TLSConfig = &tls.Config{
			Certificates:       []tls.Certificate{cert},
			InsecureSkipVerify: m.Config.TLS.SkipVerify,
		}
	}

	diagnosticsInterval := time.Duration(0)
	if m.Config.Metric.Diagnostics {
		diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval)
	}

	statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
	if err != nil {
		return errors.Wrap(err, "new stats client")
	}

	ln, err := getListener(*uri, TLSConfig)
	if err != nil {
		return errors.Wrap(err, "getting listener")
	}

	c := GetHTTPClient(TLSConfig)
	handler.API.RemoteClient = c

	m.Server, err = pilosa.NewServer(
		pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)),
		pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)),
		pilosa.OptServerDataDir(m.Config.DataDir),
		pilosa.OptServerReplicaN(m.Config.Cluster.ReplicaN),
		pilosa.OptServerMaxWritesPerRequest(m.Config.MaxWritesPerRequest),
		pilosa.OptServerMetricInterval(time.Duration(m.Config.Metric.PollInterval)),
		pilosa.OptServerDiagnosticsInterval(diagnosticsInterval),

		pilosa.OptServerLogger(m.logger),
		pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
		pilosa.OptServerHandler(handler),
		pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
		pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
		pilosa.OptServerStatsClient(statsClient),
		pilosa.OptServerListener(ln),
		pilosa.OptServerURI(uri),
		pilosa.OptServerRemoteClient(c),
	)

	return errors.Wrap(err, "new server")
}

func GetHTTPClient(t *tls.Config) *http.Client {
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          1000,
		MaxIdleConnsPerHost:   200,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}
	if t != nil {
		transport.TLSClientConfig = t
	}
	return &http.Client{Transport: transport}
}

// SetupNetworking sets up internode communication based on the configuration.
func (m *Command) SetupNetworking() error {

	m.Server.NodeID = m.Server.LoadNodeID()

	if m.Config.Cluster.Disabled {
		m.Server.Cluster.Static = true
		m.Server.Cluster.Coordinator = m.Server.NodeID
		for _, address := range m.Config.Cluster.Hosts {
			uri, err := pilosa.NewURIFromAddress(address)
			if err != nil {
				return errors.Wrap(err, "getting URI")
			}
			m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{
				URI: *uri,
			})
		}

		m.Server.Broadcaster = pilosa.NopBroadcaster
		m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes)
		m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
		m.Server.Gossiper = pilosa.NopGossiper
		return nil
	}

	gossipPort, err := strconv.Atoi(m.Config.Gossip.Port)
	if err != nil {
		return errors.Wrap(err, "parsing port")
	}

	// get the host portion of addr to use for binding
	gossipHost := m.Server.URI.Host()
	var transport *gossip.Transport
	if m.GossipTransport != nil {
		transport = m.GossipTransport
	} else {
		transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
		if err != nil {
			return errors.Wrap(err, "getting transport")
		}
	}

	// Set Coordinator.
	if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 {
		m.Server.Cluster.Coordinator = m.Server.NodeID
		m.Server.Cluster.Node.IsCoordinator = true
	}

	gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger)
	m.Server.Cluster.EventReceiver = gossipEventReceiver
	gossipMemberSet, err := gossip.NewGossipMemberSet(
		m.Server.NodeID,
		m.Server.URI.Host(),
		m.Config.Gossip,
		gossipEventReceiver,
		m.Server,
		gossip.WithLogger(m.logger.Logger()),
		gossip.WithTransport(transport),
	)
	if err != nil {
		return errors.Wrap(err, "getting memberset")
	}
	gossipMemberSet.Logger = m.logger
	m.Server.Cluster.MemberSet = gossipMemberSet
	m.Server.Broadcaster = m.Server
	m.Server.BroadcastReceiver = gossipMemberSet
	m.Server.Gossiper = gossipMemberSet
	return nil
}

// Close shuts down the server.
func (m *Command) Close() error {
	var logErr error
	serveErr := m.Server.Close()
	if closer, ok := m.logOutput.(io.Closer); ok {
		logErr = closer.Close()
	}
	close(m.Done)
	if serveErr != nil && logErr != nil {
		return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr)
	} else if logErr != nil {
		return logErr
	}
	return serveErr
}

// NewStatsClient creates a stats client from the config
func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
	switch name {
	case "expvar":
		return pilosa.NewExpvarStatsClient(), nil
	case "statsd":
		return statsd.NewStatsClient(host)
	case "nop", "none":
		return pilosa.NopStatsClient, nil
	default:
		return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].")
	}
}

// getListener gets a net.Listener based on the config.
func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
	// If bind URI has the https scheme, enable TLS
	if uri.Scheme() == "https" && tlsconf != nil {
		ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
		if err != nil {
			return nil, errors.Wrap(err, "tls.Listener")
		}
	} else if uri.Scheme() == "http" {
		// Open HTTP listener to determine port (if specified as :0).
		ln, err = net.Listen("tcp", uri.HostPort())
		if err != nil {
			return nil, errors.Wrap(err, "net.Listen")
		}
	} else {
		return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme())
	}

	return ln, nil
}
<|endoftext|>


<commit_before>package formatter

import (
	"encoding/json"
	"fmt"

	"github.com/hashicorp/hcl2/hcl"
	"github.com/wata727/tflint/tflint"
)

type jsonIssue struct {
	Rule    jsonRule    `json:"rule"`
	Message string      `json:"message"`
	Range   jsonRange   `json:"range"`
	Callers []jsonRange `json:"callers"`
}

type jsonRule struct {
	Name     string `json:"name"`
	Severity string `json:"severity"`
	Link     string `json:"link"`
}

type jsonRange struct {
	Filename string  `json:"filename"`
	Start    jsonPos `json:"start"`
	End      jsonPos `json:"end"`
}

type jsonPos struct {
	Line   int `json:"line"`
	Column int `json:"column"`
}

type jsonError struct {
	Message string `json:"message"`
}

// JSONOutput is a temporary structure for converting to JSON
type JSONOutput struct {
	Issues []jsonIssue `json:"issues"`
	Errors []jsonError `json:"errors"`
}

func (f *Formatter) jsonPrint(issues tflint.Issues, tferr *tflint.Error) {
	ret := &JSONOutput{Issues: make([]jsonIssue, len(issues)), Errors: []jsonError{}}

	for idx, issue := range issues.Sort() {
		ret.Issues[idx] = jsonIssue{
			Rule: jsonRule{
				Name:     issue.Rule.Name(),
				Severity: toSeverity(issue.Rule.Severity()),
				Link:     issue.Rule.Link(),
			},
			Message: issue.Message,
			Range: jsonRange{
				Filename: issue.Range.Filename,
				Start:    jsonPos{Line: issue.Range.Start.Line, Column: issue.Range.Start.Column},
				End:      jsonPos{Line: issue.Range.End.Line, Column: issue.Range.End.Column},
			},
			Callers: make([]jsonRange, len(issue.Callers)),
		}
		for i, caller := range issue.Callers {
			ret.Issues[idx].Callers[i] = jsonRange{
				Filename: caller.Filename,
				Start:    jsonPos{Line: caller.Start.Line, Column: caller.Start.Column},
				End:      jsonPos{Line: caller.End.Line, Column: caller.End.Column},
			}
		}
	}

	if tferr != nil {
		errs := []error{}
		if diags, ok := tferr.Cause.(hcl.Diagnostics); ok {
			errs = diags.Errs()
		} else {
			errs = []error{tferr.Cause}
		}

		ret.Errors = make([]jsonError, len(errs))
		for idx, err := range errs {
			ret.Errors[idx] = jsonError{Message: err.Error()}
		}
	}

	out, err := json.Marshal(ret)
	if err != nil {
		fmt.Fprint(f.Stderr, err)
	}
	fmt.Fprint(f.Stdout, string(out))
}
<commit_msg>Correct ineffassign<commit_after>package formatter

import (
	"encoding/json"
	"fmt"

	"github.com/hashicorp/hcl2/hcl"
	"github.com/wata727/tflint/tflint"
)

type jsonIssue struct {
	Rule    jsonRule    `json:"rule"`
	Message string      `json:"message"`
	Range   jsonRange   `json:"range"`
	Callers []jsonRange `json:"callers"`
}

type jsonRule struct {
	Name     string `json:"name"`
	Severity string `json:"severity"`
	Link     string `json:"link"`
}

type jsonRange struct {
	Filename string  `json:"filename"`
	Start    jsonPos `json:"start"`
	End      jsonPos `json:"end"`
}

type jsonPos struct {
	Line   int `json:"line"`
	Column int `json:"column"`
}

type jsonError struct {
	Message string `json:"message"`
}

// JSONOutput is a temporary structure for converting to JSON
type JSONOutput struct {
	Issues []jsonIssue `json:"issues"`
	Errors []jsonError `json:"errors"`
}

func (f *Formatter) jsonPrint(issues tflint.Issues, tferr *tflint.Error) {
	ret := &JSONOutput{Issues: make([]jsonIssue, len(issues)), Errors: []jsonError{}}

	for idx, issue := range issues.Sort() {
		ret.Issues[idx] = jsonIssue{
			Rule: jsonRule{
				Name:     issue.Rule.Name(),
				Severity: toSeverity(issue.Rule.Severity()),
				Link:     issue.Rule.Link(),
			},
			Message: issue.Message,
			Range: jsonRange{
				Filename: issue.Range.Filename,
				Start:    jsonPos{Line: issue.Range.Start.Line, Column: issue.Range.Start.Column},
				End:      jsonPos{Line: issue.Range.End.Line, Column: issue.Range.End.Column},
			},
			Callers: make([]jsonRange, len(issue.Callers)),
		}
		for i, caller := range issue.Callers {
			ret.Issues[idx].Callers[i] = jsonRange{
				Filename: caller.Filename,
				Start:    jsonPos{Line: caller.Start.Line, Column: caller.Start.Column},
				End:      jsonPos{Line: caller.End.Line, Column: caller.End.Column},
			}
		}
	}

	if tferr != nil {
		var errs []error
		if diags, ok := tferr.Cause.(hcl.Diagnostics); ok {
			errs = diags.Errs()
		} else {
			errs = []error{tferr.Cause}
		}

		ret.Errors = make([]jsonError, len(errs))
		for idx, err := range errs {
			ret.Errors[idx] = jsonError{Message: err.Error()}
		}
	}

	out, err := json.Marshal(ret)
	if err != nil {
		fmt.Fprint(f.Stderr, err)
	}
	fmt.Fprint(f.Stdout, string(out))
}
<|endoftext|>