<commit_before>package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"math/rand"
	"os"
	"time"

	"golang.org/x/tools/imports"
	"kego.io/kerr"
	"kego.io/kerr/kerrsource"
)

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	flag.Parse()
	if err := processFile(flag.Arg(0)); err != nil {
		panic(err.Error())
	}
}

func processFile(filename string) error {

	if filename == "" {
		return kerr.New("CKBKFTNVXI", "File not found")
	}

	options := &imports.Options{
		TabWidth:  4,
		TabIndent: true,
		Comments:  true,
		Fragment:  true,
	}

	f, err := os.Open(filename)
	if err != nil {
		return kerr.Wrap("WCQEVUWMAI", err)
	}
	defer f.Close()

	src, err := ioutil.ReadAll(f)
	if err != nil {
		return kerr.Wrap("DWTHNAVSVY", err)
	}

	var res []byte

	res, err = imports.Process(filename, src, options)
	if err != nil {
		return kerr.Wrap("XFILQRWHSI", err)
	}

	res, err = kerrsource.Process(filename, res)
	if err != nil {
		return kerr.Wrap("UFVVOEXSYE", err)
	}

	if !bytes.Equal(src, res) {
		// formatting has changed
		err = ioutil.WriteFile(filename, res, 0)
		if err != nil {
			return kerr.Wrap("NFFAOHJQLG", err)
		}
	}

	return nil

}
<commit_msg>Change processing order in kesource<commit_after>package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"math/rand"
	"os"
	"time"

	"golang.org/x/tools/imports"
	"kego.io/kerr"
	"kego.io/kerr/kerrsource"
)

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	flag.Parse()
	if err := processFile(flag.Arg(0)); err != nil {
		panic(err.Error())
	}
}

func processFile(filename string) error {

	if filename == "" {
		return kerr.New("CKBKFTNVXI", "File not found")
	}

	options := &imports.Options{
		TabWidth:  4,
		TabIndent: true,
		Comments:  true,
		Fragment:  true,
	}

	f, err := os.Open(filename)
	if err != nil {
		return kerr.Wrap("WCQEVUWMAI", err)
	}
	defer f.Close()

	src, err := ioutil.ReadAll(f)
	if err != nil {
		return kerr.Wrap("DWTHNAVSVY", err)
	}

	var res []byte

	res, err = kerrsource.Process(filename, res)
	if err != nil {
		return kerr.Wrap("UFVVOEXSYE", err)
	}

	res, err = imports.Process(filename, src, options)
	if err != nil {
		return kerr.Wrap("XFILQRWHSI", err)
	}

	if !bytes.Equal(src, res) {
		// formatting has changed
		err = ioutil.WriteFile(filename, res, 0)
		if err != nil {
			return kerr.Wrap("NFFAOHJQLG", err)
		}
	}

	return nil

}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// lesser — less than less. Work in progress.
package main

import (
	"fmt"
	"io/ioutil"
	"os"

	"github.com/gdamore/tcell"
	"github.com/gdamore/tcell/views"
)

type root struct {
	app *views.Application

	views.BoxLayout
}

func (r *root) HandleEvent(ev tcell.Event) bool {
	switch ev := ev.(type) {
	case *tcell.EventKey:
		switch ev.Key() {
		case tcell.KeyCtrlL:
			r.app.Refresh()
			return true
		case tcell.KeyRune:
			switch ev.Rune() {
			case 'Q', 'q':
				r.app.Quit()
				return true
			}
		}
	}
	return r.BoxLayout.HandleEvent(ev)
}

func (r *root) Draw() {
	r.BoxLayout.Draw()
}

func lesser(filename string) error {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	app := &views.Application{}
	app.SetStyle(tcell.StyleDefault.
		Foreground(tcell.ColorBlack).
		Background(tcell.ColorWhite))

	root := &root{app: app}
	root.SetOrientation(views.Vertical)

	ta := views.NewTextArea()
	ta.SetContent(string(buf))

	root.InsertWidget(0, ta, 1.0)

	app.SetRootWidget(root)
	if err := app.Run(); err != nil {
		return err
	}
	return nil
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s filename\n", os.Args[0])
	os.Exit(1)
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	if len(os.Args) != 2 {
		usage()
	}
	if err := lesser(os.Args[1]); err != nil {
		fatal(err)
	}
}
<commit_msg>remove lesser command<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/codegangsta/cli"
)

func newListMachinesCommand() cli.Command {
	return cli.Command{
		Name:   "list-machines",
		Usage:  "Enumerate the current hosts in the cluster",
		Action: listMachinesAction,
		Flags: []cli.Flag{
			cli.BoolFlag{"full, l", "Do not ellipsize fields on output"},
			cli.BoolFlag{"no-legend", "Do not print a legend (column headers)"},
		},
	}
}

func listMachinesAction(c *cli.Context) {
	r := getRegistry(c)

	if !c.Bool("no-legend") {
		fmt.Fprintln(out, "MACHINE\tIP\tMETADATA")
	}

	full := c.Bool("full")

	for _, m := range r.GetActiveMachines() {
		mach := m.BootId
		if !full {
			mach = ellipsize(mach, 8)
		}

		fmt.Fprintf(out, "%s\t%s\t%s\n", mach, m.PublicIP, formatMetadata(m.Metadata))
	}

	out.Flush()
}

func formatMetadata(metadata map[string]string) string {
	pairs := make([]string, len(metadata))
	idx := 0
	for key, value := range metadata {
		pairs[idx] = fmt.Sprintf("%s=%s", key, value)
		idx++
	}
	return strings.Join(pairs, ", ")
}
<commit_msg>feat(corectl): print hyphens for no-data in list-machines<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/codegangsta/cli"
)

func newListMachinesCommand() cli.Command {
	return cli.Command{
		Name:   "list-machines",
		Usage:  "Enumerate the current hosts in the cluster",
		Action: listMachinesAction,
		Flags: []cli.Flag{
			cli.BoolFlag{"full, l", "Do not ellipsize fields on output"},
			cli.BoolFlag{"no-legend", "Do not print a legend (column headers)"},
		},
	}
}

func listMachinesAction(c *cli.Context) {
	r := getRegistry(c)

	if !c.Bool("no-legend") {
		fmt.Fprintln(out, "MACHINE\tIP\tMETADATA")
	}

	full := c.Bool("full")

	for _, m := range r.GetActiveMachines() {
		mach := m.BootId
		if !full {
			mach = ellipsize(mach, 8)
		}

		ip := m.PublicIP
		if len(ip) == 0 {
			ip = "-"
		}

		metadata := "-"
		if len(m.Metadata) != 0 {
			metadata = formatMetadata(m.Metadata)
		}

		fmt.Fprintf(out, "%s\t%s\t%s\n", mach, ip, metadata)
	}

	out.Flush()
}

func formatMetadata(metadata map[string]string) string {
	pairs := make([]string, len(metadata))
	idx := 0
	for key, value := range metadata {
		pairs[idx] = fmt.Sprintf("%s=%s", key, value)
		idx++
	}
	return strings.Join(pairs, ", ")
}
<|endoftext|>


<commit_before>/*
Copyright © 2014–5 Brad Ackerman.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/backerman/evego"
	"github.com/backerman/evego/pkg/cache"
	"github.com/backerman/evego/pkg/dbaccess"
	"github.com/backerman/evego/pkg/eveapi"
	"github.com/backerman/evego/pkg/evesso"
	"github.com/backerman/evego/pkg/market"
	"github.com/backerman/evego/pkg/routing"
	"github.com/backerman/eveindy/pkg/api"
	"github.com/backerman/eveindy/pkg/db"
	"github.com/backerman/eveindy/pkg/server"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/zenazn/goji"
	// Register PgSQL driver
	_ "github.com/lib/pq"
)

var c config
var rootCmd *cobra.Command

type config struct {
	Dev                      bool
	DbDriver, DbPath         string
	Bind                     string
	XMLAPIEndpoint           string
	Router                   string
	Cache                    string
	RedisHost, RedisPassword string
	CookieDomain, CookiePath string
	ClientID, ClientSecret   string
	RedirectURL              string
}

func setRoutes(sde evego.Database, localdb db.LocalDB, xmlAPI evego.XMLAPI,
	eveCentral evego.Market, sessionizer server.Sessionizer, cache evego.Cache) {
	assets := http.FileServer(http.Dir("dist"))
	bower := http.FileServer(http.Dir("bower_components"))
	if c.Dev {
		goji.Get("/bower_components/*", http.StripPrefix("/bower_components/", bower))
	}
	goji.Get("/autocomplete/system/:name", api.AutocompleteSystems(sde))
	goji.Get("/autocomplete/station/:name", api.AutocompleteStations(sde, localdb, xmlAPI))
	goji.Post("/pastebin", api.ParseItems(sde))
	marketHandler := api.ItemsMarketValue(sde, eveCentral, xmlAPI)
	// For now these do the same thing. That may change.
	goji.Post("/market/region/:location", marketHandler)
	goji.Post("/market/system/:location", marketHandler)
	goji.Post("/market/station/:id", marketHandler)
	goji.Get("/market/jita", api.ReprocessOutputValues(sde, eveCentral, xmlAPI, cache))

	goji.Post("/reprocess", api.ReprocessItems(sde, eveCentral))
	// SSO!
	auth := evesso.MakeAuthenticator(evesso.Endpoint, c.ClientID, c.ClientSecret,
		c.RedirectURL, evesso.PublicData)
	goji.Get("/crestcallback", api.CRESTCallbackListener(localdb, auth, sessionizer))
	goji.Get("/authenticate", api.AuthenticateHandler(auth, sessionizer))
	goji.Get("/session", api.SessionInfo(auth, sessionizer, localdb))
	goji.Post("/logout", api.LogoutHandler(localdb, auth, sessionizer))

	// API keys
	listHandler, deleteHander, addHandler, refreshHandler := api.XMLAPIKeysHandlers(localdb, sessionizer)
	goji.Get("/apikeys/list", listHandler)
	goji.Post("/apikeys/delete/:keyid", deleteHander)
	goji.Post("/apikeys/add", addHandler)
	goji.Post("/apikeys/refresh", refreshHandler)

	// Standings and skills
	goji.Get("/standings/:charID/:npcCorpID", api.StandingsHandler(localdb, sessionizer))
	goji.Get("/skills/:charID/group/:skillGroupID", api.SkillsHandler(localdb, sessionizer))

	// Blueprints and industry
	_, getBPs := api.BlueprintsHandlers(localdb, sde, sessionizer)
	goji.Get("/blueprints/:charID", getBPs)
	goji.Get("/assets/unusedSalvage/:charID", api.UnusedSalvage(localdb, sde, sessionizer))

	// Static assets
	goji.Get("/*", assets)
}

func mainCommand(cmd *cobra.Command, args []string) {
	err := viper.Unmarshal(&c)
	if err != nil {
		log.Fatalf("Unable to marshal configuration: %v", err)
	}
	if !viper.IsSet("dbpath") {
		log.Fatalf("Please set the dbpath configuration option or EVEINDY_DBPATH " +
			"environment variable to the database's path.")
	}

	if !(viper.IsSet("CookieDomain") && viper.IsSet("CookiePath")) {
		log.Fatalf("Please set the CookieDomain and CookiePath configuration options.")
	}

	if !(viper.IsSet("ClientID") && viper.IsSet("ClientSecret") &&
		viper.IsSet("RedirectURL")) {
		log.Fatalf("Please set the ClientID, ClientSecret, and RedirectURL configuration " +
			"options as registered with CCP.")
	}

	if c.Dev {
		log.Printf("Configuration is: %+v", c)
	}

	sde := dbaccess.SQLDatabase(c.DbDriver, c.DbPath)
	var myCache evego.Cache
	switch c.Cache {
	case "inproc":
		myCache = server.InMemCache()
	case "redis":
		if c.RedisPassword != "" {
			myCache = cache.RedisCache(c.RedisHost, c.RedisPassword)
		} else {
			myCache = cache.RedisCache(c.RedisHost)
		}
	default:
		log.Fatalf(
			"The Cache configuration option must be set to \"inproc\" (default) or \"redis\".")
	}

	xmlAPI := eveapi.XML(c.XMLAPIEndpoint, sde, myCache)
	localdb, err := db.Interface(c.DbDriver, c.DbPath, xmlAPI)
	if err != nil {
		log.Fatalf("Unable to connect to local database: %v", err)
	}
	var router evego.Router

	switch c.Router {
	case "evecentral":
		router = routing.EveCentralRouter(
			"http://api.eve-central.com/api/route", myCache)
	case "sql":
		router = routing.SQLRouter(c.DbDriver, c.DbPath, myCache)
	default:
		log.Fatalf(
			"The Router configuration option must be set to \"evecentral\" (default) or \"sql\".")
	}

	eveCentralMarket := market.EveCentral(sde, router, xmlAPI,
		"http://api.eve-central.com/api/quicklook", myCache)

	sessionizer := server.GetSessionizer(c.CookieDomain, c.CookiePath, !c.Dev, localdb)

	setRoutes(sde, localdb, xmlAPI, eveCentralMarket, sessionizer, myCache)

	// Set up internal bits.

	// Start background jobs.
	server.StartJobs(localdb)

	// We like magic, but fix the magic some.
	bindArg := fmt.Sprintf("-bind=%s", c.Bind)
	if len(os.Args) > 1 {
		os.Args[1] = bindArg
		os.Args = os.Args[:2]
	} else {
		os.Args = append(os.Args, bindArg)
	}

	goji.Serve()
}

func main() {
	rootCmd = &cobra.Command{
		Use:   "server",
		Short: "The server component of eveindy",
		Run:   mainCommand,
	}

	// Configuration defaults

	// When DevMode is true, server only listens on localhost and will serve
	// external dependencies (e.g. AngularJS) from local disk instead of from
	// a CDN.
	viper.SetDefault("DevMode", false)
	// DBDriver and DBPath are the database driver name and resource path
	// as used by the Golang SQL library.
	viper.SetDefault("DBDriver", "sqlite3")
	// DBPath has no default. You must set it.
	// viper.SetDefault("DBPath", "")
	viper.SetDefault("Bind", "*:8080")
	viper.SetDefault("XMLAPIEndpoint", "https://api.eveonline.com")
	// Routing
	// Router: either "evecentral" or "sql".
	viper.SetDefault("Router", "evecentral")

	// Session cookies - you must set these explicitly.
	// viper.SetDefault("CookieDomain", "localhost")
	// viper.SetDefault("CookiePath", "/")

	// Cache
	// The default is an in-process cache, but you should probably use Redis
	// istead.
	viper.SetDefault("Cache", "inproc")
	viper.SetDefault("RedisHost", ":6379")
	viper.SetDefault("RedisPassword", "")

	// Set configuration file
	viper.SetConfigName("config")
	viper.AddConfigPath("/etc/eveindy")
	viper.AddConfigPath("$HOME/.eveindy")
	viper.ReadInConfig()

	// Environment variables
	viper.SetEnvPrefix("EVEINDY")
	viper.AutomaticEnv()

	rootCmd.Flags().Bool("dev", false, "Set development mode.")
	rootCmd.Flags().String("bind", ":8080", "The address and port to listen on.")

	flags := []string{"dev", "bind"}
	for _, flag := range flags {
		viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag))
	}
	rootCmd.Execute()
}
<commit_msg>Add workaround for viper bug.<commit_after>/*
Copyright © 2014–5 Brad Ackerman.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/backerman/evego"
	"github.com/backerman/evego/pkg/cache"
	"github.com/backerman/evego/pkg/dbaccess"
	"github.com/backerman/evego/pkg/eveapi"
	"github.com/backerman/evego/pkg/evesso"
	"github.com/backerman/evego/pkg/market"
	"github.com/backerman/evego/pkg/routing"
	"github.com/backerman/eveindy/pkg/api"
	"github.com/backerman/eveindy/pkg/db"
	"github.com/backerman/eveindy/pkg/server"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/zenazn/goji"
	// Register PgSQL driver
	_ "github.com/lib/pq"
)

var c config
var rootCmd *cobra.Command

type config struct {
	Dev                      bool
	DbDriver, DbPath         string
	Bind                     string
	XMLAPIEndpoint           string
	Router                   string
	Cache                    string
	RedisHost, RedisPassword string
	CookieDomain, CookiePath string
	ClientID, ClientSecret   string
	RedirectURL              string
}

func setRoutes(sde evego.Database, localdb db.LocalDB, xmlAPI evego.XMLAPI,
	eveCentral evego.Market, sessionizer server.Sessionizer, cache evego.Cache) {
	assets := http.FileServer(http.Dir("dist"))
	bower := http.FileServer(http.Dir("bower_components"))
	if c.Dev {
		goji.Get("/bower_components/*", http.StripPrefix("/bower_components/", bower))
	}
	goji.Get("/autocomplete/system/:name", api.AutocompleteSystems(sde))
	goji.Get("/autocomplete/station/:name", api.AutocompleteStations(sde, localdb, xmlAPI))
	goji.Post("/pastebin", api.ParseItems(sde))
	marketHandler := api.ItemsMarketValue(sde, eveCentral, xmlAPI)
	// For now these do the same thing. That may change.
	goji.Post("/market/region/:location", marketHandler)
	goji.Post("/market/system/:location", marketHandler)
	goji.Post("/market/station/:id", marketHandler)
	goji.Get("/market/jita", api.ReprocessOutputValues(sde, eveCentral, xmlAPI, cache))

	goji.Post("/reprocess", api.ReprocessItems(sde, eveCentral))
	// SSO!
	auth := evesso.MakeAuthenticator(evesso.Endpoint, c.ClientID, c.ClientSecret,
		c.RedirectURL, evesso.PublicData)
	goji.Get("/crestcallback", api.CRESTCallbackListener(localdb, auth, sessionizer))
	goji.Get("/authenticate", api.AuthenticateHandler(auth, sessionizer))
	goji.Get("/session", api.SessionInfo(auth, sessionizer, localdb))
	goji.Post("/logout", api.LogoutHandler(localdb, auth, sessionizer))

	// API keys
	listHandler, deleteHander, addHandler, refreshHandler := api.XMLAPIKeysHandlers(localdb, sessionizer)
	goji.Get("/apikeys/list", listHandler)
	goji.Post("/apikeys/delete/:keyid", deleteHander)
	goji.Post("/apikeys/add", addHandler)
	goji.Post("/apikeys/refresh", refreshHandler)

	// Standings and skills
	goji.Get("/standings/:charID/:npcCorpID", api.StandingsHandler(localdb, sessionizer))
	goji.Get("/skills/:charID/group/:skillGroupID", api.SkillsHandler(localdb, sessionizer))

	// Blueprints and industry
	_, getBPs := api.BlueprintsHandlers(localdb, sde, sessionizer)
	goji.Get("/blueprints/:charID", getBPs)
	goji.Get("/assets/unusedSalvage/:charID", api.UnusedSalvage(localdb, sde, sessionizer))

	// Static assets
	goji.Get("/*", assets)
}

func mainCommand(cmd *cobra.Command, args []string) {
	err := viper.Unmarshal(&c)
	if err != nil {
		log.Fatalf("Unable to marshal configuration: %v", err)
	}
	if !viper.IsSet("dbpath") {
		log.Fatalf("Please set the dbpath configuration option or EVEINDY_DBPATH " +
			"environment variable to the database's path.")
	}

	if !(viper.IsSet("CookieDomain") && viper.IsSet("CookiePath")) {
		log.Fatalf("Please set the CookieDomain and CookiePath configuration options.")
	}

	if !(viper.IsSet("ClientID") && viper.IsSet("ClientSecret") &&
		viper.IsSet("RedirectURL")) {
		log.Fatalf("Please set the ClientID, ClientSecret, and RedirectURL configuration " +
			"options as registered with CCP.")
	}
	// workaround for viper bug
	// c.Dev = viper.GetBool("Dev")

	if c.Dev {
		log.Printf("Configuration is: %+v", c)
	}

	sde := dbaccess.SQLDatabase(c.DbDriver, c.DbPath)
	var myCache evego.Cache
	switch c.Cache {
	case "inproc":
		myCache = server.InMemCache()
	case "redis":
		if c.RedisPassword != "" {
			myCache = cache.RedisCache(c.RedisHost, c.RedisPassword)
		} else {
			myCache = cache.RedisCache(c.RedisHost)
		}
	default:
		log.Fatalf(
			"The Cache configuration option must be set to \"inproc\" (default) or \"redis\".")
	}

	xmlAPI := eveapi.XML(c.XMLAPIEndpoint, sde, myCache)
	localdb, err := db.Interface(c.DbDriver, c.DbPath, xmlAPI)
	if err != nil {
		log.Fatalf("Unable to connect to local database: %v", err)
	}
	var router evego.Router

	switch c.Router {
	case "evecentral":
		router = routing.EveCentralRouter(
			"http://api.eve-central.com/api/route", myCache)
	case "sql":
		router = routing.SQLRouter(c.DbDriver, c.DbPath, myCache)
	default:
		log.Fatalf(
			"The Router configuration option must be set to \"evecentral\" (default) or \"sql\".")
	}

	eveCentralMarket := market.EveCentral(sde, router, xmlAPI,
		"http://api.eve-central.com/api/quicklook", myCache)

	sessionizer := server.GetSessionizer(c.CookieDomain, c.CookiePath, !c.Dev, localdb)

	setRoutes(sde, localdb, xmlAPI, eveCentralMarket, sessionizer, myCache)

	// Set up internal bits.

	// Start background jobs.
	server.StartJobs(localdb)

	// We like magic, but fix the magic some.
	bindArg := fmt.Sprintf("-bind=%s", c.Bind)
	if len(os.Args) > 1 {
		os.Args[1] = bindArg
		os.Args = os.Args[:2]
	} else {
		os.Args = append(os.Args, bindArg)
	}

	goji.Serve()
}

func main() {
	rootCmd = &cobra.Command{
		Use:   "server",
		Short: "The server component of eveindy",
		Run:   mainCommand,
	}
	// Configuration defaults
	// When dev is true, server only listens on localhost and will serve
	// external dependencies (e.g. AngularJS) from local disk instead of from
	// a CDN.
	rootCmd.Flags().Bool("Dev", false, "Set development mode.")
	rootCmd.Flags().String("Bind", ":8080", "The address and port to listen on.")
	viper.SetDefault("Dev", false)
	// DBDriver and DBPath are the database driver name and resource path
	// as used by the Golang SQL library.
	viper.SetDefault("DBDriver", "sqlite3")
	// DBPath has no default. You must set it.
	// viper.SetDefault("DBPath", "")
	viper.SetDefault("Bind", "*:8080")
	viper.SetDefault("XMLAPIEndpoint", "https://api.eveonline.com")
	// Routing
	// Router: either "evecentral" or "sql".
	viper.SetDefault("Router", "evecentral")

	// Session cookies - you must set these explicitly.
	// viper.SetDefault("CookieDomain", "localhost")
	// viper.SetDefault("CookiePath", "/")

	// Cache
	// The default is an in-process cache, but you should probably use Redis
	// istead.
	viper.SetDefault("Cache", "inproc")
	viper.SetDefault("RedisHost", ":6379")
	viper.SetDefault("RedisPassword", "")

	// Set configuration file
	viper.SetConfigName("config")
	viper.AddConfigPath("/etc/eveindy")
	viper.AddConfigPath("$HOME/.eveindy")
	viper.ReadInConfig()

	// Environment variables
	viper.SetEnvPrefix("EVEINDY")
	viper.AutomaticEnv()

	flags := []string{"Dev", "Bind"}
	for _, flag := range flags {
		viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag))
	}
	rootCmd.Execute()
}
<|endoftext|>


<commit_before>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"path/filepath"
	"strings"
	"time"

	"archive/tar"
	"archive/zip"
	"compress/gzip"

	"github.com/richardlehane/characterize"
	"github.com/richardlehane/webarchive"

	"github.com/richardlehane/siegfried/pkg/core/siegreader"
)

type decompressor interface {
	next() error // when finished, should return io.EOF
	reader() io.Reader
	path() string
	mime() string
	size() int64
	mod() string
	dirs() []string
}

type zipD struct {
	idx     int
	p       string
	rdr     *zip.Reader
	rc      io.ReadCloser
	written map[string]bool
}

func newZip(ra io.ReaderAt, path string, sz int64) (decompressor, error) {
	zr, err := zip.NewReader(ra, sz)
	return &zipD{idx: -1, p: path, rdr: zr}, err
}

func (z *zipD) close() {
	if z.rc == nil {
		return
	}
	z.rc.Close()
}

func (z *zipD) next() error {
	z.close() // close the previous entry, if any
	// proceed
	z.idx++
	// scan past directories
	for ; z.idx < len(z.rdr.File) && z.rdr.File[z.idx].FileInfo().IsDir(); z.idx++ {
	}
	if z.idx >= len(z.rdr.File) {
		return io.EOF
	}
	var err error
	z.rc, err = z.rdr.File[z.idx].Open()
	return err
}

func (z *zipD) reader() io.Reader {
	return z.rc
}

func (z *zipD) path() string {
	return arcpath(z.p, filepath.FromSlash(characterize.ZipName(z.rdr.File[z.idx].Name)))
}

func (z *zipD) mime() string {
	return ""
}

func (z *zipD) size() int64 {
	return int64(z.rdr.File[z.idx].UncompressedSize64)
}

func (z *zipD) mod() string {
	return z.rdr.File[z.idx].ModTime().Format(time.RFC3339)
}

func (z *zipD) dirs() []string {
	if z.written == nil {
		z.written = make(map[string]bool)
	}
	return dirs(z.p, characterize.ZipName(z.rdr.File[z.idx].Name), z.written)
}

type tarD struct {
	p       string
	hdr     *tar.Header
	rdr     *tar.Reader
	written map[string]bool
}

func newTar(r io.Reader, path string) (decompressor, error) {
	return &tarD{p: path, rdr: tar.NewReader(r)}, nil
}

func (t *tarD) next() error {
	var err error
	// scan past directories
	for t.hdr, err = t.rdr.Next(); err == nil && t.hdr.FileInfo().IsDir(); t.hdr, err = t.rdr.Next() {
	}
	return err
}

func (t *tarD) reader() io.Reader {
	return t.rdr
}

func (t *tarD) path() string {
	return arcpath(t.p, filepath.FromSlash(t.hdr.Name))
}

func (t *tarD) mime() string {
	return ""
}

func (t *tarD) size() int64 {
	return t.hdr.Size
}

func (t *tarD) mod() string {
	return t.hdr.ModTime.Format(time.RFC3339)
}

func (t *tarD) dirs() []string {
	if t.written == nil {
		t.written = make(map[string]bool)
	}
	return dirs(t.p, t.hdr.Name, t.written)
}

type gzipD struct {
	sz   int64
	p    string
	read bool
	rdr  *gzip.Reader
}

func newGzip(b *siegreader.Buffer, path string) (decompressor, error) {
	_ = b.SizeNow()              // in case of a stream, force full read
	buf, err := b.EofSlice(0, 4) // gzip stores uncompressed size in last 4 bytes of the stream
	if err != nil {
		return nil, err
	}
	sz := int64(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
	g, err := gzip.NewReader(siegreader.ReaderFrom(b))
	return &gzipD{sz: sz, p: path, rdr: g}, err
}

func (g *gzipD) next() error {
	if g.read {
		g.rdr.Close()
		return io.EOF
	}
	g.read = true
	return nil
}

func (g *gzipD) reader() io.Reader {
	return g.rdr
}

func (g *gzipD) path() string {
	name := g.rdr.Name
	if len(name) == 0 {
		switch filepath.Ext(g.p) {
		case ".gz", ".z", ".gzip", ".zip":
			name = strings.TrimSuffix(filepath.Base(g.p), filepath.Ext(g.p))
		default:
			name = filepath.Base(g.p)
		}
	}
	return arcpath(g.p, name)
}

func (g *gzipD) mime() string {
	return ""
}

func (g *gzipD) size() int64 {
	return g.sz
}

func (g *gzipD) mod() string {
	return g.rdr.ModTime.Format(time.RFC3339)
}

func (t *gzipD) dirs() []string {
	return nil
}

func trimWebPath(p string) string {
	d, f := filepath.Split(p)
	clean := strings.TrimSuffix(d, string(filepath.Separator))
	_, f1 := filepath.Split(clean)
	if f == strings.TrimSuffix(f1, filepath.Ext(clean)) {
		return clean
	}
	return p
}

var (
	arcReader  *webarchive.ARCReader
	warcReader *webarchive.WARCReader
)

type wa struct {
	p   string
	rec webarchive.Record
	rdr webarchive.Reader
}

func newARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if arcReader == nil {
		arcReader, err = webarchive.NewARCReader(r)
	} else {
		err = arcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: arcReader}, err
}

func newWARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if warcReader == nil {
		warcReader, err = webarchive.NewWARCReader(r)
	} else {
		err = warcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: warcReader}, err
}

func (w *wa) next() error {
	var err error
	w.rec, err = w.rdr.NextPayload()
	return err
}

func (w *wa) reader() io.Reader {
	return webarchive.DecodePayload(w.rec)
}

func (w *wa) path() string {
	return arcpath(w.p, filepath.Join(w.rec.Date().Format(webarchive.ARCTime), w.rec.URL()))
}

func (w *wa) mime() string {
	return w.rec.MIME()
}

func (w *wa) size() int64 {
	return w.rec.Size()
}

func (w *wa) mod() string {
	return w.rec.Date().Format(time.RFC3339)
}

func (w *wa) dirs() []string {
	return nil
}

func dirs(path, name string, written map[string]bool) []string {
	ds := strings.Split(filepath.ToSlash(name), "/")
	if len(ds) > 1 {
		var ret []string
		for _, p := range ds[:len(ds)-1] {
			path = path + string(filepath.Separator) + p
			if !written[path] {
				ret = append(ret, path)
				written[path] = true
			}
		}
		return ret
	}
	return nil
}

func arcpath(base, path string) string {
	if *droido {
		return base + string(filepath.Separator) + path
	}
	if idx := strings.LastIndex(base, "] "); idx > 0 && idx+2 < len(base) {
		return base[:idx+2] + "[" + base[idx+2:] + "] " + path
	}
	return "[" + base + "] " + path
}
<commit_msg>hash archive paths<commit_after>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"path/filepath"
	"strings"
	"time"

	"archive/tar"
	"archive/zip"
	"compress/gzip"

	"github.com/richardlehane/characterize"
	"github.com/richardlehane/webarchive"

	"github.com/richardlehane/siegfried/pkg/core/siegreader"
)

type decompressor interface {
	next() error // when finished, should return io.EOF
	reader() io.Reader
	path() string
	mime() string
	size() int64
	mod() string
	dirs() []string
}

type zipD struct {
	idx     int
	p       string
	rdr     *zip.Reader
	rc      io.ReadCloser
	written map[string]bool
}

func newZip(ra io.ReaderAt, path string, sz int64) (decompressor, error) {
	zr, err := zip.NewReader(ra, sz)
	return &zipD{idx: -1, p: path, rdr: zr}, err
}

func (z *zipD) close() {
	if z.rc == nil {
		return
	}
	z.rc.Close()
}

func (z *zipD) next() error {
	z.close() // close the previous entry, if any
	// proceed
	z.idx++
	// scan past directories
	for ; z.idx < len(z.rdr.File) && z.rdr.File[z.idx].FileInfo().IsDir(); z.idx++ {
	}
	if z.idx >= len(z.rdr.File) {
		return io.EOF
	}
	var err error
	z.rc, err = z.rdr.File[z.idx].Open()
	return err
}

func (z *zipD) reader() io.Reader {
	return z.rc
}

func (z *zipD) path() string {
	return arcpath(z.p, filepath.FromSlash(characterize.ZipName(z.rdr.File[z.idx].Name)))
}

func (z *zipD) mime() string {
	return ""
}

func (z *zipD) size() int64 {
	return int64(z.rdr.File[z.idx].UncompressedSize64)
}

func (z *zipD) mod() string {
	return z.rdr.File[z.idx].ModTime().Format(time.RFC3339)
}

func (z *zipD) dirs() []string {
	if z.written == nil {
		z.written = make(map[string]bool)
	}
	return dirs(z.p, characterize.ZipName(z.rdr.File[z.idx].Name), z.written)
}

type tarD struct {
	p       string
	hdr     *tar.Header
	rdr     *tar.Reader
	written map[string]bool
}

func newTar(r io.Reader, path string) (decompressor, error) {
	return &tarD{p: path, rdr: tar.NewReader(r)}, nil
}

func (t *tarD) next() error {
	var err error
	// scan past directories
	for t.hdr, err = t.rdr.Next(); err == nil && t.hdr.FileInfo().IsDir(); t.hdr, err = t.rdr.Next() {
	}
	return err
}

func (t *tarD) reader() io.Reader {
	return t.rdr
}

func (t *tarD) path() string {
	return arcpath(t.p, filepath.FromSlash(t.hdr.Name))
}

func (t *tarD) mime() string {
	return ""
}

func (t *tarD) size() int64 {
	return t.hdr.Size
}

func (t *tarD) mod() string {
	return t.hdr.ModTime.Format(time.RFC3339)
}

func (t *tarD) dirs() []string {
	if t.written == nil {
		t.written = make(map[string]bool)
	}
	return dirs(t.p, t.hdr.Name, t.written)
}

type gzipD struct {
	sz   int64
	p    string
	read bool
	rdr  *gzip.Reader
}

func newGzip(b *siegreader.Buffer, path string) (decompressor, error) {
	_ = b.SizeNow()              // in case of a stream, force full read
	buf, err := b.EofSlice(0, 4) // gzip stores uncompressed size in last 4 bytes of the stream
	if err != nil {
		return nil, err
	}
	sz := int64(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
	g, err := gzip.NewReader(siegreader.ReaderFrom(b))
	return &gzipD{sz: sz, p: path, rdr: g}, err
}

func (g *gzipD) next() error {
	if g.read {
		g.rdr.Close()
		return io.EOF
	}
	g.read = true
	return nil
}

func (g *gzipD) reader() io.Reader {
	return g.rdr
}

func (g *gzipD) path() string {
	name := g.rdr.Name
	if len(name) == 0 {
		switch filepath.Ext(g.p) {
		case ".gz", ".z", ".gzip", ".zip":
			name = strings.TrimSuffix(filepath.Base(g.p), filepath.Ext(g.p))
		default:
			name = filepath.Base(g.p)
		}
	}
	return arcpath(g.p, name)
}

func (g *gzipD) mime() string {
	return ""
}

func (g *gzipD) size() int64 {
	return g.sz
}

func (g *gzipD) mod() string {
	return g.rdr.ModTime.Format(time.RFC3339)
}

func (t *gzipD) dirs() []string {
	return nil
}

func trimWebPath(p string) string {
	d, f := filepath.Split(p)
	clean := strings.TrimSuffix(d, string(filepath.Separator))
	_, f1 := filepath.Split(clean)
	if f == strings.TrimSuffix(f1, filepath.Ext(clean)) {
		return clean
	}
	return p
}

var (
	arcReader  *webarchive.ARCReader
	warcReader *webarchive.WARCReader
)

type wa struct {
	p   string
	rec webarchive.Record
	rdr webarchive.Reader
}

func newARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if arcReader == nil {
		arcReader, err = webarchive.NewARCReader(r)
	} else {
		err = arcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: arcReader}, err
}

func newWARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if warcReader == nil {
		warcReader, err = webarchive.NewWARCReader(r)
	} else {
		err = warcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: warcReader}, err
}

func (w *wa) next() error {
	var err error
	w.rec, err = w.rdr.NextPayload()
	return err
}

func (w *wa) reader() io.Reader {
	return webarchive.DecodePayload(w.rec)
}

func (w *wa) path() string {
	return arcpath(w.p, filepath.Join(w.rec.Date().Format(webarchive.ARCTime), w.rec.URL()))
}

func (w *wa) mime() string {
	return w.rec.MIME()
}

func (w *wa) size() int64 {
	return w.rec.Size()
}

func (w *wa) mod() string {
	return w.rec.Date().Format(time.RFC3339)
}

func (w *wa) dirs() []string {
	return nil
}

func dirs(path, name string, written map[string]bool) []string {
	ds := strings.Split(filepath.ToSlash(name), "/")
	if len(ds) > 1 {
		var ret []string
		for _, p := range ds[:len(ds)-1] {
			path = path + string(filepath.Separator) + p
			if !written[path] {
				ret = append(ret, path)
				written[path] = true
			}
		}
		return ret
	}
	return nil
}

// per https://github.com/richardlehane/siegfried/issues/81
// construct paths for compressed objects acc. to KDE hash notation
func arcpath(base, path string) string {
	if *droido {
		return base + string(filepath.Separator) + path
	}
	return base + "#" + path
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main // import "k8s.io/helm/cmd/tiller"

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	goprom "github.com/grpc-ecosystem/go-grpc-prometheus"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/health"
	healthpb "google.golang.org/grpc/health/grpc_health_v1"
	"google.golang.org/grpc/keepalive"

	"k8s.io/helm/pkg/kube"
	"k8s.io/helm/pkg/proto/hapi/services"
	"k8s.io/helm/pkg/storage"
	"k8s.io/helm/pkg/storage/driver"
	"k8s.io/helm/pkg/tiller"
	"k8s.io/helm/pkg/tiller/environment"
	"k8s.io/helm/pkg/tlsutil"
	"k8s.io/helm/pkg/version"
)

const (
	// tlsEnableEnvVar names the environment variable that enables TLS.
	tlsEnableEnvVar = "TILLER_TLS_ENABLE"
	// tlsVerifyEnvVar names the environment variable that enables
	// TLS, as well as certificate verification of the remote.
	tlsVerifyEnvVar = "TILLER_TLS_VERIFY"
	// tlsCertsEnvVar names the environment variable that points to
	// the directory where Tiller's TLS certificates are located.
	tlsCertsEnvVar = "TILLER_TLS_CERTS"
	// historyMaxEnvVar is the name of the env var for setting max history.
	historyMaxEnvVar = "TILLER_HISTORY_MAX"

	storageMemory    = "memory"
	storageConfigMap = "configmap"
	storageSecret    = "secret"

	probeAddr = ":44135"
	traceAddr = ":44136"

	// defaultMaxHistory sets the maximum number of releases to 0: unlimited
	defaultMaxHistory = 0
)

var (
	grpcAddr             = flag.String("listen", ":44134", "address:port to listen on")
	enableTracing        = flag.Bool("trace", false, "enable rpc tracing")
	store                = flag.String("storage", storageConfigMap, "storage driver to use. One of 'configmap', 'memory', or 'secret'")
	remoteReleaseModules = flag.Bool("experimental-release", false, "enable experimental release modules")
	tlsEnable            = flag.Bool("tls", tlsEnableEnvVarDefault(), "enable TLS")
	tlsVerify            = flag.Bool("tls-verify", tlsVerifyEnvVarDefault(), "enable TLS and verify remote certificate")
	keyFile              = flag.String("tls-key", tlsDefaultsFromEnv("tls-key"), "path to TLS private key file")
	certFile             = flag.String("tls-cert", tlsDefaultsFromEnv("tls-cert"), "path to TLS certificate file")
	caCertFile           = flag.String("tls-ca-cert", tlsDefaultsFromEnv("tls-ca-cert"), "trust certificates signed by this CA")
	maxHistory           = flag.Int("history-max", historyMaxFromEnv(), "maximum number of releases kept in release history, with 0 meaning no limit")
	printVersion         = flag.Bool("version", false, "print the version number")

	// rootServer is the root gRPC server.
	//
	// Each gRPC service registers itself to this server during init().
	rootServer *grpc.Server

	// env is the default environment.
	//
	// Any changes to env should be done before rootServer.Serve() is called.
	env = environment.New()

	logger *log.Logger
)

func main() {
	// TODO: use spf13/cobra for tiller instead of flags
	flag.Parse()

	if *printVersion {
		fmt.Println(version.GetVersion())
		os.Exit(0)
	}

	if *enableTracing {
		log.SetFlags(log.Lshortfile)
	}
	logger = newLogger("main")

	start()
}

func start() {

	healthSrv := health.NewServer()
	healthSrv.SetServingStatus("Tiller", healthpb.HealthCheckResponse_NOT_SERVING)

	clientset, err := kube.New(nil).ClientSet()
	if err != nil {
		logger.Fatalf("Cannot initialize Kubernetes connection: %s", err)
	}

	switch *store {
	case storageMemory:
		env.Releases = storage.Init(driver.NewMemory())
	case storageConfigMap:
		cfgmaps := driver.NewConfigMaps(clientset.Core().ConfigMaps(namespace()))
		cfgmaps.Log = newLogger("storage/driver").Printf

		env.Releases = storage.Init(cfgmaps)
		env.Releases.Log = newLogger("storage").Printf
	case storageSecret:
		secrets := driver.NewSecrets(clientset.Core().Secrets(namespace()))
		secrets.Log = newLogger("storage/driver").Printf

		env.Releases = storage.Init(secrets)
		env.Releases.Log = newLogger("storage").Printf
	}

	if *maxHistory > 0 {
		env.Releases.MaxHistory = *maxHistory
	}

	kubeClient := kube.New(nil)
	kubeClient.Log = newLogger("kube").Printf
	env.KubeClient = kubeClient

	if *tlsEnable || *tlsVerify {
		opts := tlsutil.Options{CertFile: *certFile, KeyFile: *keyFile}
		if *tlsVerify {
			opts.CaCertFile = *caCertFile
		}
	}

	var opts []grpc.ServerOption
	if *tlsEnable || *tlsVerify {
		cfg, err := tlsutil.ServerConfig(tlsOptions())
		if err != nil {
			logger.Fatalf("Could not create server TLS configuration: %v", err)
		}
		opts = append(opts, grpc.Creds(credentials.NewTLS(cfg)))
	}

	opts = append(opts, grpc.KeepaliveParams(keepalive.ServerParameters{
		MaxConnectionIdle: 10 * time.Minute,
		// If needed, we can configure the max connection age
	}))
	opts = append(opts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
		MinTime: time.Duration(20) * time.Second, // For compatibility with the client keepalive.ClientParameters
	}))

	rootServer = tiller.NewServer(opts...)
	healthpb.RegisterHealthServer(rootServer, healthSrv)

	lstn, err := net.Listen("tcp", *grpcAddr)
	if err != nil {
		logger.Fatalf("Server died: %s", err)
	}

	logger.Printf("Starting Tiller %s (tls=%t)", version.GetVersion(), *tlsEnable || *tlsVerify)
	logger.Printf("GRPC listening on %s", *grpcAddr)
	logger.Printf("Probes listening on %s", probeAddr)
	logger.Printf("Storage driver is %s", env.Releases.Name())
	logger.Printf("Max history per release is %d", *maxHistory)

	if *enableTracing {
		startTracing(traceAddr)
	}

	srvErrCh := make(chan error)
	probeErrCh := make(chan error)
	go func() {
		svc := tiller.NewReleaseServer(env, clientset, *remoteReleaseModules)
		svc.Log = newLogger("tiller").Printf
		services.RegisterReleaseServiceServer(rootServer, svc)
		if err := rootServer.Serve(lstn); err != nil {
			srvErrCh <- err
		}
	}()

	go func() {
		mux := newProbesMux()

		// Register gRPC server to prometheus to initialized matrix
		goprom.Register(rootServer)
		addPrometheusHandler(mux)

		if err := http.ListenAndServe(probeAddr, mux); err != nil {
			probeErrCh <- err
		}
	}()

	healthSrv.SetServingStatus("Tiller", healthpb.HealthCheckResponse_SERVING)

	select {
	case err := <-srvErrCh:
		logger.Fatalf("Server died: %s", err)
	case err := <-probeErrCh:
		logger.Printf("Probes server died: %s", err)
	}
}

func newLogger(prefix string) *log.Logger {
	if len(prefix) > 0 {
		prefix = fmt.Sprintf("[%s] ", prefix)
	}
	return log.New(os.Stderr, prefix, log.Flags())
}

// namespace returns the namespace of tiller
func namespace() string {
	if ns := os.Getenv("TILLER_NAMESPACE"); ns != "" {
		return ns
	}

	// Fall back to the namespace associated with the service account token, if available
	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}

	return environment.DefaultTillerNamespace
}

func tlsOptions() tlsutil.Options {
	opts := tlsutil.Options{CertFile: *certFile, KeyFile: *keyFile}
	if *tlsVerify {
		opts.CaCertFile = *caCertFile

		// We want to force the client to not only provide a cert, but to
		// provide a cert that we can validate.
		// http://www.bite-code.com/2015/06/25/tls-mutual-auth-in-golang/
		opts.ClientAuth = tls.RequireAndVerifyClientCert
	}
	return opts
}

func tlsDefaultsFromEnv(name string) (value string) {
	switch certsDir := os.Getenv(tlsCertsEnvVar); name {
	case "tls-key":
		return filepath.Join(certsDir, "tls.key")
	case "tls-cert":
		return filepath.Join(certsDir, "tls.crt")
	case "tls-ca-cert":
		return filepath.Join(certsDir, "ca.crt")
	}
	return ""
}

func historyMaxFromEnv() int {
	val := os.Getenv(historyMaxEnvVar)
	if val == "" {
		return defaultMaxHistory
	}
	ret, err := strconv.Atoi(val)
	if err != nil {
		log.Printf("Invalid max history %q. Defaulting to 0.", val)
		return defaultMaxHistory
	}
	return ret
}

func tlsEnableEnvVarDefault() bool { return os.Getenv(tlsEnableEnvVar) != "" }
func tlsVerifyEnvVarDefault() bool { return os.Getenv(tlsVerifyEnvVar) != "" }
<commit_msg>Fix inaccurate comment in `tiller`<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main // import "k8s.io/helm/cmd/tiller"

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	goprom "github.com/grpc-ecosystem/go-grpc-prometheus"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/health"
	healthpb "google.golang.org/grpc/health/grpc_health_v1"
	"google.golang.org/grpc/keepalive"

	"k8s.io/helm/pkg/kube"
	"k8s.io/helm/pkg/proto/hapi/services"
	"k8s.io/helm/pkg/storage"
	"k8s.io/helm/pkg/storage/driver"
	"k8s.io/helm/pkg/tiller"
	"k8s.io/helm/pkg/tiller/environment"
	"k8s.io/helm/pkg/tlsutil"
	"k8s.io/helm/pkg/version"
)

const (
	// tlsEnableEnvVar names the environment variable that enables TLS.
	tlsEnableEnvVar = "TILLER_TLS_ENABLE"
	// tlsVerifyEnvVar names the environment variable that enables
	// TLS, as well as certificate verification of the remote.
	tlsVerifyEnvVar = "TILLER_TLS_VERIFY"
	// tlsCertsEnvVar names the environment variable that points to
	// the directory where Tiller's TLS certificates are located.
	tlsCertsEnvVar = "TILLER_TLS_CERTS"
	// historyMaxEnvVar is the name of the env var for setting max history.
	historyMaxEnvVar = "TILLER_HISTORY_MAX"

	storageMemory    = "memory"
	storageConfigMap = "configmap"
	storageSecret    = "secret"

	probeAddr = ":44135"
	traceAddr = ":44136"

	// defaultMaxHistory sets the maximum number of releases to 0: unlimited
	defaultMaxHistory = 0
)

var (
	grpcAddr             = flag.String("listen", ":44134", "address:port to listen on")
	enableTracing        = flag.Bool("trace", false, "enable rpc tracing")
	store                = flag.String("storage", storageConfigMap, "storage driver to use. One of 'configmap', 'memory', or 'secret'")
	remoteReleaseModules = flag.Bool("experimental-release", false, "enable experimental release modules")
	tlsEnable            = flag.Bool("tls", tlsEnableEnvVarDefault(), "enable TLS")
	tlsVerify            = flag.Bool("tls-verify", tlsVerifyEnvVarDefault(), "enable TLS and verify remote certificate")
	keyFile              = flag.String("tls-key", tlsDefaultsFromEnv("tls-key"), "path to TLS private key file")
	certFile             = flag.String("tls-cert", tlsDefaultsFromEnv("tls-cert"), "path to TLS certificate file")
	caCertFile           = flag.String("tls-ca-cert", tlsDefaultsFromEnv("tls-ca-cert"), "trust certificates signed by this CA")
	maxHistory           = flag.Int("history-max", historyMaxFromEnv(), "maximum number of releases kept in release history, with 0 meaning no limit")
	printVersion         = flag.Bool("version", false, "print the version number")

	// rootServer is the root gRPC server.
	//
	// Each gRPC service registers itself to this server during start().
	rootServer *grpc.Server

	// env is the default environment.
	//
	// Any changes to env should be done before rootServer.Serve() is called.
	env = environment.New()

	logger *log.Logger
)

func main() {
	// TODO: use spf13/cobra for tiller instead of flags
	flag.Parse()

	if *printVersion {
		fmt.Println(version.GetVersion())
		os.Exit(0)
	}

	if *enableTracing {
		log.SetFlags(log.Lshortfile)
	}
	logger = newLogger("main")

	start()
}

func start() {

	healthSrv := health.NewServer()
	healthSrv.SetServingStatus("Tiller", healthpb.HealthCheckResponse_NOT_SERVING)

	clientset, err := kube.New(nil).ClientSet()
	if err != nil {
		logger.Fatalf("Cannot initialize Kubernetes connection: %s", err)
	}

	switch *store {
	case storageMemory:
		env.Releases = storage.Init(driver.NewMemory())
	case storageConfigMap:
		cfgmaps := driver.NewConfigMaps(clientset.Core().ConfigMaps(namespace()))
		cfgmaps.Log = newLogger("storage/driver").Printf

		env.Releases = storage.Init(cfgmaps)
		env.Releases.Log = newLogger("storage").Printf
	case storageSecret:
		secrets := driver.NewSecrets(clientset.Core().Secrets(namespace()))
		secrets.Log = newLogger("storage/driver").Printf

		env.Releases = storage.Init(secrets)
		env.Releases.Log = newLogger("storage").Printf
	}

	if *maxHistory > 0 {
		env.Releases.MaxHistory = *maxHistory
	}

	kubeClient := kube.New(nil)
	kubeClient.Log = newLogger("kube").Printf
	env.KubeClient = kubeClient

	if *tlsEnable || *tlsVerify {
		opts := tlsutil.Options{CertFile: *certFile, KeyFile: *keyFile}
		if *tlsVerify {
			opts.CaCertFile = *caCertFile
		}
	}

	var opts []grpc.ServerOption
	if *tlsEnable || *tlsVerify {
		cfg, err := tlsutil.ServerConfig(tlsOptions())
		if err != nil {
			logger.Fatalf("Could not create server TLS configuration: %v", err)
		}
		opts = append(opts, grpc.Creds(credentials.NewTLS(cfg)))
	}

	opts = append(opts, grpc.KeepaliveParams(keepalive.ServerParameters{
		MaxConnectionIdle: 10 * time.Minute,
		// If needed, we can configure the max connection age
	}))
	opts = append(opts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
		MinTime: time.Duration(20) * time.Second, // For compatibility with the client keepalive.ClientParameters
	}))

	rootServer = tiller.NewServer(opts...)
	healthpb.RegisterHealthServer(rootServer, healthSrv)

	lstn, err := net.Listen("tcp", *grpcAddr)
	if err != nil {
		logger.Fatalf("Server died: %s", err)
	}

	logger.Printf("Starting Tiller %s (tls=%t)", version.GetVersion(), *tlsEnable || *tlsVerify)
	logger.Printf("GRPC listening on %s", *grpcAddr)
	logger.Printf("Probes listening on %s", probeAddr)
	logger.Printf("Storage driver is %s", env.Releases.Name())
	logger.Printf("Max history per release is %d", *maxHistory)

	if *enableTracing {
		startTracing(traceAddr)
	}

	srvErrCh := make(chan error)
	probeErrCh := make(chan error)
	go func() {
		svc := tiller.NewReleaseServer(env, clientset, *remoteReleaseModules)
		svc.Log = newLogger("tiller").Printf
		services.RegisterReleaseServiceServer(rootServer, svc)
		if err := rootServer.Serve(lstn); err != nil {
			srvErrCh <- err
		}
	}()

	go func() {
		mux := newProbesMux()

		// Register gRPC server to prometheus to initialized matrix
		goprom.Register(rootServer)
		addPrometheusHandler(mux)

		if err := http.ListenAndServe(probeAddr, mux); err != nil {
			probeErrCh <- err
		}
	}()

	healthSrv.SetServingStatus("Tiller", healthpb.HealthCheckResponse_SERVING)

	select {
	case err := <-srvErrCh:
		logger.Fatalf("Server died: %s", err)
	case err := <-probeErrCh:
		logger.Printf("Probes server died: %s", err)
	}
}

func newLogger(prefix string) *log.Logger {
	if len(prefix) > 0 {
		prefix = fmt.Sprintf("[%s] ", prefix)
	}
	return log.New(os.Stderr, prefix, log.Flags())
}

// namespace returns the namespace of tiller
func namespace() string {
	if ns := os.Getenv("TILLER_NAMESPACE"); ns != "" {
		return ns
	}

	// Fall back to the namespace associated with the service account token, if available
	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}

	return environment.DefaultTillerNamespace
}

func tlsOptions() tlsutil.Options {
	opts := tlsutil.Options{CertFile: *certFile, KeyFile: *keyFile}
	if *tlsVerify {
		opts.CaCertFile = *caCertFile

		// We want to force the client to not only provide a cert, but to
		// provide a cert that we can validate.
		// http://www.bite-code.com/2015/06/25/tls-mutual-auth-in-golang/
		opts.ClientAuth = tls.RequireAndVerifyClientCert
	}
	return opts
}

func tlsDefaultsFromEnv(name string) (value string) {
	switch certsDir := os.Getenv(tlsCertsEnvVar); name {
	case "tls-key":
		return filepath.Join(certsDir, "tls.key")
	case "tls-cert":
		return filepath.Join(certsDir, "tls.crt")
	case "tls-ca-cert":
		return filepath.Join(certsDir, "ca.crt")
	}
	return ""
}

func historyMaxFromEnv() int {
	val := os.Getenv(historyMaxEnvVar)
	if val == "" {
		return defaultMaxHistory
	}
	ret, err := strconv.Atoi(val)
	if err != nil {
		log.Printf("Invalid max history %q. Defaulting to 0.", val)
		return defaultMaxHistory
	}
	return ret
}

func tlsEnableEnvVarDefault() bool { return os.Getenv(tlsEnableEnvVar) != "" }
func tlsVerifyEnvVarDefault() bool { return os.Getenv(tlsVerifyEnvVar) != "" }
<|endoftext|>


<commit_before>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// yoda2rio converts YODA files containing hbook-like values (H1D, H2D, P1D, ...)
// into rio files.
//
// Example:
//
//  $> yoda2rio rivet.yoda >| rivet.rio
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"
	"strings"

	"go-hep.org/x/hep/hbook"
	"go-hep.org/x/hep/rio"
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("yoda2rio: ")
	log.SetOutput(os.Stderr)

	flag.Usage = func() {
		fmt.Fprintf(
			os.Stderr,
			`Usage: yoda2rio [options] <file1.yoda> [<file2.yoda> [...]]

ex:
 $ yoda2rio rivet.yoda >| rivet.rio
`)
	}

	flag.Parse()

	if flag.NArg() < 1 {
		log.Printf("missing input file name")
		flag.Usage()
		flag.PrintDefaults()
	}

	o, err := rio.NewWriter(os.Stdout)
	if err != nil {
		log.Fatal(err)
	}
	defer o.Close()

	for _, fname := range flag.Args() {
		convert(o, fname)
	}
}

func convert(w *rio.Writer, fname string) {
	r, err := os.Open(fname)
	if err != nil {
		log.Fatalf("error opening file [%s]: %v\n", fname, err)
	}
	defer r.Close()

	vs, err := yodaSlice(r)
	if err != nil {
		log.Fatalf("error decoding YODA file [%s]: %v\n", fname, err)
	}

	for _, v := range vs {
		err = w.WriteValue(v.Name(), v.Value())
		if err != nil {
			log.Fatalf("error writing %q from YODA file [%s]: %v\n", v.Name(), fname, err)
		}
	}
}

var (
	yodaHeader = []byte("BEGIN YODA_")
	yodaFooter = []byte("END YODA_")
)

func yodaSlice(r io.Reader) ([]Yoda, error) {
	var (
		err   error
		o     []Yoda
		block = make([]byte, 0, 1024)
		rt    reflect.Type
		name  string
	)
	scan := bufio.NewScanner(r)
	for scan.Scan() {
		raw := scan.Bytes()
		switch {
		case bytes.HasPrefix(raw, yodaHeader):
			rt, name, err = splitHeader(raw)
			if err != nil {
				log.Fatalf("error parsing YODA header: %v", err)
			}
			block = block[:0]
			block = append(block, raw...)
			block = append(block, '\n')

		default:
			block = append(block, raw...)
			block = append(block, '\n')

		case bytes.HasPrefix(raw, yodaFooter):
			block = append(block, raw...)
			block = append(block, '\n')

			v := reflect.New(rt).Elem()
			err = v.Addr().Interface().(unmarshalYoda).UnmarshalYODA(block)
			if err != nil {
				log.Fatalf("error unmarshaling YODA %q (type=%v): %v\n%v\n===\n", name, rt.Name(), err, string(block))
			}
			o = append(o, &yoda{name: name, ptr: v.Addr().Interface()})
		}

	}
	err = scan.Err()
	if err == io.EOF {
		err = nil
	}
	if err != nil {
		return nil, err
	}

	return o, nil
}

func splitHeader(raw []byte) (reflect.Type, string, error) {
	raw = raw[len(yodaHeader):]
	i := bytes.Index(raw, []byte(" "))
	if i == -1 || i >= len(raw) {
		return nil, "", fmt.Errorf("invalid YODA header (missing space)")
	}

	var rt reflect.Type

	switch string(raw[:i]) {
	case "HISTO1D":
		rt = reflect.TypeOf((*hbook.H1D)(nil)).Elem()
	case "HISTO2D":
		rt = reflect.TypeOf((*hbook.H2D)(nil)).Elem()
	case "PROFILE1D":
		rt = reflect.TypeOf((*hbook.P1D)(nil)).Elem()
	case "SCATTER2D":
		rt = reflect.TypeOf((*hbook.S2D)(nil)).Elem()
	default:
		log.Fatalf("unhandled YODA object type %q", string(raw[:i]))
	}

	name := raw[i+2:] // +2 to also remove the leading '/'
	return rt, strings.TrimSpace(string(name)), nil
}

type Yoda interface {
	Name() string
	Value() interface{}
}

type unmarshalYoda interface {
	UnmarshalYODA([]byte) error
}

type yoda struct {
	name string
	ptr  interface{}
}

func (y *yoda) Name() string {
	return y.name
}

func (y *yoda) Value() interface{} {
	return y.ptr
}
<commit_msg>cmd/yoda2rio: cosmetics<commit_after>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// yoda2rio converts YODA files containing hbook-like values (H1D, H2D, P1D, ...)
// into rio files.
//
// Example:
//
//  $> yoda2rio rivet.yoda >| rivet.rio
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"
	"strings"

	"go-hep.org/x/hep/hbook"
	"go-hep.org/x/hep/rio"
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("yoda2rio: ")
	log.SetOutput(os.Stderr)

	flag.Usage = func() {
		fmt.Fprintf(
			os.Stderr,
			`Usage: yoda2rio [options] <file1.yoda> [<file2.yoda> [...]]

ex:
 $ yoda2rio rivet.yoda >| rivet.rio
`)
	}

	flag.Parse()

	if flag.NArg() < 1 {
		log.Printf("missing input file name")
		flag.Usage()
		flag.PrintDefaults()
	}

	o, err := rio.NewWriter(os.Stdout)
	if err != nil {
		log.Fatal(err)
	}
	defer o.Close()

	for _, fname := range flag.Args() {
		convert(o, fname)
	}
}

func convert(w *rio.Writer, fname string) {
	r, err := os.Open(fname)
	if err != nil {
		log.Fatalf("error opening file [%s]: %v\n", fname, err)
	}
	defer r.Close()

	vs, err := yodaSlice(r)
	if err != nil {
		log.Fatalf("error decoding YODA file [%s]: %v\n", fname, err)
	}

	for _, v := range vs {
		err = w.WriteValue(v.Name(), v.Value())
		if err != nil {
			log.Fatalf("error writing %q from YODA file [%s]: %v\n", v.Name(), fname, err)
		}
	}
}

var (
	yodaHeader = []byte("BEGIN YODA_")
	yodaFooter = []byte("END YODA_")
)

func yodaSlice(r io.Reader) ([]Yoda, error) {
	var (
		err   error
		o     []Yoda
		block = make([]byte, 0, 1024)
		rt    reflect.Type
		name  string
	)
	scan := bufio.NewScanner(r)
	for scan.Scan() {
		raw := scan.Bytes()
		switch {
		case bytes.HasPrefix(raw, yodaHeader):
			rt, name, err = splitHeader(raw)
			if err != nil {
				log.Fatalf("error parsing YODA header: %v", err)
			}
			block = block[:0]
			block = append(block, raw...)
			block = append(block, '\n')

		default:
			block = append(block, raw...)
			block = append(block, '\n')

		case bytes.HasPrefix(raw, yodaFooter):
			block = append(block, raw...)
			block = append(block, '\n')

			v := reflect.New(rt).Elem()
			err = v.Addr().Interface().(unmarshalYoda).UnmarshalYODA(block)
			if err != nil {
				log.Fatalf("error unmarshaling YODA %q (type=%v): %v\n%v\n===\n", name, rt.Name(), err, string(block))
			}
			o = append(o, &yoda{name: name, ptr: v.Addr().Interface()})
		}
	}
	err = scan.Err()
	if err != nil {
		return nil, err
	}
	return o, nil
}

func splitHeader(raw []byte) (reflect.Type, string, error) {
	raw = raw[len(yodaHeader):]
	i := bytes.Index(raw, []byte(" "))
	if i == -1 || i >= len(raw) {
		return nil, "", fmt.Errorf("invalid YODA header (missing space)")
	}

	var rt reflect.Type

	switch string(raw[:i]) {
	case "HISTO1D":
		rt = reflect.TypeOf((*hbook.H1D)(nil)).Elem()
	case "HISTO2D":
		rt = reflect.TypeOf((*hbook.H2D)(nil)).Elem()
	case "PROFILE1D":
		rt = reflect.TypeOf((*hbook.P1D)(nil)).Elem()
	case "SCATTER2D":
		rt = reflect.TypeOf((*hbook.S2D)(nil)).Elem()
	default:
		log.Fatalf("unhandled YODA object type %q", string(raw[:i]))
	}

	name := raw[i+2:] // +2 to also remove the leading '/'
	return rt, strings.TrimSpace(string(name)), nil
}

type Yoda interface {
	Name() string
	Value() interface{}
}

type unmarshalYoda interface {
	UnmarshalYODA([]byte) error
}

type yoda struct {
	name string
	ptr  interface{}
}

func (y *yoda) Name() string {
	return y.name
}

func (y *yoda) Value() interface{} {
	return y.ptr
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"
	"encoding/gob"
	"fmt"
	"bufio"
	"io"
	"log"
	"net"
	"time"
	"os"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/titanous/go-dockerclient"
	"github.com/flynn/sampi/types"
	sc "github.com/flynn/sampi/client"
	lc "github.com/flynn/lorne/client"
)

// WARNING: assumes one host at the moment (firstHost will always be the same)

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	scheduleAndAttach(app + ".build", docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := app + ".web"

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://%s:%s\n", hostname, getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
        out, err := exec.Command("bash", "-c", cmdline).Output()
        if err != nil {
                panic(err)
        }
        return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {

	services, err := sd.Services("flynn-lorne-attach." + hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.(*net.TCPConn).CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<commit_msg>need a return<commit_after>package main

import (
	"os/exec"
	"strings"
	"encoding/gob"
	"fmt"
	"bufio"
	"io"
	"log"
	"net"
	"time"
	"os"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/titanous/go-dockerclient"
	"github.com/flynn/sampi/types"
	sc "github.com/flynn/sampi/client"
	lc "github.com/flynn/lorne/client"
)

// WARNING: assumes one host at the moment (firstHost will always be the same)

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	scheduleAndAttach(app + ".build", docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := app + ".web"

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://%s:%s\n", hostname, getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
        out, err := exec.Command("bash", "-c", cmdline).Output()
        if err != nil {
                panic(err)
        }
        return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
	return ""
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {

	services, err := sd.Services("flynn-lorne-attach." + hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.(*net.TCPConn).CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<|endoftext|>


<commit_before>package google

func canonicalizeServiceScope(scope string) string {
	// This is a convenience map of short names used by the gcloud tool
	// to the GCE auth endpoints they alias to.
	scopeMap := map[string]string{
		"bigquery":        "https://www.googleapis.com/auth/bigquery",
		"cloud-platform":  "https://www.googleapis.com/auth/cloud-platform",
		"compute-ro":      "https://www.googleapis.com/auth/compute.readonly",
		"compute-rw":      "https://www.googleapis.com/auth/compute",
		"datastore":       "https://www.googleapis.com/auth/datastore",
		"logging-write":   "https://www.googleapis.com/auth/logging.write",
		"monitoring":      "https://www.googleapis.com/auth/monitoring",
		"sql":             "https://www.googleapis.com/auth/sqlservice",
		"sql-admin":       "https://www.googleapis.com/auth/sqlservice.admin",
		"storage-full":    "https://www.googleapis.com/auth/devstorage.full_control",
		"storage-ro":      "https://www.googleapis.com/auth/devstorage.read_only",
		"storage-rw":      "https://www.googleapis.com/auth/devstorage.read_write",
		"taskqueue":       "https://www.googleapis.com/auth/taskqueue",
		"useraccounts-ro": "https://www.googleapis.com/auth/cloud.useraccounts.readonly",
		"useraccounts-rw": "https://www.googleapis.com/auth/cloud.useraccounts",
		"userinfo-email":  "https://www.googleapis.com/auth/userinfo.email",
	}

	if matchedUrl, ok := scopeMap[scope]; ok {
		return matchedUrl
	} else {
		return scope
	}
}
<commit_msg>providers/google: add pubsub auth endpoint #3803<commit_after>package google

func canonicalizeServiceScope(scope string) string {
	// This is a convenience map of short names used by the gcloud tool
	// to the GCE auth endpoints they alias to.
	scopeMap := map[string]string{
		"bigquery":        "https://www.googleapis.com/auth/bigquery",
		"cloud-platform":  "https://www.googleapis.com/auth/cloud-platform",
		"compute-ro":      "https://www.googleapis.com/auth/compute.readonly",
		"compute-rw":      "https://www.googleapis.com/auth/compute",
		"datastore":       "https://www.googleapis.com/auth/datastore",
		"logging-write":   "https://www.googleapis.com/auth/logging.write",
		"monitoring":      "https://www.googleapis.com/auth/monitoring",
		"pubsub":          "https://www.googleapis.com/auth/pubsub",
		"sql":             "https://www.googleapis.com/auth/sqlservice",
		"sql-admin":       "https://www.googleapis.com/auth/sqlservice.admin",
		"storage-full":    "https://www.googleapis.com/auth/devstorage.full_control",
		"storage-ro":      "https://www.googleapis.com/auth/devstorage.read_only",
		"storage-rw":      "https://www.googleapis.com/auth/devstorage.read_write",
		"taskqueue":       "https://www.googleapis.com/auth/taskqueue",
		"useraccounts-ro": "https://www.googleapis.com/auth/cloud.useraccounts.readonly",
		"useraccounts-rw": "https://www.googleapis.com/auth/cloud.useraccounts",
		"userinfo-email":  "https://www.googleapis.com/auth/userinfo.email",
	}

	if matchedURL, ok := scopeMap[scope]; ok {
		return matchedURL
	}

	return scope
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/kataras/iris"
)

func main() {

	/*  These are the optionally fields to configurate the sessions, using the station's Config field (iris.Config.Sessions)

	// Cookie string, the session's client cookie name, for example: "irissessionid"
	Cookie string
	// DecodeCookie set it to true to decode the cookie key with base64 URLEncoding
	// Defaults to false
	DecodeCookie bool
	//Expires the date which the cookie must expires. Default infinitive/unlimited life
	Expires time.Time
	// GcDuration every how much duration(GcDuration) the memory should be clear for unused cookies (GcDuration)
	// for example: time.Duration(2)*time.Hour. it will check every 2 hours if cookie hasn't be used for 2 hours,
	// deletes it from memory until the user comes back, then the session continue to work as it was
	//
	// Default 2 hours
	GcDuration time.Duration

	// DisableSubdomainPersistence set it to true in order dissallow your iris subdomains to have access to the session cookie
	// defaults to false
	DisableSubdomainPersistence bool
	*/

	iris.Get("/set", func(c *iris.Context) {

		//set session values
		c.Session().Set("name", "iris")

		//test if setted here
		c.Write("All ok session setted to: %s", c.Session().GetString("name"))
	})

	iris.Get("/get", func(c *iris.Context) {
		// get a specific key, as string, if no found returns just an empty string
		name := c.Session().GetString("name")

		c.Write("The name on the /set was: %s", name)
	})

	iris.Get("/delete", func(c *iris.Context) {
		// delete a specific key
		c.Session().Delete("name")
	})

	iris.Get("/clear", func(c *iris.Context) {
		// removes all entries
		c.Session().Clear()
	})

	iris.Get("/destroy", func(c *iris.Context) {
		//destroy, removes the entire session and cookie
		c.SessionDestroy()
		c.Log("You have to refresh the page to completely remove the session (on browsers), so the name should NOT be empty NOW, is it?\n ame: %s\n\nAlso check your cookies in your browser's cookies, should be no field for localhost/127.0.0.1 (or what ever you use)", c.Session().GetString("name"))
		c.Write("You have to refresh the page to completely remove the session (on browsers), so the name should NOT be empty NOW, is it?\nName: %s\n\nAlso check your cookies in your browser's cookies, should be no field for localhost/127.0.0.1 (or what ever you use)", c.Session().GetString("name"))
	})

	iris.Listen(":8080")
	//iris.ListenTLS("0.0.0.0:443", "mycert.cert", "mykey.key")
}
<commit_msg>Remove time from sessions example...<commit_after>package main

import (
	"github.com/kataras/iris"
)

func main() {

	/*  These are the optionally fields to configurate the sessions, using the station's Config field (iris.Config.Sessions)

	// Cookie string, the session's client cookie name, for example: "irissessionid"
	Cookie string
	// DecodeCookie set it to true to decode the cookie key with base64 URLEncoding
	// Defaults to false
	DecodeCookie bool
	//Expires the date which the cookie must expires. Default infinitive/unlimited life
	Expires time.Time
	// GcDuration every how much duration(GcDuration) the memory should be clear for unused cookies (GcDuration)
	// for example: time.Duration(2)*time.Hour. it will check every 2 hours if cookie hasn't be used for 2 hours,
	// deletes it from memory until the user comes back, then the session continue to work as it was
	//
	// Default 2 hours
	GcDuration time.Duration

	// DisableSubdomainPersistence set it to true in order dissallow your iris subdomains to have access to the session cookie
	// defaults to false
	DisableSubdomainPersistence bool
	*/

	iris.Get("/set", func(c *iris.Context) {

		//set session values
		c.Session().Set("name", "iris")

		//test if setted here
		c.Write("All ok session setted to: %s", c.Session().GetString("name"))
	})

	iris.Get("/get", func(c *iris.Context) {
		// get a specific key, as string, if no found returns just an empty string
		name := c.Session().GetString("name")

		c.Write("The name on the /set was: %s", name)
	})

	iris.Get("/delete", func(c *iris.Context) {
		// delete a specific key
		c.Session().Delete("name")
	})

	iris.Get("/clear", func(c *iris.Context) {
		// removes all entries
		c.Session().Clear()
	})

	iris.Get("/destroy", func(c *iris.Context) {
		//destroy, removes the entire session and cookie
		c.SessionDestroy()
		c.Log("You have to refresh the page to completely remove the session (on browsers), so the name should NOT be empty NOW, is it?\n ame: %s\n\nAlso check your cookies in your browser's cookies, should be no field for localhost/127.0.0.1 (or what ever you use)", c.Session().GetString("name"))
		c.Write("You have to refresh the page to completely remove the session (on browsers), so the name should NOT be empty NOW, is it?\nName: %s\n\nAlso check your cookies in your browser's cookies, should be no field for localhost/127.0.0.1 (or what ever you use)", c.Session().GetString("name"))
	})

	iris.Listen(":8080")
	//iris.ListenTLS("0.0.0.0:443", "mycert.cert", "mykey.key")
}
<|endoftext|>


<commit_before>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"errors"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"net"
	"net/http"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 50

// how many hours an entry should be considered valid
const cacheValidHours = 12

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// abort HTTP requests if it takes longer than X seconds. Not sure, it’s
// definitely magic involved. Must be larger than 5.
const httpGetDeadline = 10

// don’t repost the same title within this period
const noRepostWithinSeconds = 10

// new line replace regex
var newlineReplacer = regexp.MustCompile(`\s+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var noSpoilerRegex = regexp.MustCompile(`(don't|no|kein|nicht) spoiler`)

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if title := cacheGetTitleByUrl(url); title != "" {
			log.Printf("using cache for URL: %s", url)
			postTitle(conn, line, title, "Cache Info")
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if title != "" {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	find := exec.Command("./urifind", "-u", "-S", "http", "-S", "https")
	pipe, err := find.StdinPipe()
	if err != nil {
		log.Printf("WTF: couldn’t open stdin pipe to urifind: %s", err)
		return nil
	}
	pipe.Write([]byte(msg))
	pipe.Close()
	out, err := find.Output()
	if err != nil {
		log.Printf("WTF: urlfind failed with: %s", err)
		return nil
	}
	return strings.Split(strings.TrimSpace(string(out)), "\n")
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	// via http://www.reddit.com/r/golang/comments/10awvj/timeout_on_httpget/c6bz49s
	c := http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(time.Second * httpGetDeadline)
				c, err := net.DialTimeout(netw, addr, time.Second*(httpGetDeadline-5))
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))
	title = newlineReplacer.ReplaceAllString(title, " ")
	log.Printf("Title for URL %s: %s\n", url, title)

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	return title, lastUrl, nil
}

func titleParseHtml(r io.Reader) string {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\r\n", err)
		return ""
	}

	title := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = ""
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				if c.Type != html.TextNode {
					continue
				}
				title += c.Data
			}

		} else { // recurse down
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				f(c)
			}
		}
	}
	f(doc)

	return strings.TrimSpace(title)
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetTitleByUrl(url string) string {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return cc.title
		}
	}
	return ""
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = newlineReplacer.ReplaceAllString(prefix, " ")
	}
	title = newlineReplacer.ReplaceAllString(title, " ")
	// use notice instead of PrivMsg to avoid bots answering each other
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}
<commit_msg>make no spoiler regex case insensitive<commit_after>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"errors"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"net"
	"net/http"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 50

// how many hours an entry should be considered valid
const cacheValidHours = 12

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// abort HTTP requests if it takes longer than X seconds. Not sure, it’s
// definitely magic involved. Must be larger than 5.
const httpGetDeadline = 10

// don’t repost the same title within this period
const noRepostWithinSeconds = 10

// new line replace regex
var newlineReplacer = regexp.MustCompile(`\s+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if title := cacheGetTitleByUrl(url); title != "" {
			log.Printf("using cache for URL: %s", url)
			postTitle(conn, line, title, "Cache Info")
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if title != "" {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	find := exec.Command("./urifind", "-u", "-S", "http", "-S", "https")
	pipe, err := find.StdinPipe()
	if err != nil {
		log.Printf("WTF: couldn’t open stdin pipe to urifind: %s", err)
		return nil
	}
	pipe.Write([]byte(msg))
	pipe.Close()
	out, err := find.Output()
	if err != nil {
		log.Printf("WTF: urlfind failed with: %s", err)
		return nil
	}
	return strings.Split(strings.TrimSpace(string(out)), "\n")
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	// via http://www.reddit.com/r/golang/comments/10awvj/timeout_on_httpget/c6bz49s
	c := http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(time.Second * httpGetDeadline)
				c, err := net.DialTimeout(netw, addr, time.Second*(httpGetDeadline-5))
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))
	title = newlineReplacer.ReplaceAllString(title, " ")
	log.Printf("Title for URL %s: %s\n", url, title)

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	return title, lastUrl, nil
}

func titleParseHtml(r io.Reader) string {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\r\n", err)
		return ""
	}

	title := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = ""
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				if c.Type != html.TextNode {
					continue
				}
				title += c.Data
			}

		} else { // recurse down
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				f(c)
			}
		}
	}
	f(doc)

	return strings.TrimSpace(title)
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetTitleByUrl(url string) string {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return cc.title
		}
	}
	return ""
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = newlineReplacer.ReplaceAllString(prefix, " ")
	}
	title = newlineReplacer.ReplaceAllString(title, " ")
	// use notice instead of PrivMsg to avoid bots answering each other
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"strconv"
	"sync"
	"testing"

	driver "github.com/arangodb/go-driver"
)

// TestConcurrentCreateSmallDocuments make a lot of concurrent CreateDocument calls.
// It then verifies that all documents "have arrived".
func TestConcurrentCreateSmallDocuments(t *testing.T) {
	if testing.Short() {
		t.Skip("Skip on short tests")
	}
	c := createClientFromEnv(t, true)
	db := ensureDatabase(nil, c, "document_test", nil, t)
	col := ensureCollection(nil, db, "TestConcurrentCreateSmallDocuments", nil, t)

	docChan := make(chan driver.DocumentMeta, 16*1024)

	creator := func(limit, interval int) {
		for i := 0; i < limit; i++ {
			ctx := context.Background()
			doc := UserDoc{
				"Jan",
				i * interval,
			}
			meta, err := col.CreateDocument(ctx, doc)
			if err != nil {
				t.Fatalf("Failed to create new document: %s", describe(err))
			}
			docChan <- meta
		}
	}

	reader := func() {
		for {
			meta, ok := <-docChan
			if !ok {
				return
			}
			// Document must exists now
			if found, err := col.DocumentExists(nil, meta.Key); err != nil {
				t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err))
			} else if !found {
				t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key)
			}
			// Read document
			var readDoc UserDoc
			if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil {
				t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err))
			}
		}
	}

	noCreators := getIntFromEnv("NOCREATORS", 25)
	noReaders := getIntFromEnv("NOREADERS", 50)
	noDocuments := getIntFromEnv("NODOCUMENTS", 10000) // per creator

	wgCreators := sync.WaitGroup{}
	// Run N concurrent creators
	for i := 0; i < noCreators; i++ {
		wgCreators.Add(1)
		go func() {
			defer wgCreators.Done()
			creator(noDocuments, noCreators)
		}()
	}
	wgReaders := sync.WaitGroup{}
	// Run M readers
	for i := 0; i < noReaders; i++ {
		wgReaders.Add(1)
		go func() {
			defer wgReaders.Done()
			reader()
		}()
	}
	wgCreators.Wait()
	close(docChan)
	wgReaders.Wait()
}

// TestConcurrentCreateBigDocuments make a lot of concurrent CreateDocument calls.
// It then verifies that all documents "have arrived".
func TestConcurrentCreateBigDocuments(t *testing.T) {
	if testing.Short() {
		t.Skip("Skip on short tests")
	}
	c := createClientFromEnv(t, true)
	db := ensureDatabase(nil, c, "document_test", nil, t)
	col := ensureCollection(nil, db, "TestConcurrentCreateBigDocuments", nil, t)

	docChan := make(chan driver.DocumentMeta, 16*1024)

	creator := func(limit, interval int) {
		data := make([]byte, 1024)
		for i := 0; i < limit; i++ {
			rand.Read(data)
			ctx := context.Background()
			doc := UserDoc{
				"Jan" + strconv.Itoa(i) + hex.EncodeToString(data),
				i * interval,
			}
			meta, err := col.CreateDocument(ctx, doc)
			if err != nil {
				t.Fatalf("Failed to create new document: %s", describe(err))
			}
			docChan <- meta
		}
	}

	reader := func() {
		for {
			meta, ok := <-docChan
			if !ok {
				return
			}
			// Document must exists now
			if found, err := col.DocumentExists(nil, meta.Key); err != nil {
				t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err))
			} else if !found {
				t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key)
			}
			// Read document
			var readDoc UserDoc
			if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil {
				t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err))
			}
		}
	}

	noCreators := getIntFromEnv("NOCREATORS", 25)
	noReaders := getIntFromEnv("NOREADERS", 50)
	noDocuments := getIntFromEnv("NODOCUMENTS", 1000) // per creator

	wgCreators := sync.WaitGroup{}
	// Run N concurrent creators
	for i := 0; i < noCreators; i++ {
		wgCreators.Add(1)
		go func() {
			defer wgCreators.Done()
			creator(noDocuments, noCreators)
		}()
	}
	wgReaders := sync.WaitGroup{}
	// Run M readers
	for i := 0; i < noReaders; i++ {
		wgReaders.Add(1)
		go func() {
			defer wgReaders.Done()
			reader()
		}()
	}
	wgCreators.Wait()
	close(docChan)
	wgReaders.Wait()
}
<commit_msg>Limit test default range<commit_after>//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"strconv"
	"sync"
	"testing"

	driver "github.com/arangodb/go-driver"
)

// TestConcurrentCreateSmallDocuments make a lot of concurrent CreateDocument calls.
// It then verifies that all documents "have arrived".
func TestConcurrentCreateSmallDocuments(t *testing.T) {
	if testing.Short() {
		t.Skip("Skip on short tests")
	}
	c := createClientFromEnv(t, true)
	db := ensureDatabase(nil, c, "document_test", nil, t)
	col := ensureCollection(nil, db, "TestConcurrentCreateSmallDocuments", nil, t)

	docChan := make(chan driver.DocumentMeta, 16*1024)

	creator := func(limit, interval int) {
		for i := 0; i < limit; i++ {
			ctx := context.Background()
			doc := UserDoc{
				"Jan",
				i * interval,
			}
			meta, err := col.CreateDocument(ctx, doc)
			if err != nil {
				t.Fatalf("Failed to create new document: %s", describe(err))
			}
			docChan <- meta
		}
	}

	reader := func() {
		for {
			meta, ok := <-docChan
			if !ok {
				return
			}
			// Document must exists now
			if found, err := col.DocumentExists(nil, meta.Key); err != nil {
				t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err))
			} else if !found {
				t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key)
			}
			// Read document
			var readDoc UserDoc
			if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil {
				t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err))
			}
		}
	}

	noCreators := getIntFromEnv("NOCREATORS", 25)
	noReaders := getIntFromEnv("NOREADERS", 50)
	noDocuments := getIntFromEnv("NODOCUMENTS", 1000) // per creator

	wgCreators := sync.WaitGroup{}
	// Run N concurrent creators
	for i := 0; i < noCreators; i++ {
		wgCreators.Add(1)
		go func() {
			defer wgCreators.Done()
			creator(noDocuments, noCreators)
		}()
	}
	wgReaders := sync.WaitGroup{}
	// Run M readers
	for i := 0; i < noReaders; i++ {
		wgReaders.Add(1)
		go func() {
			defer wgReaders.Done()
			reader()
		}()
	}
	wgCreators.Wait()
	close(docChan)
	wgReaders.Wait()
}

// TestConcurrentCreateBigDocuments make a lot of concurrent CreateDocument calls.
// It then verifies that all documents "have arrived".
func TestConcurrentCreateBigDocuments(t *testing.T) {
	if testing.Short() {
		t.Skip("Skip on short tests")
	}
	c := createClientFromEnv(t, true)
	db := ensureDatabase(nil, c, "document_test", nil, t)
	col := ensureCollection(nil, db, "TestConcurrentCreateBigDocuments", nil, t)

	docChan := make(chan driver.DocumentMeta, 16*1024)

	creator := func(limit, interval int) {
		data := make([]byte, 1024)
		for i := 0; i < limit; i++ {
			rand.Read(data)
			ctx := context.Background()
			doc := UserDoc{
				"Jan" + strconv.Itoa(i) + hex.EncodeToString(data),
				i * interval,
			}
			meta, err := col.CreateDocument(ctx, doc)
			if err != nil {
				t.Fatalf("Failed to create new document: %s", describe(err))
			}
			docChan <- meta
		}
	}

	reader := func() {
		for {
			meta, ok := <-docChan
			if !ok {
				return
			}
			// Document must exists now
			if found, err := col.DocumentExists(nil, meta.Key); err != nil {
				t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err))
			} else if !found {
				t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key)
			}
			// Read document
			var readDoc UserDoc
			if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil {
				t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err))
			}
		}
	}

	noCreators := getIntFromEnv("NOCREATORS", 25)
	noReaders := getIntFromEnv("NOREADERS", 50)
	noDocuments := getIntFromEnv("NODOCUMENTS", 100) // per creator

	wgCreators := sync.WaitGroup{}
	// Run N concurrent creators
	for i := 0; i < noCreators; i++ {
		wgCreators.Add(1)
		go func() {
			defer wgCreators.Done()
			creator(noDocuments, noCreators)
		}()
	}
	wgReaders := sync.WaitGroup{}
	// Run M readers
	for i := 0; i < noReaders; i++ {
		wgReaders.Add(1)
		go func() {
			defer wgReaders.Done()
			reader()
		}()
	}
	wgCreators.Wait()
	close(docChan)
	wgReaders.Wait()
}
<|endoftext|>