<commit_before>package app

import (
	"fmt"
	"os"

	"github.com/inconshreveable/log15"
)

// Logger holds the application logger
type Logger struct {
	log15.Logger
}

var (
	// Log Is the global log variable.
	Log = newLog()

	// stdHandler is the log handler with level applied
	stdHandler = log15.StreamHandler(os.Stdout, log15.LogfmtFormat())

	// Default handler used in the package.
	defaultHandler log15.Handler
)

func newLog() *Logger {
	return &Logger{
		Logger: log15.New(),
	}
}

func init() {
	SetDefaultLogHandler(log15.LvlFilterHandler(log15.LvlInfo, stdHandler))
	Log.SetHandler(defaultHandler)
}

// NewLog creates a new instance of the logger using the current default handler
// for its output.
func NewLog(ctx ...interface{}) *Logger {
	l := log15.New(ctx...)
	l.SetHandler(defaultHandler)
	return &Logger{Logger: l}
}

// Errorf will write a formatted Error to the default log.
func (l *Logger) Errorf(format string, args ...interface{}) {
	l.Error(fmt.Sprintf(format, args...))
}

// Debugf will write a formatted Debug to the default log.
func (l *Logger) Debugf(format string, args ...interface{}) {
	l.Debug(fmt.Sprintf(format, args...))
}

// Critf will write a formatted Crit to the default log.
func (l *Logger) Critf(format string, args ...interface{}) {
	l.Crit(fmt.Sprintf(format, args...))
}

// Infof will write a formatted Info to the default log.
func (l *Logger) Infof(format string, args ...interface{}) {
	l.Info(fmt.Sprintf(format, args...))
}

// Warnf will write a formatted Warn to the default log.
func (l *Logger) Warnf(format string, args ...interface{}) {
	l.Warn(fmt.Sprintf(format, args...))
}

// Logf is short hand to create a message string using fmt.Sprintf.
func Logf(format string, args ...interface{}) string {
	return fmt.Sprintf(format, args...)
}

// SetDefaultLogHandler sets the handler for the logger. It wraps handlers in a SyncHandler. You
// should not pass in handlers that are already wrapped in a SyncHandler.
func SetDefaultLogHandler(handler log15.Handler) {
	defaultHandler = log15.SyncHandler(handler)
	Log.SetHandler(defaultHandler)
}

// Sets a new log level for the global logging and the default handler.
func SetLogLvl(lvl log15.Lvl) {
	SetDefaultLogHandler(log15.LvlFilterHandler(lvl, stdHandler))
	Log.SetHandler(defaultHandler)
}

// DefaultLogHandler returns the current handler. It can be used to create additional
// logger instances that all use the same handler for output.
func DefaultLogHandler() log15.Handler {
	return defaultHandler
}
<commit_msg>Add log with panic.<commit_after>package app

import (
	"fmt"
	"os"

	"github.com/inconshreveable/log15"
)

// Logger holds the application logger
type Logger struct {
	log15.Logger
}

var (
	// Log Is the global log variable.
	Log = newLog()

	// stdHandler is the log handler with level applied
	stdHandler = log15.StreamHandler(os.Stdout, log15.LogfmtFormat())

	// Default handler used in the package.
	defaultHandler log15.Handler
)

func newLog() *Logger {
	return &Logger{
		Logger: log15.New(),
	}
}

func init() {
	SetDefaultLogHandler(log15.LvlFilterHandler(log15.LvlInfo, stdHandler))
	Log.SetHandler(defaultHandler)
}

// NewLog creates a new instance of the logger using the current default handler
// for its output.
func NewLog(ctx ...interface{}) *Logger {
	l := log15.New(ctx...)
	l.SetHandler(defaultHandler)
	return &Logger{Logger: l}
}

// Errorf will write a formatted Error to the default log.
func (l *Logger) Errorf(format string, args ...interface{}) {
	l.Error(fmt.Sprintf(format, args...))
}

// Debugf will write a formatted Debug to the default log.
func (l *Logger) Debugf(format string, args ...interface{}) {
	l.Debug(fmt.Sprintf(format, args...))
}

// Critf will write a formatted Crit to the default log.
func (l *Logger) Critf(format string, args ...interface{}) {
	l.Crit(fmt.Sprintf(format, args...))
}

// Infof will write a formatted Info to the default log.
func (l *Logger) Infof(format string, args ...interface{}) {
	l.Info(fmt.Sprintf(format, args...))
}

// Warnf will write a formatted Warn to the default log.
func (l *Logger) Warnf(format string, args ...interface{}) {
	l.Warn(fmt.Sprintf(format, args...))
}

func (l *Logger) Panicf(format string, args ...interface{}) {
	l.Crit(fmt.Sprintf(format, args...))
	Panicf(format, args...)
}

// Logf is short hand to create a message string using fmt.Sprintf.
func Logf(format string, args ...interface{}) string {
	return fmt.Sprintf(format, args...)
}

// SetDefaultLogHandler sets the handler for the logger. It wraps handlers in a SyncHandler. You
// should not pass in handlers that are already wrapped in a SyncHandler.
func SetDefaultLogHandler(handler log15.Handler) {
	defaultHandler = log15.SyncHandler(handler)
	Log.SetHandler(defaultHandler)
}

// Sets a new log level for the global logging and the default handler.
func SetLogLvl(lvl log15.Lvl) {
	SetDefaultLogHandler(log15.LvlFilterHandler(lvl, stdHandler))
	Log.SetHandler(defaultHandler)
}

// DefaultLogHandler returns the current handler. It can be used to create additional
// logger instances that all use the same handler for output.
func DefaultLogHandler() log15.Handler {
	return defaultHandler
}
<|endoftext|>


<commit_before>package cfg

import (
	"errors"
	"flag"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/github"
	"golang.org/x/oauth2/google"

	"github.com/spf13/viper"
)

// config lasso jwt cookie configuration
type config struct {
	LogLevel      string   `mapstructure:"logLevel"`
	Listen        string   `mapstructure:"listen"`
	Port          int      `mapstructure:"port"`
	Domains       []string `mapstructure:"domains"`
	WhiteList     []string `mapstructure:"whitelist"`
	AllowAllUsers bool     `mapstructure:"allowAllUsers"`
	PublicAccess  bool     `mapstructure:"publicAccess"`
	JWT           struct {
		MaxAge   int    `mapstructure:"maxAge"`
		Issuer   string `mapstructure:"issuer"`
		Secret   string `mapstructure:"secret"`
		Compress bool   `mapstructure:"compress"`
	}
	Cookie struct {
		Name     string `mapstructure:"name"`
		Domain   string `mapstructure:"domain"`
		Secure   bool   `mapstructure:"secure"`
		HTTPOnly bool   `mapstructure:"httpOnly"`
	}
	Headers struct {
		JWT         string `mapstructure:"jwt"`
		User        string `mapstructure:"user"`
		QueryString string `mapstructure:"querystring"`
		Redirect    string `mapstructure:"redirect"`
		Success     string `mapstructure:"success"`
	}
	DB struct {
		File string `mapstructure:"file"`
	}
	Session struct {
		Name string `mapstructure:"name"`
	}
	TestURL  string   `mapstructure:"test_url"`
	TestURLs []string `mapstructure:"test_urls"`
	Testing  bool     `mapstructure:"testing"`
	WebApp   bool     `mapstructure:"webapp"`
}

// oauth config items endoint for access
type oauthConfig struct {
	Provider        string   `mapstructure:"provider"`
	ClientID        string   `mapstructure:"client_id"`
	ClientSecret    string   `mapstructure:"client_secret"`
	AuthURL         string   `mapstructure:"auth_url"`
	TokenURL        string   `mapstructure:"token_url"`
	RedirectURL     string   `mapstructure:"callback_url"`
	RedirectURLs    []string `mapstructure:"callback_urls"`
	Scopes          []string `mapstructure:"scopes"`
	UserInfoURL     string   `mapstructure:"user_info_url"`
	PreferredDomain string   `mapstructre:"preferredDomain"`
}

// OAuthProviders holds the stings for
type OAuthProviders struct {
	Google    string
	GitHub    string
	IndieAuth string
	OIDC      string
}

type branding struct {
	LCName string // lower case
	UCName string // upper case
	CcName string // camel case
}

var (
	// Branding that's our name
	Branding = branding{"lasso", "LASSO", "Lasso"}

	// Cfg the main exported config variable
	Cfg config

	// GenOAuth exported OAuth config variable
	// TODO: I think GenOAuth and OAuthConfig can be combined!
	// perhaps by https://golang.org/doc/effective_go.html#embedding
	GenOAuth *oauthConfig

	// OAuthClient is the configured client which will call the provider
	// this actually carries the oauth2 client ala oauthclient.Client(oauth2.NoContext, providerToken)
	OAuthClient *oauth2.Config
	// OAuthopts authentication options
	OAuthopts oauth2.AuthCodeOption

	// Providers static strings to test against
	Providers = &OAuthProviders{
		Google:    "google",
		GitHub:    "github",
		IndieAuth: "indieauth",
		OIDC:      "oidc",
	}
)

// RequiredOptions must have these fields set for minimum viable config
var RequiredOptions = []string{"oauth.provider", "oauth.client_id"}

func init() {
	// from config file
	ParseConfig()

	// can pass loglevel on the command line
	var ll = flag.String("loglevel", Cfg.LogLevel, "enable debug log output")
	flag.Parse()
	if *ll == "debug" {
		log.SetLevel(log.DebugLevel)
		log.Debug("logLevel set to debug")
	}

	setDefaults()

	errT := BasicTest()
	if errT != nil {
		// log.Fatalf(errT.Error())
		panic(errT)
	}

	var listen = Cfg.Listen + ":" + strconv.Itoa(Cfg.Port)
	if !isTCPPortAvailable(listen) {
		log.Fatal(errors.New("check the port availability (is " + Branding.CcName + " already running?)"))
	}

	log.Debug(viper.AllSettings())
}

// ParseConfig parse the config file
func ParseConfig() {
	log.Debug("opening config")
	if os.Getenv(Branding.UCName+"_CONFIG") != "" {
		log.Infof("config file loaded from environmental variable %s: %s", Branding.UCName+"_CONFIG", os.Getenv(Branding.UCName+"_CONFIG"))
		viper.SetConfigFile(os.Getenv(Branding.UCName + "_CONFIG"))
	} else {
	viper.SetConfigName("config")
	viper.SetConfigType("yaml")
	viper.AddConfigPath(os.Getenv(Branding.UCName+"_ROOT") + "config")
	}
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		log.Fatalf("Fatal error config file: %s", err.Error())
		panic(err)
	}
	UnmarshalKey(Branding.LCName, &Cfg)
	// don't log the secret!
	// log.Debugf("secret: %s", string(Cfg.JWT.Secret))
}

// UnmarshalKey populate struct from contents of cfg tree at key
func UnmarshalKey(key string, rawVal interface{}) error {
	return viper.UnmarshalKey(key, rawVal)
}

// Get string value for key
func Get(key string) string {
	return viper.GetString(key)
}

// BasicTest just a quick sanity check to see if the config is sound
func BasicTest() error {
	for _, opt := range RequiredOptions {
		if !viper.IsSet(opt) {
			return errors.New("configuration option " + opt + " is not set in config")
		}
	}
	return nil
}

// setDefaults set default options for some items
func setDefaults() {

	// this should really be done by Viper up in parseConfig but..
	// nested defaults is currently *broken*
	// https://github.com/spf13/viper/issues/309
	// viper.SetDefault("listen", "0.0.0.0")
	// viper.SetDefault(Cfg.Port, 9090)
	// viper.SetDefault("Headers.SSO", "X-"+Branding.CcName+"-Token")
	// viper.SetDefault("Headers.Redirect", "X-"+Branding.CcName+"-Requested-URI")
	// viper.SetDefault("Cookie.Name", "Lasso")

	// logging
	if !viper.IsSet(Branding.LCName + ".logLevel") {
		Cfg.LogLevel = "info"
	}
	// network defaults
	if !viper.IsSet(Branding.LCName + ".listen") {
		Cfg.Listen = "0.0.0.0"
	}
	if !viper.IsSet(Branding.LCName + ".port") {
		Cfg.Port = 9090
	}
	if !viper.IsSet(Branding.LCName + ".allowAllUsers") {
		Cfg.AllowAllUsers = false
	}
	if !viper.IsSet(Branding.LCName + ".publicAccess") {
		Cfg.PublicAccess = false
	}

	// jwt defaults
	if !viper.IsSet(Branding.LCName + ".jwt.secret") {
		Cfg.JWT.Secret = getOrGenerateJWTSecret()
	}
	if !viper.IsSet(Branding.LCName + ".jwt.issuer") {
		Cfg.JWT.Issuer = Branding.CcName
	}
	if !viper.IsSet(Branding.LCName + ".jwt.maxAge") {
		Cfg.JWT.MaxAge = 240
	}
	if !viper.IsSet(Branding.LCName + ".jwt.compress") {
		Cfg.JWT.Compress = true
	}

	// cookie defaults
	if !viper.IsSet(Branding.LCName + ".cookie.name") {
		Cfg.Cookie.Name = "LassoCookie"
	}
	if !viper.IsSet(Branding.LCName + ".cookie.secure") {
		Cfg.Cookie.Secure = false
	}
	if !viper.IsSet(Branding.LCName + ".cookie.httpOnly") {
		Cfg.Cookie.HTTPOnly = true
	}

	// headers defaults
	if !viper.IsSet(Branding.LCName + ".headers.jwt") {
		Cfg.Headers.JWT = "X-" + Branding.CcName + "-Token"
	}
	if !viper.IsSet(Branding.LCName + ".headers.querystring") {
		Cfg.Headers.QueryString = "access_token"
	}
	if !viper.IsSet(Branding.LCName + ".headers.redirect") {
		Cfg.Headers.Redirect = "X-" + Branding.CcName + "-Requested-URI"
	}
	if !viper.IsSet(Branding.LCName + ".headers.user") {
		Cfg.Headers.User = "X-" + Branding.CcName + "-User"
	}
	if !viper.IsSet(Branding.LCName + ".headers.success") {
		Cfg.Headers.Success = "X-" + Branding.CcName + "-Success"
	}

	// db defaults
	if !viper.IsSet(Branding.LCName + ".db.file") {
		Cfg.DB.File = "data/" + Branding.LCName + "_bolt.db"
	}

	// session HERE
	if !viper.IsSet(Branding.LCName + ".session.name") {
		Cfg.Session.Name = Branding.LCName + "Session"
	}

	// testing convenience variable
	if !viper.IsSet(Branding.LCName + ".testing") {
		Cfg.Testing = false
	}
	if viper.IsSet(Branding.LCName + ".test_url") {
		Cfg.TestURLs = append(Cfg.TestURLs, Cfg.TestURL)
	}
	// TODO: proably change this name, maybe set the domain/port the webapp runs on
	if !viper.IsSet(Branding.LCName + ".webapp") {
		Cfg.WebApp = false
	}

	// OAuth defaults and client configuration
	err := UnmarshalKey("oauth", &GenOAuth)
	if err == nil {
		if GenOAuth.Provider == Providers.Google {
			setDefaultsGoogle()
			// setDefaultsGoogle also configures the OAuthClient
		} else if GenOAuth.Provider == Providers.GitHub {
			setDefaultsGitHub()
			configureOAuthClient()
		} else {
			configureOAuthClient()
		}
	}
}

func setDefaultsGoogle() {
	log.Info("configuring Google OAuth")
	GenOAuth.UserInfoURL = "https://www.googleapis.com/oauth2/v3/userinfo"
	OAuthClient = &oauth2.Config{
		ClientID:     GenOAuth.ClientID,
		ClientSecret: GenOAuth.ClientSecret,
		Scopes: []string{
			// You have to select a scope from
			// https://developers.google.com/identity/protocols/googlescopes#google_sign-in
			"https://www.googleapis.com/auth/userinfo.email",
		},
		Endpoint: google.Endpoint,
	}
	if GenOAuth.PreferredDomain != "" {
		log.Infof("setting Google OAuth preferred login domain param 'hd' to %s", GenOAuth.PreferredDomain)
		OAuthopts = oauth2.SetAuthURLParam("hd", GenOAuth.PreferredDomain)
	} else {
		OAuthopts = oauth2.SetAuthURLParam("hd", "")
	}
}

func setDefaultsGitHub() {
	// log.Info("configuring GitHub OAuth")
	if GenOAuth.AuthURL == "" {
		GenOAuth.AuthURL = github.Endpoint.AuthURL
	}
	if GenOAuth.TokenURL == "" {
		GenOAuth.TokenURL = github.Endpoint.TokenURL
	}
	if GenOAuth.UserInfoURL == "" {
		GenOAuth.UserInfoURL = "https://api.github.com/user?access_token="
	}
	if len(GenOAuth.Scopes) == 0 {
		GenOAuth.Scopes = []string{"user"}
	}
}

func configureOAuthClient() {
	log.Infof("configuring %s OAuth with Endpoint %s", GenOAuth.Provider, GenOAuth.AuthURL)
	OAuthClient = &oauth2.Config{
		ClientID:     GenOAuth.ClientID,
		ClientSecret: GenOAuth.ClientSecret,
		Endpoint: oauth2.Endpoint{
			AuthURL:  GenOAuth.AuthURL,
			TokenURL: GenOAuth.TokenURL,
		},
		RedirectURL: GenOAuth.RedirectURL,
		Scopes:      GenOAuth.Scopes,
	}
}

var secretFile = os.Getenv("LASSO_ROOT") + "config/secret"

// a-z A-Z 0-9 except no l, o, O
const charRunes = "abcdefghijkmnpqrstuvwxyzABCDEFGHIJKLMNPQRSTUVWXYZ012346789"

const secretLen = 18

func getOrGenerateJWTSecret() string {
	b, err := ioutil.ReadFile(secretFile)
	if err == nil {
		log.Info("jwt.secret read from " + secretFile)
	} else {
		// then generate a new secret and store it in the file
		log.Debug(err)
		log.Info("jwt.secret not found in " + secretFile)
		log.Warn("generating random jwt.secret and storing it in " + secretFile)

		rand.Seed(time.Now().UnixNano())
		b := make([]byte, secretLen)
		for i := range b {
			b[i] = charRunes[rand.Intn(len(charRunes))]
		}
		err := ioutil.WriteFile(secretFile, b, 0600)
		if err != nil {
			log.Debug(err)
		}
	}
	return string(b)
}

func isTCPPortAvailable(listen string) bool {
	log.Debug("checking availability of tcp port: " + listen)
	conn, err := net.Listen("tcp", listen)
	if err != nil {
		log.Error(err)
		return false
	}
	conn.Close()
	return true
}
<commit_msg>additional domain and callback checks<commit_after>package cfg

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/github"
	"golang.org/x/oauth2/google"

	"github.com/spf13/viper"
)

// config lasso jwt cookie configuration
type config struct {
	LogLevel      string   `mapstructure:"logLevel"`
	Listen        string   `mapstructure:"listen"`
	Port          int      `mapstructure:"port"`
	Domains       []string `mapstructure:"domains"`
	WhiteList     []string `mapstructure:"whitelist"`
	AllowAllUsers bool     `mapstructure:"allowAllUsers"`
	PublicAccess  bool     `mapstructure:"publicAccess"`
	JWT           struct {
		MaxAge   int    `mapstructure:"maxAge"`
		Issuer   string `mapstructure:"issuer"`
		Secret   string `mapstructure:"secret"`
		Compress bool   `mapstructure:"compress"`
	}
	Cookie struct {
		Name     string `mapstructure:"name"`
		Domain   string `mapstructure:"domain"`
		Secure   bool   `mapstructure:"secure"`
		HTTPOnly bool   `mapstructure:"httpOnly"`
	}
	Headers struct {
		JWT         string `mapstructure:"jwt"`
		User        string `mapstructure:"user"`
		QueryString string `mapstructure:"querystring"`
		Redirect    string `mapstructure:"redirect"`
		Success     string `mapstructure:"success"`
	}
	DB struct {
		File string `mapstructure:"file"`
	}
	Session struct {
		Name string `mapstructure:"name"`
	}
	TestURL  string   `mapstructure:"test_url"`
	TestURLs []string `mapstructure:"test_urls"`
	Testing  bool     `mapstructure:"testing"`
	WebApp   bool     `mapstructure:"webapp"`
}

// oauth config items endoint for access
type oauthConfig struct {
	Provider        string   `mapstructure:"provider"`
	ClientID        string   `mapstructure:"client_id"`
	ClientSecret    string   `mapstructure:"client_secret"`
	AuthURL         string   `mapstructure:"auth_url"`
	TokenURL        string   `mapstructure:"token_url"`
	RedirectURL     string   `mapstructure:"callback_url"`
	RedirectURLs    []string `mapstructure:"callback_urls"`
	Scopes          []string `mapstructure:"scopes"`
	UserInfoURL     string   `mapstructure:"user_info_url"`
	PreferredDomain string   `mapstructre:"preferredDomain"`
}

// OAuthProviders holds the stings for
type OAuthProviders struct {
	Google    string
	GitHub    string
	IndieAuth string
	OIDC      string
}

type branding struct {
	LCName string // lower case
	UCName string // upper case
	CcName string // camel case
}

var (
	// Branding that's our name
	Branding = branding{"lasso", "LASSO", "Lasso"}

	// Cfg the main exported config variable
	Cfg config

	// GenOAuth exported OAuth config variable
	// TODO: I think GenOAuth and OAuthConfig can be combined!
	// perhaps by https://golang.org/doc/effective_go.html#embedding
	GenOAuth *oauthConfig

	// OAuthClient is the configured client which will call the provider
	// this actually carries the oauth2 client ala oauthclient.Client(oauth2.NoContext, providerToken)
	OAuthClient *oauth2.Config
	// OAuthopts authentication options
	OAuthopts oauth2.AuthCodeOption

	// Providers static strings to test against
	Providers = &OAuthProviders{
		Google:    "google",
		GitHub:    "github",
		IndieAuth: "indieauth",
		OIDC:      "oidc",
	}
)

// RequiredOptions must have these fields set for minimum viable config
var RequiredOptions = []string{"oauth.provider", "oauth.client_id"}

func init() {
	// from config file
	ParseConfig()

	// can pass loglevel on the command line
	var ll = flag.String("loglevel", Cfg.LogLevel, "enable debug log output")
	flag.Parse()
	if *ll == "debug" {
		log.SetLevel(log.DebugLevel)
		log.Debug("logLevel set to debug")
	}

	setDefaults()

	errT := BasicTest()
	if errT != nil {
		// log.Fatalf(errT.Error())
		panic(errT)
	}

	var listen = Cfg.Listen + ":" + strconv.Itoa(Cfg.Port)
	if !isTCPPortAvailable(listen) {
		log.Fatal(errors.New(listen + " is not available (is " + Branding.CcName + " already running?)"))
	}

	log.Debug(viper.AllSettings())
}

// ParseConfig parse the config file
func ParseConfig() {
	log.Debug("opening config")
	if os.Getenv(Branding.UCName+"_CONFIG") != "" {
		log.Infof("config file loaded from environmental variable %s: %s", Branding.UCName+"_CONFIG", os.Getenv(Branding.UCName+"_CONFIG"))
		viper.SetConfigFile(os.Getenv(Branding.UCName + "_CONFIG"))
	} else {
		viper.SetConfigName("config")
		viper.SetConfigType("yaml")
		viper.AddConfigPath(os.Getenv(Branding.UCName+"_ROOT") + "config")
	}
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		log.Fatalf("Fatal error config file: %s", err.Error())
		panic(err)
	}
	UnmarshalKey(Branding.LCName, &Cfg)
	// don't log the secret!
	// log.Debugf("secret: %s", string(Cfg.JWT.Secret))
}

// UnmarshalKey populate struct from contents of cfg tree at key
func UnmarshalKey(key string, rawVal interface{}) error {
	return viper.UnmarshalKey(key, rawVal)
}

// Get string value for key
func Get(key string) string {
	return viper.GetString(key)
}

// BasicTest just a quick sanity check to see if the config is sound
func BasicTest() error {
	for _, opt := range RequiredOptions {
		if !viper.IsSet(opt) {
			return errors.New("configuration error: required configuration option " + opt + " is not set")
		}
	}
	// Domains is required _unless_ Cfg.AllowAllUsers is set
	if !viper.IsSet(Branding.LCName+".allowAllUsers") && !viper.IsSet(Branding.LCName+".domains") {
		return fmt.Errorf("configuration error: either one of %s or %s needs to be set (but not both)", Branding.LCName+".domains", Branding.LCName+".allowAllUsers")
	}

	if !viper.IsSet(Branding.LCName + ".allowAllUsers") {
		if GenOAuth.RedirectURL != "" {
			if err := checkCallbackConfig(GenOAuth.RedirectURL); err != nil {
				return err
			}
		}
		if len(GenOAuth.RedirectURLs) > 0 {
			for _, cb := range GenOAuth.RedirectURLs {
				if err := checkCallbackConfig(cb); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func checkCallbackConfig(url string) error {
	inDomain := false
	for _, d := range Cfg.Domains {
		if strings.Contains(url, d) {
			inDomain = true
			break
		}
	}
	if !inDomain {
		return fmt.Errorf("configuration error: oauth.callback_url (%s) must be within the configured domain where the cookie will be set %s", url, Cfg.Domains)
	}

	if !strings.Contains(url, "/auth") {
		return fmt.Errorf("configuration error: oauth.callback_url (%s) must contain '/auth'", OAuthClient.RedirectURL)
	}
	return nil
}

// setDefaults set default options for some items
func setDefaults() {

	// this should really be done by Viper up in parseConfig but..
	// nested defaults is currently *broken*
	// https://github.com/spf13/viper/issues/309
	// viper.SetDefault("listen", "0.0.0.0")
	// viper.SetDefault(Cfg.Port, 9090)
	// viper.SetDefault("Headers.SSO", "X-"+Branding.CcName+"-Token")
	// viper.SetDefault("Headers.Redirect", "X-"+Branding.CcName+"-Requested-URI")
	// viper.SetDefault("Cookie.Name", "Lasso")

	// logging
	if !viper.IsSet(Branding.LCName + ".logLevel") {
		Cfg.LogLevel = "info"
	}
	// network defaults
	if !viper.IsSet(Branding.LCName + ".listen") {
		Cfg.Listen = "0.0.0.0"
	}
	if !viper.IsSet(Branding.LCName + ".port") {
		Cfg.Port = 9090
	}
	if !viper.IsSet(Branding.LCName + ".allowAllUsers") {
		Cfg.AllowAllUsers = false
	}
	if !viper.IsSet(Branding.LCName + ".publicAccess") {
		Cfg.PublicAccess = false
	}

	// jwt defaults
	if !viper.IsSet(Branding.LCName + ".jwt.secret") {
		Cfg.JWT.Secret = getOrGenerateJWTSecret()
	}
	if !viper.IsSet(Branding.LCName + ".jwt.issuer") {
		Cfg.JWT.Issuer = Branding.CcName
	}
	if !viper.IsSet(Branding.LCName + ".jwt.maxAge") {
		Cfg.JWT.MaxAge = 240
	}
	if !viper.IsSet(Branding.LCName + ".jwt.compress") {
		Cfg.JWT.Compress = true
	}

	// cookie defaults
	if !viper.IsSet(Branding.LCName + ".cookie.name") {
		Cfg.Cookie.Name = Branding.CcName + "Cookie"
	}
	if !viper.IsSet(Branding.LCName + ".cookie.secure") {
		Cfg.Cookie.Secure = false
	}
	if !viper.IsSet(Branding.LCName + ".cookie.httpOnly") {
		Cfg.Cookie.HTTPOnly = true
	}

	// headers defaults
	if !viper.IsSet(Branding.LCName + ".headers.jwt") {
		Cfg.Headers.JWT = "X-" + Branding.CcName + "-Token"
	}
	if !viper.IsSet(Branding.LCName + ".headers.querystring") {
		Cfg.Headers.QueryString = "access_token"
	}
	if !viper.IsSet(Branding.LCName + ".headers.redirect") {
		Cfg.Headers.Redirect = "X-" + Branding.CcName + "-Requested-URI"
	}
	if !viper.IsSet(Branding.LCName + ".headers.user") {
		Cfg.Headers.User = "X-" + Branding.CcName + "-User"
	}
	if !viper.IsSet(Branding.LCName + ".headers.success") {
		Cfg.Headers.Success = "X-" + Branding.CcName + "-Success"
	}

	// db defaults
	if !viper.IsSet(Branding.LCName + ".db.file") {
		Cfg.DB.File = "data/" + Branding.LCName + "_bolt.db"
	}

	// session HERE
	if !viper.IsSet(Branding.LCName + ".session.name") {
		Cfg.Session.Name = Branding.CcName + "Session"
	}

	// testing convenience variable
	if !viper.IsSet(Branding.LCName + ".testing") {
		Cfg.Testing = false
	}
	if viper.IsSet(Branding.LCName + ".test_url") {
		Cfg.TestURLs = append(Cfg.TestURLs, Cfg.TestURL)
	}
	// TODO: proably change this name, maybe set the domain/port the webapp runs on
	if !viper.IsSet(Branding.LCName + ".webapp") {
		Cfg.WebApp = false
	}

	// OAuth defaults and client configuration
	err := UnmarshalKey("oauth", &GenOAuth)
	if err == nil {
		if GenOAuth.Provider == Providers.Google {
			setDefaultsGoogle()
			// setDefaultsGoogle also configures the OAuthClient
		} else if GenOAuth.Provider == Providers.GitHub {
			setDefaultsGitHub()
			configureOAuthClient()
		} else {
			configureOAuthClient()
		}
	}
}

func setDefaultsGoogle() {
	log.Info("configuring Google OAuth")
	GenOAuth.UserInfoURL = "https://www.googleapis.com/oauth2/v3/userinfo"
	OAuthClient = &oauth2.Config{
		ClientID:     GenOAuth.ClientID,
		ClientSecret: GenOAuth.ClientSecret,
		Scopes: []string{
			// You have to select a scope from
			// https://developers.google.com/identity/protocols/googlescopes#google_sign-in
			"https://www.googleapis.com/auth/userinfo.email",
		},
		Endpoint: google.Endpoint,
	}
	if GenOAuth.PreferredDomain != "" {
		log.Infof("setting Google OAuth preferred login domain param 'hd' to %s", GenOAuth.PreferredDomain)
		OAuthopts = oauth2.SetAuthURLParam("hd", GenOAuth.PreferredDomain)
	} else {
		OAuthopts = oauth2.SetAuthURLParam("hd", "")
	}
}

func setDefaultsGitHub() {
	// log.Info("configuring GitHub OAuth")
	if GenOAuth.AuthURL == "" {
		GenOAuth.AuthURL = github.Endpoint.AuthURL
	}
	if GenOAuth.TokenURL == "" {
		GenOAuth.TokenURL = github.Endpoint.TokenURL
	}
	if GenOAuth.UserInfoURL == "" {
		GenOAuth.UserInfoURL = "https://api.github.com/user?access_token="
	}
	if len(GenOAuth.Scopes) == 0 {
		GenOAuth.Scopes = []string{"user"}
	}
}

func configureOAuthClient() {
	log.Infof("configuring %s OAuth with Endpoint %s", GenOAuth.Provider, GenOAuth.AuthURL)
	OAuthClient = &oauth2.Config{
		ClientID:     GenOAuth.ClientID,
		ClientSecret: GenOAuth.ClientSecret,
		Endpoint: oauth2.Endpoint{
			AuthURL:  GenOAuth.AuthURL,
			TokenURL: GenOAuth.TokenURL,
		},
		RedirectURL: GenOAuth.RedirectURL,
		Scopes:      GenOAuth.Scopes,
	}
}

var secretFile = os.Getenv("LASSO_ROOT") + "config/secret"

// a-z A-Z 0-9 except no l, o, O
const charRunes = "abcdefghijkmnpqrstuvwxyzABCDEFGHIJKLMNPQRSTUVWXYZ012346789"

const secretLen = 18

func getOrGenerateJWTSecret() string {
	b, err := ioutil.ReadFile(secretFile)
	if err == nil {
		log.Info("jwt.secret read from " + secretFile)
	} else {
		// then generate a new secret and store it in the file
		log.Debug(err)
		log.Info("jwt.secret not found in " + secretFile)
		log.Warn("generating random jwt.secret and storing it in " + secretFile)

		rand.Seed(time.Now().UnixNano())
		b := make([]byte, secretLen)
		for i := range b {
			b[i] = charRunes[rand.Intn(len(charRunes))]
		}
		err := ioutil.WriteFile(secretFile, b, 0600)
		if err != nil {
			log.Debug(err)
		}
	}
	return string(b)
}

func isTCPPortAvailable(listen string) bool {
	log.Debug("checking availability of tcp port: " + listen)
	conn, err := net.Listen("tcp", listen)
	if err != nil {
		log.Error(err)
		return false
	}
	conn.Close()
	return true
}
<|endoftext|>


<commit_before>package mux

import (
	"errors"
	"fmt"
	"log"
	"sync"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/client"
)

// Mux will manage all connections and subscriptions. Will check if subscriptions
// limit is reached and spawn new connection when that happens. It will also listen
// to all incomming client messages and reconnect client with all its subscriptions
// in case of a failure
type Mux struct {
	CID     int
	Inbound chan client.Msg
	Clients map[int]*client.Client
	mtx     *sync.RWMutex
	Err     error
	APIKey  string
	APISec  string
}

// New returns pointer to instance of mux
func New() *Mux {
	return &Mux{
		Inbound: make(chan client.Msg),
		Clients: make(map[int]*client.Client),
		mtx:     &sync.RWMutex{},
	}
}

// Subscribe - given the details in form of hash table, subscribes client
func (m *Mux) Subscribe(sub map[string]string) *Mux {
	if m.Err != nil {
		return m
	}

	if alreadySubscribed := m.Clients[m.CID].Subs.Added(sub); alreadySubscribed {
		return m
	}

	m.Clients[m.CID].Subscribe(sub)

	if limitReached := m.Clients[m.CID].Subs.LimitReached(); limitReached {
		log.Printf("30 subs limit is reached on cID: %d, spawning new conn\n", m.CID)
		m.AddClient()
	}
	return m
}

// AddClient adds public or authenticated client depending
// on mux api keys presence
func (m *Mux) AddClient() *Mux {
	m.mtx.Lock()
	defer m.mtx.Unlock()

	if len(m.APIKey) == 0 && len(m.APISec) == 0 {
		return m.addPublicClient()
	}

	return m.addPrivateClient()
}

// Listen accepts a callback func that will get called each time mux receives a
// message from any of its clients/subscriptions. It should be called last, after
// all setup calls are made as it's blocking
func (m *Mux) Listen(cb func([]byte, error)) error {
	if m.Err != nil {
		return m.Err
	}

	for {
		select {
		case msg, ok := <-m.Inbound:
			if !ok {
				return errors.New("channel has closed unexpectedly, restart")
			}

			if msg.Err != nil {
				cb(nil, fmt.Errorf("conn:%d has failed | err:%s | reconnecting", msg.CID, msg.Err))
				m.reconnect(msg.CID)
				continue
			}

			cb(msg.Msg, nil)
		}
	}
}

func (m *Mux) reconnect(cID int) {
	// pull old client subscriptions
	subs := m.Clients[cID].Subs.GetAll()
	// add fresh client
	m.AddClient()
	// resubscribe old events
	for subID, sub := range subs {
		log.Printf("resubscribing: %s\n", subID)
		m.Subscribe(sub)
	}
	// remove old, closed channel from the lost
	delete(m.Clients, cID)
}

func (m *Mux) addPublicClient() *Mux {
	if m.Err != nil {
		return m
	}

	// adding new client so making sure we increment cid
	m.CID++

	c := client.New(m.CID).Public()
	if c.Err != nil {
		m.Err = c.Err
		return m
	}

	m.Clients[m.CID] = c
	// start listening for incoming messages
	go c.Read(m.Inbound)
	return m
}

func (m *Mux) addPrivateClient() *Mux {
	if m.Err != nil {
		return m
	}

	// TODO: implement auth channel handler
	return m
}
<commit_msg>making some mux properties private and adding option to transform raw data into data structs<commit_after>package mux

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sync"
	"unicode"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/event"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/client"
)

// Mux will manage all connections and subscriptions. Will check if subscriptions
// limit is reached and spawn new connection when that happens. It will also listen
// to all incomming client messages and reconnect client with all its subscriptions
// in case of a failure
type Mux struct {
	cid           int
	publicInbound chan client.Msg
	authInbound   chan client.Msg
	clients       map[int]*client.Client
	mtx           *sync.RWMutex
	Err           error
	transform     bool
	apiKey        string
	apiSec        string
}

// New returns pointer to instance of mux
func New() *Mux {
	return &Mux{
		publicInbound: make(chan client.Msg),
		clients:       make(map[int]*client.Client),
		mtx:           &sync.RWMutex{},
	}
}

func (m *Mux) TransformRaw() *Mux {
	m.transform = true
	return m
}

// Subscribe - given the details in form of hash table, subscribes client
func (m *Mux) Subscribe(sub map[string]string) *Mux {
	if m.Err != nil {
		return m
	}

	if alreadySubscribed := m.clients[m.cid].Subs.Added(sub); alreadySubscribed {
		return m
	}

	m.clients[m.cid].Subscribe(sub)

	if limitReached := m.clients[m.cid].Subs.LimitReached(); limitReached {
		log.Printf("30 subs limit is reached on cid: %d, spawning new conn\n", m.cid)
		m.AddClient()
	}
	return m
}

// AddClient adds public or authenticated client depending
// on mux api keys presence
func (m *Mux) AddClient() *Mux {
	m.mtx.Lock()
	defer m.mtx.Unlock()

	if len(m.apiKey) == 0 && len(m.apiSec) == 0 {
		return m.addPublicClient()
	}

	return m.addPrivateClient()
}

// Listen accepts a callback func that will get called each time mux receives a
// message from any of its clients/subscriptions. It should be called last, after
// all setup calls are made as it's blocking
func (m *Mux) Listen(cb func(interface{}, error)) error {
	if m.Err != nil {
		return m.Err
	}

	for {
		select {
		case msg, ok := <-m.authInbound:
			if !ok {
				return errors.New("authenticated channel has closed unexpectedly")
			}
			cb(msg.Data, nil)
		case msg, ok := <-m.publicInbound:
			if !ok {
				return errors.New("channel has closed unexpectedly")
			}

			if msg.Err != nil {
				cb(nil, fmt.Errorf("conn:%d has failed | err:%s | reconnecting", msg.CID, msg.Err))
				m.reconnect(msg.CID)
				continue
			}

			if !m.transform {
				cb(msg.Data, nil)
				continue
			}

			t := bytes.TrimLeftFunc(msg.Data, unicode.IsSpace)
			if bytes.HasPrefix(t, []byte("[")) {
				cb(msg.Data, nil)
				continue
			}

			if bytes.HasPrefix(t, []byte("{")) {
				e := &event.Event{}
				if err := json.Unmarshal(msg.Data, e); err != nil {
					cb(nil, fmt.Errorf("failed parsing msg: %s", err))
					continue
				}
				cb(e, nil)
				continue
			}

			cb(nil, fmt.Errorf("unrecognized msg signature: %s", msg.Data))
		}
	}
}

func (m *Mux) reconnect(cid int) {
	// pull old client subscriptions
	subs := m.clients[cid].Subs.GetAll()
	// add fresh client
	m.AddClient()
	// resubscribe old events
	for subID, sub := range subs {
		log.Printf("resubscribing: %s\n", subID)
		m.Subscribe(sub)
	}
	// remove old, closed channel from the lost
	delete(m.clients, cid)
}

func (m *Mux) addPublicClient() *Mux {
	if m.Err != nil {
		return m
	}
	// adding new client so making sure we increment cid
	m.cid++

	c := client.New(m.cid).Public()
	if c.Err != nil {
		m.Err = c.Err
		return m
	}

	m.clients[m.cid] = c
	// start listening for incoming messages
	go c.Read(m.publicInbound)
	return m
}

func (m *Mux) addPrivateClient() *Mux {
	if m.Err != nil {
		return m
	}

	// TODO: implement auth channel handler
	return m
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"os"

	"flag"
	"io"

	"github.com/bborbe/log"

	"fmt"
	"runtime"
	"sync"

	http_client "github.com/bborbe/http/client"
	"github.com/bborbe/http/client/builder"
	http_downloader "github.com/bborbe/http/downloader"
	http_downloader_by_url "github.com/bborbe/http/downloader/by_url"
	io_util "github.com/bborbe/io/util"
)

var logger = log.DefaultLogger

const (
	PARAMETER_LOGLEVEL           = "loglevel"
	PARAMETER_PARALLEL_DOWNLOADS = "max"
	PARAMETER_TARGET             = "target"
	DEFAULT_PARALLEL_DOWNLOADS   = 2
	DEFAULT_TARGET               = "~/Downloads"
)

func main() {
	defer logger.Close()
	logLevelPtr := flag.String(PARAMETER_LOGLEVEL, log.INFO_STRING, log.FLAG_USAGE)
	maxConcurrencyDownloadsPtr := flag.Int(PARAMETER_PARALLEL_DOWNLOADS, DEFAULT_PARALLEL_DOWNLOADS, "max parallel downloads")
	targetDirectoryPtr := flag.String(PARAMETER_TARGET, DEFAULT_TARGET, "directory")
	flag.Parse()
	logger.SetLevelThreshold(log.LogStringToLevel(*logLevelPtr))
	logger.Debugf("set log level to %s", *logLevelPtr)

	runtime.GOMAXPROCS(runtime.NumCPU())

	writer := os.Stdout
	input := os.Stdin
	wg := new(sync.WaitGroup)
	httpClientBuilder := builder.New()
	client := http_client.New(httpClientBuilder.Build())
	downloader := http_downloader_by_url.New(client.Get)
	err := do(writer, input, *maxConcurrencyDownloadsPtr, wg, downloader, *targetDirectoryPtr)
	wg.Wait()
	if err != nil {
		logger.Fatal(err)
		logger.Close()
		os.Exit(1)
	}
}

func do(writer io.Writer, input io.Reader, maxConcurrencyDownloads int, wg *sync.WaitGroup, downloader http_downloader.Downloader, targetDirectoryName string) error {
	var err error
	if targetDirectoryName, err = io_util.NormalizePath(targetDirectoryName); err != nil {
		return err
	}
	if err = io_util.IsDirectory(targetDirectoryName); err != nil {
		fmt.Fprintf(writer, "parameter %s is invalid\n", PARAMETER_TARGET)
		return err
	}
	targetDirectory, err := os.Open(targetDirectoryName)
	if err != nil {
		return err
	}
	throttle := make(chan bool, maxConcurrencyDownloads)
	reader := bufio.NewReader(input)
	for {
		line, _, err := reader.ReadLine()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		wg.Add(1)
		go func() {
			link := string(line)
			throttle <- true
			downloader.Download(link, targetDirectory)
			<-throttle
			wg.Done()
		}()
	}
	return nil
}
<commit_msg>use http client builder<commit_after>package main

import (
	"bufio"
	"os"

	"flag"
	"io"

	"github.com/bborbe/log"

	"fmt"
	"runtime"
	"sync"

	http_client "github.com/bborbe/http/client"
	http_client_builder "github.com/bborbe/http/client/builder"
	http_downloader "github.com/bborbe/http/downloader"
	http_downloader_by_url "github.com/bborbe/http/downloader/by_url"
	io_util "github.com/bborbe/io/util"
)

var logger = log.DefaultLogger

const (
	PARAMETER_LOGLEVEL           = "loglevel"
	PARAMETER_PARALLEL_DOWNLOADS = "max"
	PARAMETER_TARGET             = "target"
	DEFAULT_PARALLEL_DOWNLOADS   = 2
	DEFAULT_TARGET               = "~/Downloads"
)

func main() {
	defer logger.Close()
	logLevelPtr := flag.String(PARAMETER_LOGLEVEL, log.INFO_STRING, log.FLAG_USAGE)
	maxConcurrencyDownloadsPtr := flag.Int(PARAMETER_PARALLEL_DOWNLOADS, DEFAULT_PARALLEL_DOWNLOADS, "max parallel downloads")
	targetDirectoryPtr := flag.String(PARAMETER_TARGET, DEFAULT_TARGET, "directory")
	flag.Parse()
	logger.SetLevelThreshold(log.LogStringToLevel(*logLevelPtr))
	logger.Debugf("set log level to %s", *logLevelPtr)

	runtime.GOMAXPROCS(runtime.NumCPU())

	writer := os.Stdout
	input := os.Stdin
	wg := new(sync.WaitGroup)
	httpClientBuilder := builder.New()
	client := http_client.New(httpClientBuilder.Build())
	downloader := http_downloader_by_url.New(client.Get)
	err := do(writer, input, *maxConcurrencyDownloadsPtr, wg, downloader, *targetDirectoryPtr)
	wg.Wait()
	if err != nil {
		logger.Fatal(err)
		logger.Close()
		os.Exit(1)
	}
}

func do(writer io.Writer, input io.Reader, maxConcurrencyDownloads int, wg *sync.WaitGroup, downloader http_downloader.Downloader, targetDirectoryName string) error {
	var err error
	if targetDirectoryName, err = io_util.NormalizePath(targetDirectoryName); err != nil {
		return err
	}
	if err = io_util.IsDirectory(targetDirectoryName); err != nil {
		fmt.Fprintf(writer, "parameter %s is invalid\n", PARAMETER_TARGET)
		return err
	}
	targetDirectory, err := os.Open(targetDirectoryName)
	if err != nil {
		return err
	}
	throttle := make(chan bool, maxConcurrencyDownloads)
	reader := bufio.NewReader(input)
	for {
		line, _, err := reader.ReadLine()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		wg.Add(1)
		go func() {
			link := string(line)
			throttle <- true
			downloader.Download(link, targetDirectory)
			<-throttle
			wg.Done()
		}()
	}
	return nil
}
<|endoftext|>


<commit_before>package pool

import (
	"errors"
	"github.com/DVI-GI-2017/Jira__backend/db"
	"github.com/DVI-GI-2017/Jira__backend/services"
)

var typesActionList = [...]string{
	"Insert", "Find", "Insert and Find",
}

type Action struct {
	Type string
}

func NewAction(actionType string) (*Action, error) {
	if checkActionType(actionType) {
		return &Action{
			Type: actionType,
		}, nil
	} else {
		return &Action{}, errors.New("Can't create new action!")
	}
}

func checkActionType(actionType string) bool {
	for _, value := range typesActionList {
		if value == actionType {
			return true
		}
	}

	return false
}
<commit_msg>del path from action<commit_after>package pool

import (
	"errors"
)

var typesActionList = [...]string{
	"Insert", "Find", "Insert and Find",
}

type Action struct {
	Type string
}

func NewAction(actionType string) (*Action, error) {
	if checkActionType(actionType) {
		return &Action{
			Type: actionType,
		}, nil
	} else {
		return &Action{}, errors.New("Can't create new action!")
	}
}

func checkActionType(actionType string) bool {
	for _, value := range typesActionList {
		if value == actionType {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package post

import (
	"database/sql"
	"fmt"
	"log"
	"math"

	"github.com/rwcarlsen/cyan/query"
)

// The number of sql commands to buffer before dumping to the output database.
const DumpFreq = 100000

var (
	preExecStmts = []string{
		"CREATE TABLE IF NOT EXISTS AgentExit (SimId BLOB,AgentId INTEGER,ExitTime INTEGER);",
		"CREATE TABLE IF NOT EXISTS Compositions (SimId BLOB,QualId INTEGER,NucId INTEGER, MassFrac REAL);",
		"CREATE TABLE IF NOT EXISTS Products (SimId BLOB,QualId INTEGER,Quality TEXT);",
		"CREATE TABLE IF NOT EXISTS Resources (SimId INTEGER,ResourceId INTEGER,ObjId INTEGER,Type TEXT,TimeCreated INTEGER,Quantity REAL,Units TEXT,QualId INTEGER,Parent1 INTEGER,Parent2 INTEGER);",
		"CREATE TABLE IF NOT EXISTS ResCreators (SimId INTEGER,ResourceId INTEGER,AgentId INTEGER);",
		"CREATE TABLE IF NOT EXISTS Agents (SimId BLOB,AgentId INTEGER,Kind TEXT,Spec TEXT,Prototype TEXT,ParentId INTEGER,Lifetime INTEGER,EnterTime INTEGER,ExitTime INTEGER);",
		"CREATE TABLE IF NOT EXISTS Inventories (SimId BLOB,ResourceId INTEGER,AgentId INTEGER,StartTime INTEGER,EndTime INTEGER,QualId INTEGER,Quantity REAL);",
		"CREATE TABLE IF NOT EXISTS TimeList (SimId BLOB, Time INTEGER);",
		"CREATE TABLE IF NOT EXISTS Transactions (SimId BLOB, TransactionId INTEGER, SenderId INTEGER, ReceiverId INTEGER, ResourceId INTEGER, Commodity TEXT, Time INTEGER);",
		query.Index("TimeList", "Time"),
		query.Index("Resources", "SimId", "ResourceId", "QualId"),
		query.Index("Compositions", "SimId", "QualId", "NucId"),
		query.Index("Transactions", "SimId", "ResourceId"),
		query.Index("Transactions", "TransactionId"),
		query.Index("ResCreators", "SimId", "ResourceId"),
	}
	postExecStmts = []string{
		query.Index("Agents", "SimId", "Prototype"),
		query.Index("Inventories", "SimId", "AgentId"),
		query.Index("Inventories", "SimId", "ResourceId", "StartTime"),
		"ANALYZE;",
	}
	dumpSql    = "INSERT INTO Inventories VALUES (?,?,?,?,?,?,?);"
	resSqlHead = "SELECT ResourceId,TimeCreated,QualId,Quantity FROM "
	resSqlTail = " WHERE Parent1 = ? OR Parent2 = ?;"

	ownerSql = `SELECT tr.ReceiverId, tr.Time FROM Transactions AS tr
				  WHERE tr.ResourceId = ? AND tr.SimId = ?
				  ORDER BY tr.Time ASC;`
	rootsSql = `SELECT res.ResourceId,res.TimeCreated,rc.AgentId,res.QualId,Quantity FROM Resources AS res
				  INNER JOIN ResCreators AS rc ON res.ResourceId = rc.ResourceId
				  WHERE res.SimId = ? AND rc.SimId = ?;`
)

// Prepare creates necessary indexes and tables required for efficient
// calculation of cyclus simulation inventory information.  Should be called
// once before walking begins.
func Prepare(db *sql.DB) (err error) {
	for _, s := range preExecStmts {
		if _, err := db.Exec(s); err != nil {
			log.Println("    ", err)
		}
	}
	return nil
}

// Finish should be called for a cyclus database after all walkers have
// completed processing inventory data. It creates final indexes and other
// finishing tasks.
func Finish(db *sql.DB) (err error) {
	for _, s := range postExecStmts {
		if _, err := db.Exec(s); err != nil {
			return err
		}
	}
	return nil
}

type Node struct {
	ResId     int
	OwnerId   int
	StartTime int
	EndTime   int
	QualId    int
	Quantity  float64
}

// Context encapsulates the logic for building a fast, queryable inventories
// table for a specific simulation from raw cyclus output database.
type Context struct {
	*sql.DB
	// Simid is the cyclus simulation id targeted by this context.  Must be
	// set.
	Simid       []byte
	Log         *log.Logger
	mappednodes map[int32]struct{}
	tmpResTbl   string
	tmpResStmt  *sql.Stmt
	dumpStmt    *sql.Stmt
	ownerStmt   *sql.Stmt
	resCount    int
	nodes       []*Node
}

func NewContext(db *sql.DB, simid []byte) *Context {
	return &Context{
		DB:    db,
		Simid: simid,
		Log:   log.New(NullWriter{}, "", 0),
	}
}

type AlreadyPostErr []byte

func (s AlreadyPostErr) Error() string {
	return fmt.Sprintf("SimId %x is already post processed", []byte(s))
}

func IsAlreadyPostErr(err error) bool {
	_, ok := err.(AlreadyPostErr)
	return ok
}

func (c *Context) init() {
	// skip if the post processing already exists for this simid in the db
	dummy := 0
	err := c.QueryRow("SELECT AgentId FROM Agents WHERE SimId = ? LIMIT 1", c.Simid).Scan(&dummy)
	if err == nil {
		panic(AlreadyPostErr(c.Simid))
	} else if err != sql.ErrNoRows {
		panicif(err)
	}

	tx, err := c.Begin()
	panicif(err)

	// build Agents table
	sql := `INSERT INTO Agents
				SELECT n.SimId,n.AgentId,n.Kind,n.Spec,n.Prototype,n.ParentId,n.Lifetime,n.EnterTime,x.ExitTime
				FROM
					AgentEntry AS n
					LEFT JOIN AgentExit AS x ON n.AgentId = x.AgentId AND n.SimId = x.SimId AND n.SimId = ?;`
	_, err = tx.Exec(sql, c.Simid)
	panicif(err)

	c.nodes = make([]*Node, 0, 10000)
	c.mappednodes = map[int32]struct{}{}

	// build TimeList table
	sql = "SELECT Duration FROM Info WHERE SimId = ?;"
	rows, err := tx.Query(sql, c.Simid)
	panicif(err)
	defer rows.Close()
	for rows.Next() {
		var dur int
		panicif(rows.Scan(&dur))
		for i := 0; i < dur; i++ {
			_, err := tx.Exec("INSERT INTO TimeList VALUES (?, ?);", c.Simid, i)
			panicif(err)
		}
	}
	panicif(rows.Err())

	// create temp res table without simid
	c.Log.Println("Creating temporary resource table...")
	c.tmpResTbl = "tmp_restbl_" + fmt.Sprintf("%x", c.Simid)
	_, err = tx.Exec("DROP TABLE IF EXISTS " + c.tmpResTbl)
	panicif(err)

	sql = "CREATE TABLE " + c.tmpResTbl + " AS SELECT ResourceId,TimeCreated,Parent1,Parent2,QualId,Quantity FROM Resources WHERE SimId = ?;"
	_, err = tx.Exec(sql, c.Simid)
	panicif(err)

	c.Log.Println("Indexing temporary resource table...")
	_, err = tx.Exec(query.Index(c.tmpResTbl, "Parent1"))
	panicif(err)

	_, err = tx.Exec(query.Index(c.tmpResTbl, "Parent2"))
	panicif(err)

	tx.Commit()

	// create prepared statements
	c.tmpResStmt, err = c.Prepare(resSqlHead + c.tmpResTbl + resSqlTail)
	panicif(err)

	c.dumpStmt, err = c.Prepare(dumpSql)
	panicif(err)

	c.ownerStmt, err = c.Prepare(ownerSql)
	panicif(err)
}

// WalkAll constructs the inventories table in the cyclus database alongside
// other tables. Creates several indexes in the process.  Finish should be
// called on the database connection after all simulation id's have been
// walked.
func (c *Context) WalkAll() (err error) {
	defer func() {
		if r := recover(); r != nil {
			if er, ok := r.(error); ok {
				err = er
			} else {
				err = fmt.Errorf("%v", r)
			}
		}
	}()

	c.Log.Printf("--- Building inventories for simid %x ---\n", c.Simid)
	c.init()

	c.Log.Println("Retrieving root resource nodes...")
	roots := c.getRoots()

	c.Log.Printf("Found %v root nodes\n", len(roots))
	for i, n := range roots {
		c.Log.Printf("    Processing root %d...\n", i)
		c.walkDown(n)
	}

	c.Log.Println("Dropping temporary resource table...")
	_, err = c.Exec("DROP TABLE " + c.tmpResTbl)
	panicif(err)

	c.dumpNodes()

	return nil
}

func (c *Context) getRoots() (roots []*Node) {
	sql := "SELECT COUNT(*) FROM ResCreators WHERE SimId = ?"
	row := c.QueryRow(sql, c.Simid)

	n := 0
	err := row.Scan(&n)
	panicif(err)

	roots = make([]*Node, 0, n)
	rows, err := c.Query(rootsSql, c.Simid, c.Simid)
	panicif(err)
	defer rows.Close()
	for rows.Next() {
		node := &Node{EndTime: math.MaxInt32}
		err := rows.Scan(&node.ResId, &node.StartTime, &node.OwnerId, &node.QualId, &node.Quantity)
		panicif(err)

		roots = append(roots, node)
	}
	panicif(rows.Err())
	return roots
}

func (c *Context) walkDown(node *Node) {
	if _, ok := c.mappednodes[int32(node.ResId)]; ok {
		return
	}
	c.mappednodes[int32(node.ResId)] = struct{}{}

	// dump if necessary
	c.resCount++
	if c.resCount%DumpFreq == 0 {
		c.dumpNodes()
	}

	// find resource's children
	kids := make([]*Node, 0, 2)
	rows, err := c.tmpResStmt.Query(node.ResId, node.ResId)
	panicif(err)
	defer rows.Close()

	for rows.Next() {
		child := &Node{EndTime: math.MaxInt32}
		err := rows.Scan(&child.ResId, &child.StartTime, &child.QualId, &child.Quantity)
		panicif(err)
		node.EndTime = child.StartTime
		kids = append(kids, child)
	}
	panicif(rows.Err())
	rows.Close() // Close is idempotent - and this function is recursive.

	// find resources owner changes (that occurred before children)
	owners, times := c.getNewOwners(node.ResId)

	childOwner := node.OwnerId
	if len(owners) > 0 {
		node.EndTime = times[0]
		childOwner = owners[len(owners)-1]

		lastend := math.MaxInt32
		if len(kids) > 0 {
			lastend = kids[0].StartTime
		}
		times = append(times, lastend)
		for i := range owners {
			n := &Node{ResId: node.ResId,
				OwnerId:   owners[i],
				StartTime: times[i],
				EndTime:   times[i+1],
				QualId:    node.QualId,
				Quantity:  node.Quantity,
			}
			c.nodes = append(c.nodes, n)
		}
	}

	c.nodes = append(c.nodes, node)

	// walk down resource's children
	for _, child := range kids {
		child.OwnerId = childOwner
		c.walkDown(child)
	}
}

func (c *Context) getNewOwners(id int) (owners, times []int) {
	var owner, t int
	rows, err := c.ownerStmt.Query(id, c.Simid)
	panicif(err)
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&owner, &t)
		panicif(err)

		if id == owner {
			continue
		}
		owners = append(owners, owner)
		times = append(times, t)
	}
	panicif(rows.Err())
	return owners, times
}

func (c *Context) dumpNodes() {
	c.Log.Printf("    Dumping inventories (%d resources done)...\n", c.resCount)
	tx, err := c.Begin()
	panicif(err)
	stmt := tx.Stmt(c.dumpStmt)

	for _, n := range c.nodes {
		if n.EndTime > n.StartTime {
			_, err = stmt.Exec(c.Simid, n.ResId, n.OwnerId, n.StartTime, n.EndTime, n.QualId, n.Quantity)
			panicif(err)
		}
	}

	err = tx.Commit()
	panicif(err)
	c.nodes = c.nodes[:0]
}
<commit_msg>add convenience process function to post pkg<commit_after>package post

import (
	"database/sql"
	"fmt"
	"log"
	"math"

	"github.com/rwcarlsen/cyan/query"
)

// The number of sql commands to buffer before dumping to the output database.
const DumpFreq = 100000

var (
	preExecStmts = []string{
		"CREATE TABLE IF NOT EXISTS AgentExit (SimId BLOB,AgentId INTEGER,ExitTime INTEGER);",
		"CREATE TABLE IF NOT EXISTS Compositions (SimId BLOB,QualId INTEGER,NucId INTEGER, MassFrac REAL);",
		"CREATE TABLE IF NOT EXISTS Products (SimId BLOB,QualId INTEGER,Quality TEXT);",
		"CREATE TABLE IF NOT EXISTS Resources (SimId INTEGER,ResourceId INTEGER,ObjId INTEGER,Type TEXT,TimeCreated INTEGER,Quantity REAL,Units TEXT,QualId INTEGER,Parent1 INTEGER,Parent2 INTEGER);",
		"CREATE TABLE IF NOT EXISTS ResCreators (SimId INTEGER,ResourceId INTEGER,AgentId INTEGER);",
		"CREATE TABLE IF NOT EXISTS Agents (SimId BLOB,AgentId INTEGER,Kind TEXT,Spec TEXT,Prototype TEXT,ParentId INTEGER,Lifetime INTEGER,EnterTime INTEGER,ExitTime INTEGER);",
		"CREATE TABLE IF NOT EXISTS Inventories (SimId BLOB,ResourceId INTEGER,AgentId INTEGER,StartTime INTEGER,EndTime INTEGER,QualId INTEGER,Quantity REAL);",
		"CREATE TABLE IF NOT EXISTS TimeList (SimId BLOB, Time INTEGER);",
		"CREATE TABLE IF NOT EXISTS Transactions (SimId BLOB, TransactionId INTEGER, SenderId INTEGER, ReceiverId INTEGER, ResourceId INTEGER, Commodity TEXT, Time INTEGER);",
		query.Index("TimeList", "Time"),
		query.Index("Resources", "SimId", "ResourceId", "QualId"),
		query.Index("Compositions", "SimId", "QualId", "NucId"),
		query.Index("Transactions", "SimId", "ResourceId"),
		query.Index("Transactions", "TransactionId"),
		query.Index("ResCreators", "SimId", "ResourceId"),
	}
	postExecStmts = []string{
		query.Index("Agents", "SimId", "Prototype"),
		query.Index("Inventories", "SimId", "AgentId"),
		query.Index("Inventories", "SimId", "ResourceId", "StartTime"),
		"ANALYZE;",
	}
	dumpSql    = "INSERT INTO Inventories VALUES (?,?,?,?,?,?,?);"
	resSqlHead = "SELECT ResourceId,TimeCreated,QualId,Quantity FROM "
	resSqlTail = " WHERE Parent1 = ? OR Parent2 = ?;"

	ownerSql = `SELECT tr.ReceiverId, tr.Time FROM Transactions AS tr
				  WHERE tr.ResourceId = ? AND tr.SimId = ?
				  ORDER BY tr.Time ASC;`
	rootsSql = `SELECT res.ResourceId,res.TimeCreated,rc.AgentId,res.QualId,Quantity FROM Resources AS res
				  INNER JOIN ResCreators AS rc ON res.ResourceId = rc.ResourceId
				  WHERE res.SimId = ? AND rc.SimId = ?;`
)

func Process(db *sql.DB) (simids [][]byte, err error) {
	err = Prepare(db)
	if err != nil {
		return nil, err
	}
	defer Finish(db)

	simids, err = GetSimIds(db)
	if err != nil {
		return nil, err
	}

	for _, id := range simids {
		ctx := NewContext(db, id)
		if err := ctx.WalkAll(); err != nil {
			log.Print(err)
		}
	}
	return simids, nil
}

// Prepare creates necessary indexes and tables required for efficient
// calculation of cyclus simulation inventory information.  Should be called
// once before walking begins.
func Prepare(db *sql.DB) (err error) {
	for _, s := range preExecStmts {
		if _, err := db.Exec(s); err != nil {
			log.Println("    ", err)
		}
	}
	return nil
}

// Finish should be called for a cyclus database after all walkers have
// completed processing inventory data. It creates final indexes and other
// finishing tasks.
func Finish(db *sql.DB) (err error) {
	for _, s := range postExecStmts {
		if _, err := db.Exec(s); err != nil {
			return err
		}
	}
	return nil
}

type Node struct {
	ResId     int
	OwnerId   int
	StartTime int
	EndTime   int
	QualId    int
	Quantity  float64
}

// Context encapsulates the logic for building a fast, queryable inventories
// table for a specific simulation from raw cyclus output database.
type Context struct {
	*sql.DB
	// Simid is the cyclus simulation id targeted by this context.  Must be
	// set.
	Simid       []byte
	Log         *log.Logger
	mappednodes map[int32]struct{}
	tmpResTbl   string
	tmpResStmt  *sql.Stmt
	dumpStmt    *sql.Stmt
	ownerStmt   *sql.Stmt
	resCount    int
	nodes       []*Node
}

func NewContext(db *sql.DB, simid []byte) *Context {
	return &Context{
		DB:    db,
		Simid: simid,
		Log:   log.New(NullWriter{}, "", 0),
	}
}

type AlreadyPostErr []byte

func (s AlreadyPostErr) Error() string {
	return fmt.Sprintf("SimId %x is already post processed", []byte(s))
}

func IsAlreadyPostErr(err error) bool {
	_, ok := err.(AlreadyPostErr)
	return ok
}

func (c *Context) init() {
	// skip if the post processing already exists for this simid in the db
	dummy := 0
	err := c.QueryRow("SELECT AgentId FROM Agents WHERE SimId = ? LIMIT 1", c.Simid).Scan(&dummy)
	if err == nil {
		panic(AlreadyPostErr(c.Simid))
	} else if err != sql.ErrNoRows {
		panicif(err)
	}

	tx, err := c.Begin()
	panicif(err)

	// build Agents table
	sql := `INSERT INTO Agents
				SELECT n.SimId,n.AgentId,n.Kind,n.Spec,n.Prototype,n.ParentId,n.Lifetime,n.EnterTime,x.ExitTime
				FROM
					AgentEntry AS n
					LEFT JOIN AgentExit AS x ON n.AgentId = x.AgentId AND n.SimId = x.SimId AND n.SimId = ?;`
	_, err = tx.Exec(sql, c.Simid)
	panicif(err)

	c.nodes = make([]*Node, 0, 10000)
	c.mappednodes = map[int32]struct{}{}

	// build TimeList table
	sql = "SELECT Duration FROM Info WHERE SimId = ?;"
	rows, err := tx.Query(sql, c.Simid)
	panicif(err)
	defer rows.Close()
	for rows.Next() {
		var dur int
		panicif(rows.Scan(&dur))
		for i := 0; i < dur; i++ {
			_, err := tx.Exec("INSERT INTO TimeList VALUES (?, ?);", c.Simid, i)
			panicif(err)
		}
	}
	panicif(rows.Err())

	// create temp res table without simid
	c.Log.Println("Creating temporary resource table...")
	c.tmpResTbl = "tmp_restbl_" + fmt.Sprintf("%x", c.Simid)
	_, err = tx.Exec("DROP TABLE IF EXISTS " + c.tmpResTbl)
	panicif(err)

	sql = "CREATE TABLE " + c.tmpResTbl + " AS SELECT ResourceId,TimeCreated,Parent1,Parent2,QualId,Quantity FROM Resources WHERE SimId = ?;"
	_, err = tx.Exec(sql, c.Simid)
	panicif(err)

	c.Log.Println("Indexing temporary resource table...")
	_, err = tx.Exec(query.Index(c.tmpResTbl, "Parent1"))
	panicif(err)

	_, err = tx.Exec(query.Index(c.tmpResTbl, "Parent2"))
	panicif(err)

	tx.Commit()

	// create prepared statements
	c.tmpResStmt, err = c.Prepare(resSqlHead + c.tmpResTbl + resSqlTail)
	panicif(err)

	c.dumpStmt, err = c.Prepare(dumpSql)
	panicif(err)

	c.ownerStmt, err = c.Prepare(ownerSql)
	panicif(err)
}

// WalkAll constructs the inventories table in the cyclus database alongside
// other tables. Creates several indexes in the process.  Finish should be
// called on the database connection after all simulation id's have been
// walked.
func (c *Context) WalkAll() (err error) {
	defer func() {
		if r := recover(); r != nil {
			if er, ok := r.(error); ok {
				err = er
			} else {
				err = fmt.Errorf("%v", r)
			}
		}
	}()

	c.Log.Printf("--- Building inventories for simid %x ---\n", c.Simid)
	c.init()

	c.Log.Println("Retrieving root resource nodes...")
	roots := c.getRoots()

	c.Log.Printf("Found %v root nodes\n", len(roots))
	for i, n := range roots {
		c.Log.Printf("    Processing root %d...\n", i)
		c.walkDown(n)
	}

	c.Log.Println("Dropping temporary resource table...")
	_, err = c.Exec("DROP TABLE " + c.tmpResTbl)
	panicif(err)

	c.dumpNodes()

	return nil
}

func (c *Context) getRoots() (roots []*Node) {
	sql := "SELECT COUNT(*) FROM ResCreators WHERE SimId = ?"
	row := c.QueryRow(sql, c.Simid)

	n := 0
	err := row.Scan(&n)
	panicif(err)

	roots = make([]*Node, 0, n)
	rows, err := c.Query(rootsSql, c.Simid, c.Simid)
	panicif(err)
	defer rows.Close()
	for rows.Next() {
		node := &Node{EndTime: math.MaxInt32}
		err := rows.Scan(&node.ResId, &node.StartTime, &node.OwnerId, &node.QualId, &node.Quantity)
		panicif(err)

		roots = append(roots, node)
	}
	panicif(rows.Err())
	return roots
}

func (c *Context) walkDown(node *Node) {
	if _, ok := c.mappednodes[int32(node.ResId)]; ok {
		return
	}
	c.mappednodes[int32(node.ResId)] = struct{}{}

	// dump if necessary
	c.resCount++
	if c.resCount%DumpFreq == 0 {
		c.dumpNodes()
	}

	// find resource's children
	kids := make([]*Node, 0, 2)
	rows, err := c.tmpResStmt.Query(node.ResId, node.ResId)
	panicif(err)
	defer rows.Close()

	for rows.Next() {
		child := &Node{EndTime: math.MaxInt32}
		err := rows.Scan(&child.ResId, &child.StartTime, &child.QualId, &child.Quantity)
		panicif(err)
		node.EndTime = child.StartTime
		kids = append(kids, child)
	}
	panicif(rows.Err())
	rows.Close() // Close is idempotent - and this function is recursive.

	// find resources owner changes (that occurred before children)
	owners, times := c.getNewOwners(node.ResId)

	childOwner := node.OwnerId
	if len(owners) > 0 {
		node.EndTime = times[0]
		childOwner = owners[len(owners)-1]

		lastend := math.MaxInt32
		if len(kids) > 0 {
			lastend = kids[0].StartTime
		}
		times = append(times, lastend)
		for i := range owners {
			n := &Node{ResId: node.ResId,
				OwnerId:   owners[i],
				StartTime: times[i],
				EndTime:   times[i+1],
				QualId:    node.QualId,
				Quantity:  node.Quantity,
			}
			c.nodes = append(c.nodes, n)
		}
	}

	c.nodes = append(c.nodes, node)

	// walk down resource's children
	for _, child := range kids {
		child.OwnerId = childOwner
		c.walkDown(child)
	}
}

func (c *Context) getNewOwners(id int) (owners, times []int) {
	var owner, t int
	rows, err := c.ownerStmt.Query(id, c.Simid)
	panicif(err)
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&owner, &t)
		panicif(err)

		if id == owner {
			continue
		}
		owners = append(owners, owner)
		times = append(times, t)
	}
	panicif(rows.Err())
	return owners, times
}

func (c *Context) dumpNodes() {
	c.Log.Printf("    Dumping inventories (%d resources done)...\n", c.resCount)
	tx, err := c.Begin()
	panicif(err)
	stmt := tx.Stmt(c.dumpStmt)

	for _, n := range c.nodes {
		if n.EndTime > n.StartTime {
			_, err = stmt.Exec(c.Simid, n.ResId, n.OwnerId, n.StartTime, n.EndTime, n.QualId, n.Quantity)
			panicif(err)
		}
	}

	err = tx.Commit()
	panicif(err)
	c.nodes = c.nodes[:0]
}
<|endoftext|>


<commit_before>package docker

import (
	"archive/tar"
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
	"time"

	"github.com/NetSys/quilt/util"

	log "github.com/Sirupsen/logrus"
	dkc "github.com/fsouza/go-dockerclient"
)

const (
	// The root namespace for all labels
	labelBase = "q."

	// This is the namespace for user defined labels
	userLabelPrefix = labelBase + "user.label."

	// This is the namespace for system defined labels
	systemLabelPrefix = labelBase + "system.label."

	// LabelTrueValue is needed because a label has to be a key/value pair, hence
	// this is the value that will be used if we're only interested in the key
	LabelTrueValue = "1"

	// SchedulerLabelKey is the key, used by the scheduler.
	SchedulerLabelKey = systemLabelPrefix + "Quilt"

	// SchedulerLabelValue is the value, used by the scheduler.
	SchedulerLabelValue = "Scheduler"

	// SchedulerLabelPair is the key/value pair, used by the scheduler.
	SchedulerLabelPair = SchedulerLabelKey + "=" + SchedulerLabelValue
)

var errNoSuchContainer = errors.New("container does not exist")

// A Container as returned by the docker client API.
type Container struct {
	ID     string
	Name   string
	Image  string
	IP     string
	Path   string
	Args   []string
	Pid    int
	Env    map[string]string
	Labels map[string]string
}

// ContainerSlice is an alias for []Container to allow for joins
type ContainerSlice []Container

// A Client to the local docker daemon.
type Client interface {
	Run(opts RunOptions) error
	Exec(name string, cmd ...string) error
	ExecVerbose(name string, cmd ...string) ([]byte, error)
	Remove(name string) error
	RemoveID(id string) error
	Pull(image string) error
	List(filters map[string][]string) ([]Container, error)
	Get(id string) (Container, error)
	WriteToContainer(id, src, dst, archiveName string, permission int) error
	GetFromContainer(id string, src string) (string, error)
	IsRunning(name string) (bool, error)
}

// RunOptions changes the behavior of the Run function.
type RunOptions struct {
	Name   string
	Image  string
	Args   []string
	Labels map[string]string
	Env    map[string]struct{}

	NetworkMode string
	PidMode     string
	Privileged  bool
	VolumesFrom []string
}

type pullRequest struct {
	image string
	done  chan error
}

type docker struct {
	*dkc.Client

	pullChan chan pullRequest
}

// New creates client to the docker daemon.
func New(sock string) Client {
	var client *dkc.Client
	for {
		var err error
		client, err = dkc.NewClient(sock)
		if err != nil {
			log.WithError(err).Warn("Failed to create docker client.")
			time.Sleep(10 * time.Second)
			continue
		}
		break
	}

	dk := docker{client, make(chan pullRequest)}
	go pullServer(dk)

	return dk
}

func pullServer(dk docker) {
	images := make(map[string]struct{})

	for req := range dk.pullChan {
		if _, ok := images[req.image]; ok {
			req.done <- nil
			continue
		}

		log.Infof("Pulling docker image %s.", req.image)
		opts := dkc.PullImageOptions{Repository: string(req.image)}
		err := dk.PullImage(opts, dkc.AuthConfiguration{})

		if err != nil {
			log.WithError(err).Errorf("Failed to pull image %s.", req.image)
		} else {
			images[req.image] = struct{}{}
		}
		req.done <- err
	}
}

func (dk docker) Run(opts RunOptions) error {
	if opts.Name != "" {
		_, err := dk.getID(opts.Name)
		if err == errNoSuchContainer {
			// Only log the first time we attempt to boot.
			log.Infof("Start Container: %s", opts.Name)
		} else if err != nil {
			return err
		}
	}

	hc := dkc.HostConfig{
		NetworkMode: opts.NetworkMode,
		PidMode:     opts.PidMode,
		Privileged:  opts.Privileged,
		VolumesFrom: opts.VolumesFrom,
	}
	id, err := dk.create(opts.Name, opts.Image, opts.Args, opts.Labels, opts.Env, &hc)
	if err != nil {
		return err
	}

	if err = dk.StartContainer(id, &hc); err != nil {
		if _, ok := err.(*dkc.ContainerAlreadyRunning); ok {
			return nil
		}
		dk.removeID(id) // Remove the container to avoid a zombie.
		return err
	}

	return nil
}

func (dk docker) Exec(name string, cmd ...string) error {
	_, err := dk.ExecVerbose(name, cmd...)
	return err
}

func (dk docker) ExecVerbose(name string, cmd ...string) ([]byte, error) {
	id, err := dk.getID(name)
	if err != nil {
		return nil, err
	}

	var inBuff, outBuff bytes.Buffer
	exec, err := dk.CreateExec(dkc.CreateExecOptions{
		Container:    id,
		Cmd:          cmd,
		AttachStdout: true})

	if err != nil {
		return nil, err
	}

	err = dk.StartExec(exec.ID, dkc.StartExecOptions{
		OutputStream: &inBuff,
	})

	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(bytes.NewReader(inBuff.Bytes()))
	for scanner.Scan() {
		outBuff.WriteString(scanner.Text() + "\n")
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return outBuff.Bytes(), nil
}

// WriteToContainer writes the contents of SRC into the file at path DST on the
// container with id ID. Overwrites DST if it already exists.
func (dk docker) WriteToContainer(id, src, dst, archiveName string,
	permission int) error {

	tarBuf, err := util.ToTar(archiveName, permission, src)

	if err != nil {
		return err
	}

	err = dk.UploadToContainer(id, dkc.UploadToContainerOptions{
		InputStream: tarBuf,
		Path:        dst,
	})
	if err != nil {
		return err
	}

	return nil
}

// GetFromContainer returns a string containing the content of the file named
// SRC on the container with id ID.
func (dk docker) GetFromContainer(id string, src string) (string, error) {
	var buffIn bytes.Buffer
	var buffOut bytes.Buffer
	err := dk.DownloadFromContainer(id, dkc.DownloadFromContainerOptions{
		OutputStream: &buffIn,
		Path:         src,
	})
	if err != nil {
		return "", err
	}

	writer := io.Writer(&buffOut)

	for tr := tar.NewReader(&buffIn); err != io.EOF; _, err = tr.Next() {

		if err != nil {
			return "", err
		}

		_, err = io.Copy(writer, tr)
		if err != nil {
			return "", err
		}
	}

	return buffOut.String(), nil
}

func (dk docker) Remove(name string) error {
	id, err := dk.getID(name)
	if err != nil {
		return nil // Can't remove a non-existent container.
	}

	log.WithFields(log.Fields{
		"name": name,
		"id":   id,
	}).Info("Remove container.")
	return dk.removeID(id)
}

func (dk docker) RemoveID(id string) error {
	log.WithField("id", id).Info("Remove Container.")
	return dk.removeID(id)
}

func (dk docker) removeID(id string) error {
	err := dk.RemoveContainer(dkc.RemoveContainerOptions{ID: id, Force: true})
	if err != nil {
		return err
	}

	return nil
}

func (dk docker) Pull(image string) error {
	done := make(chan error)
	dk.pullChan <- pullRequest{image, done}
	return <-done
}

func (dk docker) List(filters map[string][]string) ([]Container, error) {
	return dk.list(filters, false)
}

func (dk docker) list(filters map[string][]string, all bool) ([]Container, error) {
	opts := dkc.ListContainersOptions{All: all, Filters: filters}
	apics, err := dk.ListContainers(opts)
	if err != nil {
		return nil, err
	}

	var containers []Container
	for _, apic := range apics {
		c, err := dk.Get(apic.ID)
		if err != nil {
			log.WithError(err).Warnf("Failed to inspect container: %s",
				apic.ID)
			continue
		}

		containers = append(containers, c)
	}

	return containers, nil
}

func (dk docker) Get(id string) (Container, error) {
	c, err := dk.InspectContainer(id)
	if err != nil {
		return Container{}, err
	}

	env := make(map[string]string)
	for _, value := range c.Config.Env {
		e := strings.Split(value, "=")
		if len(e) > 1 {
			env[e[0]] = e[1]
		}
	}

	return Container{
		Name:   c.Name,
		ID:     c.ID,
		IP:     c.NetworkSettings.IPAddress,
		Image:  c.Config.Image,
		Path:   c.Path,
		Args:   c.Args,
		Pid:    c.State.Pid,
		Env:    env,
		Labels: c.Config.Labels,
	}, nil
}

func (dk docker) IsRunning(name string) (bool, error) {
	containers, err := dk.List(map[string][]string{
		"name": {name},
	})
	if err != nil {
		return false, err
	}
	return len(containers) != 0, nil
}

func (dk docker) create(name, image string, args []string, labels map[string]string,
	env map[string]struct{}, hc *dkc.HostConfig) (string, error) {
	if err := dk.Pull(image); err != nil {
		return "", err
	}

	envList := make([]string, len(env))
	for k := range env {
		envList = append(envList, k)
	}

	container, err := dk.CreateContainer(dkc.CreateContainerOptions{
		Name: name,
		Config: &dkc.Config{
			Image:  string(image),
			Cmd:    args,
			Labels: labels,
			Env:    envList},
		HostConfig: hc,
	})
	if err != nil {
		return "", err
	}

	return container.ID, nil
}

func (dk docker) getID(name string) (string, error) {
	containers, err := dk.list(nil, true)
	if err != nil {
		return "", err
	}

	name = "/" + name
	for _, c := range containers {
		if name == c.Name {
			return c.ID, nil
		}
	}

	return "", errNoSuchContainer
}

// UserLabel returns the supplied label tagged with the user prefix.
func UserLabel(label string) string {
	return userLabelPrefix + label
}

// IsUserLabel returns whether the supplied label represents a Quilt user label.
func IsUserLabel(label string) bool {
	return strings.HasPrefix(label, userLabelPrefix)
}

// ParseUserLabel returns the supplied label with the user prefix stripped.
func ParseUserLabel(label string) string {
	return strings.TrimPrefix(label, userLabelPrefix)
}

// SystemLabel returns the supplied label tagged with the system prefix.
func SystemLabel(label string) string {
	return systemLabelPrefix + label
}

// Get returns the value contained at the given index
func (cs ContainerSlice) Get(ii int) interface{} {
	return cs[ii]
}

// Len returns the number of items in the slice
func (cs ContainerSlice) Len() int {
	return len(cs)
}
<commit_msg>docker: Simplify pull logic<commit_after>package docker

import (
	"archive/tar"
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
	"sync"
	"time"

	"github.com/NetSys/quilt/util"

	log "github.com/Sirupsen/logrus"
	dkc "github.com/fsouza/go-dockerclient"
)

const (
	// The root namespace for all labels
	labelBase = "q."

	// This is the namespace for user defined labels
	userLabelPrefix = labelBase + "user.label."

	// This is the namespace for system defined labels
	systemLabelPrefix = labelBase + "system.label."

	// LabelTrueValue is needed because a label has to be a key/value pair, hence
	// this is the value that will be used if we're only interested in the key
	LabelTrueValue = "1"

	// SchedulerLabelKey is the key, used by the scheduler.
	SchedulerLabelKey = systemLabelPrefix + "Quilt"

	// SchedulerLabelValue is the value, used by the scheduler.
	SchedulerLabelValue = "Scheduler"

	// SchedulerLabelPair is the key/value pair, used by the scheduler.
	SchedulerLabelPair = SchedulerLabelKey + "=" + SchedulerLabelValue
)

var errNoSuchContainer = errors.New("container does not exist")

// A Container as returned by the docker client API.
type Container struct {
	ID     string
	Name   string
	Image  string
	IP     string
	Path   string
	Args   []string
	Pid    int
	Env    map[string]string
	Labels map[string]string
}

// ContainerSlice is an alias for []Container to allow for joins
type ContainerSlice []Container

// A Client to the local docker daemon.
type Client interface {
	Run(opts RunOptions) error
	Exec(name string, cmd ...string) error
	ExecVerbose(name string, cmd ...string) ([]byte, error)
	Remove(name string) error
	RemoveID(id string) error
	Pull(image string) error
	List(filters map[string][]string) ([]Container, error)
	Get(id string) (Container, error)
	WriteToContainer(id, src, dst, archiveName string, permission int) error
	GetFromContainer(id string, src string) (string, error)
	IsRunning(name string) (bool, error)
}

// RunOptions changes the behavior of the Run function.
type RunOptions struct {
	Name   string
	Image  string
	Args   []string
	Labels map[string]string
	Env    map[string]struct{}

	NetworkMode string
	PidMode     string
	Privileged  bool
	VolumesFrom []string
}

type docker struct {
	*dkc.Client
	*sync.Mutex
	imageCache map[string]struct{}
}

// New creates client to the docker daemon.
func New(sock string) Client {
	var client *dkc.Client
	for {
		var err error
		client, err = dkc.NewClient(sock)
		if err != nil {
			log.WithError(err).Warn("Failed to create docker client.")
			time.Sleep(10 * time.Second)
			continue
		}
		break
	}

	return docker{client, &sync.Mutex{}, map[string]struct{}{}}
}

func (dk docker) Run(opts RunOptions) error {
	if opts.Name != "" {
		_, err := dk.getID(opts.Name)
		if err == errNoSuchContainer {
			// Only log the first time we attempt to boot.
			log.Infof("Start Container: %s", opts.Name)
		} else if err != nil {
			return err
		}
	}

	hc := dkc.HostConfig{
		NetworkMode: opts.NetworkMode,
		PidMode:     opts.PidMode,
		Privileged:  opts.Privileged,
		VolumesFrom: opts.VolumesFrom,
	}
	id, err := dk.create(opts.Name, opts.Image, opts.Args, opts.Labels, opts.Env, &hc)
	if err != nil {
		return err
	}

	if err = dk.StartContainer(id, &hc); err != nil {
		if _, ok := err.(*dkc.ContainerAlreadyRunning); ok {
			return nil
		}
		dk.removeID(id) // Remove the container to avoid a zombie.
		return err
	}

	return nil
}

func (dk docker) Exec(name string, cmd ...string) error {
	_, err := dk.ExecVerbose(name, cmd...)
	return err
}

func (dk docker) ExecVerbose(name string, cmd ...string) ([]byte, error) {
	id, err := dk.getID(name)
	if err != nil {
		return nil, err
	}

	var inBuff, outBuff bytes.Buffer
	exec, err := dk.CreateExec(dkc.CreateExecOptions{
		Container:    id,
		Cmd:          cmd,
		AttachStdout: true})

	if err != nil {
		return nil, err
	}

	err = dk.StartExec(exec.ID, dkc.StartExecOptions{
		OutputStream: &inBuff,
	})

	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(bytes.NewReader(inBuff.Bytes()))
	for scanner.Scan() {
		outBuff.WriteString(scanner.Text() + "\n")
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return outBuff.Bytes(), nil
}

// WriteToContainer writes the contents of SRC into the file at path DST on the
// container with id ID. Overwrites DST if it already exists.
func (dk docker) WriteToContainer(id, src, dst, archiveName string,
	permission int) error {

	tarBuf, err := util.ToTar(archiveName, permission, src)

	if err != nil {
		return err
	}

	err = dk.UploadToContainer(id, dkc.UploadToContainerOptions{
		InputStream: tarBuf,
		Path:        dst,
	})
	if err != nil {
		return err
	}

	return nil
}

// GetFromContainer returns a string containing the content of the file named
// SRC on the container with id ID.
func (dk docker) GetFromContainer(id string, src string) (string, error) {
	var buffIn bytes.Buffer
	var buffOut bytes.Buffer
	err := dk.DownloadFromContainer(id, dkc.DownloadFromContainerOptions{
		OutputStream: &buffIn,
		Path:         src,
	})
	if err != nil {
		return "", err
	}

	writer := io.Writer(&buffOut)

	for tr := tar.NewReader(&buffIn); err != io.EOF; _, err = tr.Next() {

		if err != nil {
			return "", err
		}

		_, err = io.Copy(writer, tr)
		if err != nil {
			return "", err
		}
	}

	return buffOut.String(), nil
}

func (dk docker) Remove(name string) error {
	id, err := dk.getID(name)
	if err != nil {
		return nil // Can't remove a non-existent container.
	}

	log.WithFields(log.Fields{
		"name": name,
		"id":   id,
	}).Info("Remove container.")
	return dk.removeID(id)
}

func (dk docker) RemoveID(id string) error {
	log.WithField("id", id).Info("Remove Container.")
	return dk.removeID(id)
}

func (dk docker) removeID(id string) error {
	err := dk.RemoveContainer(dkc.RemoveContainerOptions{ID: id, Force: true})
	if err != nil {
		return err
	}

	return nil
}

func (dk docker) Pull(image string) error {
	dk.Lock()
	defer dk.Unlock()

	if _, ok := dk.imageCache[image]; ok {
		return nil
	}

	log.Infof("Pulling docker image %s.", image)
	opts := dkc.PullImageOptions{Repository: image}
	if err := dk.PullImage(opts, dkc.AuthConfiguration{}); err != nil {
		return err
	}

	dk.imageCache[image] = struct{}{}
	return nil
}

func (dk docker) List(filters map[string][]string) ([]Container, error) {
	return dk.list(filters, false)
}

func (dk docker) list(filters map[string][]string, all bool) ([]Container, error) {
	opts := dkc.ListContainersOptions{All: all, Filters: filters}
	apics, err := dk.ListContainers(opts)
	if err != nil {
		return nil, err
	}

	var containers []Container
	for _, apic := range apics {
		c, err := dk.Get(apic.ID)
		if err != nil {
			log.WithError(err).Warnf("Failed to inspect container: %s",
				apic.ID)
			continue
		}

		containers = append(containers, c)
	}

	return containers, nil
}

func (dk docker) Get(id string) (Container, error) {
	c, err := dk.InspectContainer(id)
	if err != nil {
		return Container{}, err
	}

	env := make(map[string]string)
	for _, value := range c.Config.Env {
		e := strings.Split(value, "=")
		if len(e) > 1 {
			env[e[0]] = e[1]
		}
	}

	return Container{
		Name:   c.Name,
		ID:     c.ID,
		IP:     c.NetworkSettings.IPAddress,
		Image:  c.Config.Image,
		Path:   c.Path,
		Args:   c.Args,
		Pid:    c.State.Pid,
		Env:    env,
		Labels: c.Config.Labels,
	}, nil
}

func (dk docker) IsRunning(name string) (bool, error) {
	containers, err := dk.List(map[string][]string{
		"name": {name},
	})
	if err != nil {
		return false, err
	}
	return len(containers) != 0, nil
}

func (dk docker) create(name, image string, args []string, labels map[string]string,
	env map[string]struct{}, hc *dkc.HostConfig) (string, error) {
	if err := dk.Pull(image); err != nil {
		return "", err
	}

	envList := make([]string, len(env))
	for k := range env {
		envList = append(envList, k)
	}

	container, err := dk.CreateContainer(dkc.CreateContainerOptions{
		Name: name,
		Config: &dkc.Config{
			Image:  string(image),
			Cmd:    args,
			Labels: labels,
			Env:    envList},
		HostConfig: hc,
	})
	if err != nil {
		return "", err
	}

	return container.ID, nil
}

func (dk docker) getID(name string) (string, error) {
	containers, err := dk.list(nil, true)
	if err != nil {
		return "", err
	}

	name = "/" + name
	for _, c := range containers {
		if name == c.Name {
			return c.ID, nil
		}
	}

	return "", errNoSuchContainer
}

// UserLabel returns the supplied label tagged with the user prefix.
func UserLabel(label string) string {
	return userLabelPrefix + label
}

// IsUserLabel returns whether the supplied label represents a Quilt user label.
func IsUserLabel(label string) bool {
	return strings.HasPrefix(label, userLabelPrefix)
}

// ParseUserLabel returns the supplied label with the user prefix stripped.
func ParseUserLabel(label string) string {
	return strings.TrimPrefix(label, userLabelPrefix)
}

// SystemLabel returns the supplied label tagged with the system prefix.
func SystemLabel(label string) string {
	return systemLabelPrefix + label
}

// Get returns the value contained at the given index
func (cs ContainerSlice) Get(ii int) interface{} {
	return cs[ii]
}

// Len returns the number of items in the slice
func (cs ContainerSlice) Len() int {
	return len(cs)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"strings"

	"github.com/jweslley/procker"
)

var (
	cmdRun = &command{
		desc: "Run a command using your application's environment",
		help: `Usage: procker run COMMAND

Run a command using your application's environment

Available options:`,
		exec: run,
		flag: runFlags}

	// flags
	runFlags   = flag.NewFlagSet("run", flag.ExitOnError)
	runEnvfile = runFlags.String("e", defaultEnvfile,
		"File containing environment variables to be used")
)

func run(args []string) {
	if len(args) == 0 {
		fail("you must specify a command. See 'procker help run'.\n")
	}

	env := parseEnv(*runEnvfile)
	command := strings.Join(args, " ")
	process := &procker.SysProcess{
		Command: command,
		Env:     env,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
	}

	err := process.Start()
	failIf(err)

	process.Wait()
}
<commit_msg>allows to run interactive commands using run<commit_after>package main

import (
	"flag"
	"os"
	"strings"

	"github.com/jweslley/procker"
)

var (
	cmdRun = &command{
		desc: "Run a command using your application's environment",
		help: `Usage: procker run COMMAND

Run a command using your application's environment

Available options:`,
		exec: run,
		flag: runFlags}

	// flags
	runFlags   = flag.NewFlagSet("run", flag.ExitOnError)
	runEnvfile = runFlags.String("e", defaultEnvfile,
		"File containing environment variables to be used")
)

func run(args []string) {
	if len(args) == 0 {
		fail("you must specify a command. See 'procker help run'.\n")
	}

	env := parseEnv(*runEnvfile)
	command := strings.Join(args, " ")
	process := &procker.SysProcess{
		Command: command,
		Env:     env,
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
	}

	err := process.Start()
	failIf(err)

	process.Wait()
}
<|endoftext|>


<commit_before>package bitwarden

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/config/config"
	"golang.org/x/net/html"
)

const cacheTTL = 7 * 24 * time.Hour // 1 week

var iconClient = &http.Client{
	Timeout: 10 * time.Second,
}

// Icon is a simple struct with a content-type and the content of an icon.
type Icon struct {
	Mime string `json:"mime"`
	Body []byte `json:"body"`
}

// GetIcon returns an icon for the given domain.
func GetIcon(domain string) (*Icon, error) {
	if err := validateDomain(domain); err != nil {
		return nil, err
	}

	cache := config.GetConfig().CacheStorage
	key := "bw-icons:" + domain
	if data, ok := cache.Get(key); ok {
		if len(data) == 0 {
			return nil, errors.New("No icon")
		}
		icon := &Icon{}
		if err := json.Unmarshal(data, icon); err != nil {
			return nil, err
		}
		return icon, nil
	}

	icon, err := fetchIcon(domain)
	if err != nil {
		cache.Set(key, nil, cacheTTL)
	} else {
		if data, err := json.Marshal(icon); err == nil {
			cache.Set(key, data, cacheTTL)
		}
	}
	return icon, err
}

func validateDomain(domain string) error {
	if domain == "" || len(domain) > 255 || strings.Contains(domain, "..") {
		return errors.New("Unauthorized domain")
	}

	for _, c := range domain {
		if c == ' ' || !strconv.IsPrint(c) {
			return errors.New("Invalid domain")
		}
	}

	if _, _, err := net.ParseCIDR(domain + "/24"); err == nil {
		return errors.New("IP address are not authorized")
	}

	return nil
}

func fetchIcon(domain string) (*Icon, error) {
	if html, err := getPage(domain); err == nil {
		candidates := getCandidateIcons(domain, html)
		html.Close()
		for _, candidate := range candidates {
			if icon, err := downloadIcon(candidate); err == nil {
				return icon, nil
			}
		}
	}
	return downloadFavicon(domain)
}

func getPage(domain string) (io.ReadCloser, error) {
	req, err := http.NewRequest(http.MethodGet, "https://"+domain, nil)
	if err != nil {
		return nil, err
	}
	res, err := iconClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != http.StatusOK {
		res.Body.Close()
		return nil, errors.New("Not status OK")
	}
	ct := strings.ToLower(res.Header.Get("Content-Type"))
	if !strings.Contains(ct, "text/html") {
		res.Body.Close()
		return nil, errors.New("Not html")
	}
	return res.Body, nil
}

func getCandidateIcons(domain string, r io.Reader) []string {
	tokenizer := html.NewTokenizer(r)
	candidates := make(map[string]int)

	// Consider only the first 1000 tokens, as the candidates icons must be in
	// the <head>, and it avoid reading the whole html page.
	for i := 0; i < 1000; i++ {
		switch tokenizer.Next() {
		case html.ErrorToken:
			// End of the document, we're done
			break
		case html.StartTagToken, html.SelfClosingTagToken:
			t := tokenizer.Token()
			if u, p := getLinkIcon(domain, t); p >= 0 {
				candidates[u] = p
			}
		}
	}

	sorted := make([]string, 0, len(candidates))
	for k := range candidates {
		sorted = append(sorted, k)
	}
	sort.SliceStable(sorted, func(i, j int) bool {
		return candidates[sorted[i]] > candidates[sorted[j]]
	})
	return sorted
}

// getLinkIcon returns the href and the priority for the link.
// -1 means that it is not a suitable icon link.
// Higher priority is better.
func getLinkIcon(domain string, t html.Token) (string, int) {
	if strings.ToLower(t.Data) != "link" {
		return "", -1
	}

	isIcon := false
	href := ""
	priority := 100
	for _, attr := range t.Attr {
		switch strings.ToLower(attr.Key) {
		case "rel":
			vals := strings.Split(strings.ToLower(attr.Val), " ")
			for _, val := range vals {
				if val == "icon" || val == "apple-touch-icon" {
					isIcon = true
					if val == "icon" {
						priority += 10
					}
				}
			}

		case "href":
			href = attr.Val
			if strings.HasSuffix(href, ".png") {
				priority += 2
			}

		case "sizes":
			w, h := parseSizes(attr.Val)
			if w != h {
				priority -= 100
			} else if w == 32 {
				priority += 400
			} else if w == 64 {
				priority += 300
			} else if w >= 24 && w <= 128 {
				priority += 200
			} else if w == 16 {
				priority += 100
			}
		}
	}

	if !isIcon || href == "" {
		return "", -1
	}
	if !strings.Contains(href, "://") {
		href = strings.TrimPrefix(href, "./")
		href = strings.TrimPrefix(href, "/")
		href = "https://" + domain + "/" + href
	}
	return href, priority
}

func parseSizes(val string) (int, int) {
	parts := strings.Split(val, "x")
	if len(parts) != 2 {
		return 0, 0
	}
	w, err := strconv.Atoi(parts[0])
	if err != nil {
		return 0, 0
	}
	h, err := strconv.Atoi(parts[1])
	if err != nil {
		return 0, 0
	}
	return w, h
}

func downloadFavicon(domain string) (*Icon, error) {
	icon, err := downloadIcon("https://" + domain + "/favicon.ico")
	if err == nil {
		return icon, nil
	}
	// Try again
	time.Sleep(1 * time.Second)
	return downloadIcon("https://" + domain + "/favicon.ico")
}

func downloadIcon(u string) (*Icon, error) {
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return nil, err
	}
	res, err := iconClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return nil, errors.New("Not status OK")
	}
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	ico := Icon{
		Mime: res.Header.Get("Content-Type"),
		Body: b,
	}
	if strings.Split(ico.Mime, "/")[0] != "image" {
		return nil, errors.New("Invalid mime-type")
	}
	return &ico, nil
}
<commit_msg>Limit the size of bitwarden icons to 100kb<commit_after>package bitwarden

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/config/config"
	"golang.org/x/net/html"
)

const (
	cacheTTL = 7 * 24 * time.Hour // 1 week
	maxSize  = 100000             // 100kb
)

var iconClient = &http.Client{
	Timeout: 10 * time.Second,
}

// Icon is a simple struct with a content-type and the content of an icon.
type Icon struct {
	Mime string `json:"mime"`
	Body []byte `json:"body"`
}

// GetIcon returns an icon for the given domain.
func GetIcon(domain string) (*Icon, error) {
	if err := validateDomain(domain); err != nil {
		return nil, err
	}

	cache := config.GetConfig().CacheStorage
	key := "bw-icons:" + domain
	if data, ok := cache.Get(key); ok {
		if len(data) == 0 {
			return nil, errors.New("No icon")
		}
		icon := &Icon{}
		if err := json.Unmarshal(data, icon); err != nil {
			return nil, err
		}
		return icon, nil
	}

	icon, err := fetchIcon(domain)
	if err != nil {
		cache.Set(key, nil, cacheTTL)
	} else {
		if data, err := json.Marshal(icon); err == nil {
			cache.Set(key, data, cacheTTL)
		}
	}
	return icon, err
}

func validateDomain(domain string) error {
	if domain == "" || len(domain) > 255 || strings.Contains(domain, "..") {
		return errors.New("Unauthorized domain")
	}

	for _, c := range domain {
		if c == ' ' || !strconv.IsPrint(c) {
			return errors.New("Invalid domain")
		}
	}

	if _, _, err := net.ParseCIDR(domain + "/24"); err == nil {
		return errors.New("IP address are not authorized")
	}

	return nil
}

func fetchIcon(domain string) (*Icon, error) {
	if html, err := getPage(domain); err == nil {
		candidates := getCandidateIcons(domain, html)
		html.Close()
		for _, candidate := range candidates {
			if icon, err := downloadIcon(candidate); err == nil {
				return icon, nil
			}
		}
	}
	return downloadFavicon(domain)
}

func getPage(domain string) (io.ReadCloser, error) {
	req, err := http.NewRequest(http.MethodGet, "https://"+domain, nil)
	if err != nil {
		return nil, err
	}
	res, err := iconClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != http.StatusOK {
		res.Body.Close()
		return nil, errors.New("Not status OK")
	}
	ct := strings.ToLower(res.Header.Get("Content-Type"))
	if !strings.Contains(ct, "text/html") {
		res.Body.Close()
		return nil, errors.New("Not html")
	}
	return res.Body, nil
}

func getCandidateIcons(domain string, r io.Reader) []string {
	tokenizer := html.NewTokenizer(r)
	candidates := make(map[string]int)

	// Consider only the first 1000 tokens, as the candidates icons must be in
	// the <head>, and it avoid reading the whole html page.
	for i := 0; i < 1000; i++ {
		switch tokenizer.Next() {
		case html.ErrorToken:
			// End of the document, we're done
			break
		case html.StartTagToken, html.SelfClosingTagToken:
			t := tokenizer.Token()
			if u, p := getLinkIcon(domain, t); p >= 0 {
				candidates[u] = p
			}
		}
	}

	sorted := make([]string, 0, len(candidates))
	for k := range candidates {
		sorted = append(sorted, k)
	}
	sort.SliceStable(sorted, func(i, j int) bool {
		return candidates[sorted[i]] > candidates[sorted[j]]
	})
	return sorted
}

// getLinkIcon returns the href and the priority for the link.
// -1 means that it is not a suitable icon link.
// Higher priority is better.
func getLinkIcon(domain string, t html.Token) (string, int) {
	if strings.ToLower(t.Data) != "link" {
		return "", -1
	}

	isIcon := false
	href := ""
	priority := 100
	for _, attr := range t.Attr {
		switch strings.ToLower(attr.Key) {
		case "rel":
			vals := strings.Split(strings.ToLower(attr.Val), " ")
			for _, val := range vals {
				if val == "icon" || val == "apple-touch-icon" {
					isIcon = true
					if val == "icon" {
						priority += 10
					}
				}
			}

		case "href":
			href = attr.Val
			if strings.HasSuffix(href, ".png") {
				priority += 2
			}

		case "sizes":
			w, h := parseSizes(attr.Val)
			if w != h {
				priority -= 100
			} else if w == 32 {
				priority += 400
			} else if w == 64 {
				priority += 300
			} else if w >= 24 && w <= 128 {
				priority += 200
			} else if w == 16 {
				priority += 100
			}
		}
	}

	if !isIcon || href == "" {
		return "", -1
	}
	if !strings.Contains(href, "://") {
		href = strings.TrimPrefix(href, "./")
		href = strings.TrimPrefix(href, "/")
		href = "https://" + domain + "/" + href
	}
	return href, priority
}

func parseSizes(val string) (int, int) {
	parts := strings.Split(val, "x")
	if len(parts) != 2 {
		return 0, 0
	}
	w, err := strconv.Atoi(parts[0])
	if err != nil {
		return 0, 0
	}
	h, err := strconv.Atoi(parts[1])
	if err != nil {
		return 0, 0
	}
	return w, h
}

func downloadFavicon(domain string) (*Icon, error) {
	icon, err := downloadIcon("https://" + domain + "/favicon.ico")
	if err == nil {
		return icon, nil
	}
	// Try again
	time.Sleep(1 * time.Second)
	return downloadIcon("https://" + domain + "/favicon.ico")
}

func downloadIcon(u string) (*Icon, error) {
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return nil, err
	}
	res, err := iconClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return nil, errors.New("Not status OK")
	}
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	if len(b) > maxSize {
		return nil, errors.New("Max size exceeded")
	}
	ico := Icon{
		Mime: res.Header.Get("Content-Type"),
		Body: b,
	}
	if strings.Split(ico.Mime, "/")[0] != "image" {
		return nil, errors.New("Invalid mime-type")
	}
	return &ico, nil
}
<|endoftext|>


<commit_before>package model

type UpdateManager struct {
	ClockConsumer
	UUIDConsumer

	transactionProvider TransactionProvider
}

func NewUpdateManager(transactionProvider TransactionProvider) func(UpdateEvent) {
	manager := &UpdateManager{transactionProvider: transactionProvider}
	return manager.Update
}

func (manager *UpdateManager) Update(event UpdateEvent) {
	switch event.EventKind() {
	case STOP_AREA_EVENT:
		manager.updateStopArea(event.(*StopAreaUpdateEvent))
	case LINE_EVENT:
		manager.updateLine(event.(*LineUpdateEvent))
	}
}

func (manager *UpdateManager) updateStopArea(event *StopAreaUpdateEvent) {
	tx := manager.transactionProvider.NewTransaction()

	stopArea, found := tx.Model().StopAreas().FindByObjectId(event.ObjectId)
	if !found {
		stopArea = tx.Model().StopAreas().New()

		stopArea.SetObjectID(event.ObjectId)
		stopArea.CollectGeneralMessages = true
	}

	stopArea.Name = event.Name
	stopArea.CollectedAlways = event.CollectedAlways
	stopArea.Longitude = event.Longitude
	stopArea.Latitude = event.Latitude

	if stopArea.ParentId == "" && event.ParentObjectId.Value() != "" {
		parentSA, _ := tx.Model().StopAreas().FindByObjectId(event.ParentObjectId)
		stopArea.ParentId = parentSA.Id()
	}

	stopArea.Updated(manager.Clock().Now())

	tx.Model().StopAreas().Save(&stopArea)
	tx.Commit()
	tx.Close()
}

func (manager *UpdateManager) updateLine(event *LineUpdateEvent) {
	tx := manager.transactionProvider.NewTransaction()

	line, ok := tx.Model().Lines().FindByObjectId(event.ObjectId)
	if ok {
		line = tx.Model().Lines().New()

		line.SetObjectID(event.ObjectId)
		line.SetObjectID(NewObjectID("_default", event.ObjectId.HashValue()))
	}

	line.Name = event.Name
	line.CollectGeneralMessages = true
	line.SetOrigin(event.Origin)

	line.Updated(manager.Clock().Now())

	tx.Model().Lines().Save(&line)
	tx.Commit()
	tx.Close()
}
<commit_msg>Fixes line creation in UpdateManager#updateLine<commit_after>package model

type UpdateManager struct {
	ClockConsumer
	UUIDConsumer

	transactionProvider TransactionProvider
}

func NewUpdateManager(transactionProvider TransactionProvider) func(UpdateEvent) {
	manager := &UpdateManager{transactionProvider: transactionProvider}
	return manager.Update
}

func (manager *UpdateManager) Update(event UpdateEvent) {
	switch event.EventKind() {
	case STOP_AREA_EVENT:
		manager.updateStopArea(event.(*StopAreaUpdateEvent))
	case LINE_EVENT:
		manager.updateLine(event.(*LineUpdateEvent))
	}
}

func (manager *UpdateManager) updateStopArea(event *StopAreaUpdateEvent) {
	tx := manager.transactionProvider.NewTransaction()

	stopArea, found := tx.Model().StopAreas().FindByObjectId(event.ObjectId)
	if !found {
		stopArea = tx.Model().StopAreas().New()

		stopArea.SetObjectID(event.ObjectId)
		stopArea.CollectGeneralMessages = true
	}

	stopArea.Name = event.Name
	stopArea.CollectedAlways = event.CollectedAlways
	stopArea.Longitude = event.Longitude
	stopArea.Latitude = event.Latitude

	if stopArea.ParentId == "" && event.ParentObjectId.Value() != "" {
		parentSA, _ := tx.Model().StopAreas().FindByObjectId(event.ParentObjectId)
		stopArea.ParentId = parentSA.Id()
	}

	stopArea.Updated(manager.Clock().Now())

	tx.Model().StopAreas().Save(&stopArea)
	tx.Commit()
	tx.Close()
}

func (manager *UpdateManager) updateLine(event *LineUpdateEvent) {
	tx := manager.transactionProvider.NewTransaction()

	line, found := tx.Model().Lines().FindByObjectId(event.ObjectId)
	if !found {
		line = tx.Model().Lines().New()

		line.SetObjectID(event.ObjectId)
		line.SetObjectID(NewObjectID("_default", event.ObjectId.HashValue()))
	}

	line.Name = event.Name
	line.CollectGeneralMessages = true
	line.SetOrigin(event.Origin)

	line.Updated(manager.Clock().Now())

	tx.Model().Lines().Save(&line)
	tx.Commit()
	tx.Close()
}
<|endoftext|>