<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/gravitational/teleport/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/gravitational/teleport/service"
	"github.com/gravitational/teleport/utils"
)

func main() {
	cfg := service.Config{}

	flag.StringVar(
		&cfg.Log, "log", "console",
		"log output, currently 'console' or 'syslog'")

	flag.StringVar(
		&cfg.LogSeverity, "logSeverity", "WARN",
		"log severity, INFO or WARN or ERROR")

	flag.StringVar(
		&cfg.DataDir, "dataDir", "",
		"path to directory where teleport stores it's state")

	flag.StringVar(
		&cfg.FQDN, "fqdn", "",
		"fqdn of this server, e.g. node1.example.com, should be unique")

	flag.Var(utils.NewNetAddrList(&cfg.AuthServers),
		"authServer", "list of SSH auth server endpoints")

	// SSH specific role options
	flag.BoolVar(&cfg.SSH.Enabled, "ssh", false,
		"enable SSH server endpoint")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "localhost:33001",
			}, &cfg.SSH.Addr),
		"sshAddr", "SSH endpoint listening address")

	flag.StringVar(
		&cfg.SSH.Shell, "sshShell", "/bin/bash",
		"path to shell to launch for interactive sessions")

	// Auth server role options
	flag.BoolVar(&cfg.Auth.Enabled, "auth", false,
		"enable Authentication server endpoint")

	flag.StringVar(
		&cfg.Auth.Backend, "authBackend", "etcd",
		"auth backend type, 'etcd' or 'bolt'")

	flag.StringVar(
		&cfg.Auth.BackendConfig, "authBackendConfig", "",
		"auth backend-specific configuration string")

	flag.StringVar(
		&cfg.Auth.EventBackend, "authEventBackend", "bolt",
		"event backend type, currently only 'bolt'")

	flag.StringVar(
		&cfg.Auth.EventBackendConfig, "authEventBackendConfig", "",
		"event backend-specific configuration string")

	flag.StringVar(
		&cfg.Auth.RecordBackend, "authRecordBackend", "bolt",
		"event backend type, currently only 'bolt'")

	flag.StringVar(
		&cfg.Auth.RecordBackendConfig, "authRecordBackendConfig", "",
		"event backend-specific configuration string")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "unix",
				Addr:    "/tmp/teleport.auth.sock",
			}, &cfg.Auth.HTTPAddr),
		"authHTTPAddr", "Auth Server HTTP API listening address")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "localhost:33000",
			}, &cfg.Auth.SSHAddr),
		"authSSHAddr", "Auth Server SSH tunnel API listening address")

	flag.StringVar(
		&cfg.Auth.Domain, "authDomain", "",
		"authentication server domain name, e.g. example.com")

	flag.StringVar(
		&cfg.SSH.Token, "sshToken", "",
		"one time provisioning token for SSH node to register with authority")

	// CP role options
	flag.BoolVar(&cfg.CP.Enabled, "cp", false,
		"enable Control Panel endpoint")

	flag.StringVar(
		&cfg.CP.AssetsDir, "cpAssetsDir", "",
		"path to control panel assets")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "localhost:33002",
			}, &cfg.CP.Addr),
		"cpAddr", "CP server web listening address")

	flag.StringVar(
		&cfg.CP.Domain, "cpDomain", "",
		"control panel domain to serve, e.g. example.com")

	// Outbound tunnel role options
	flag.BoolVar(&cfg.Tun.Enabled, "tun", false, "enable outbound tunnel")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "localhost:33006",
			}, &cfg.Tun.SrvAddr),
		"tunSrvAddr", "tun agent dial address")

	flag.StringVar(
		&cfg.Tun.Token, "tunToken", "",
		"one time provisioning token for tun agent to register with authority")

	flag.Parse()

	// some variables can be set via environment variables
	// TODO(klizhentas) - implement
	if os.Getenv("TELEPORT_SSH_TOKEN") != "" {
		cfg.SSH.Token = os.Getenv("TELEPORT_SSH_TOKEN")
	}

	if os.Getenv("TELEPORT_TUN_TOKEN") != "" {
		cfg.Tun.Token = os.Getenv("TELEPORT_TUN_TOKEN")
	}

	srv, err := service.NewTeleport(cfg)
	if err != nil {
		fmt.Printf("error starting teleport: %v\n", err)
		return
	}

	if err := srv.Start(); err != nil {
		log.Errorf("teleport failed to start with error: %v", err)
		return
	}
	srv.Wait()
}
<commit_msg>remove error-prone localhost with the IP address<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/gravitational/teleport/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/gravitational/teleport/service"
	"github.com/gravitational/teleport/utils"
)

func main() {
	cfg := service.Config{}

	flag.StringVar(
		&cfg.Log, "log", "console",
		"log output, currently 'console' or 'syslog'")

	flag.StringVar(
		&cfg.LogSeverity, "logSeverity", "WARN",
		"log severity, INFO or WARN or ERROR")

	flag.StringVar(
		&cfg.DataDir, "dataDir", "",
		"path to directory where teleport stores it's state")

	flag.StringVar(
		&cfg.FQDN, "fqdn", "",
		"fqdn of this server, e.g. node1.example.com, should be unique")

	flag.Var(utils.NewNetAddrList(&cfg.AuthServers),
		"authServer", "list of SSH auth server endpoints")

	// SSH specific role options
	flag.BoolVar(&cfg.SSH.Enabled, "ssh", false,
		"enable SSH server endpoint")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "127.0.0.1:33001",
			}, &cfg.SSH.Addr),
		"sshAddr", "SSH endpoint listening address")

	flag.StringVar(
		&cfg.SSH.Shell, "sshShell", "/bin/bash",
		"path to shell to launch for interactive sessions")

	// Auth server role options
	flag.BoolVar(&cfg.Auth.Enabled, "auth", false,
		"enable Authentication server endpoint")

	flag.StringVar(
		&cfg.Auth.Backend, "authBackend", "etcd",
		"auth backend type, 'etcd' or 'bolt'")

	flag.StringVar(
		&cfg.Auth.BackendConfig, "authBackendConfig", "",
		"auth backend-specific configuration string")

	flag.StringVar(
		&cfg.Auth.EventBackend, "authEventBackend", "bolt",
		"event backend type, currently only 'bolt'")

	flag.StringVar(
		&cfg.Auth.EventBackendConfig, "authEventBackendConfig", "",
		"event backend-specific configuration string")

	flag.StringVar(
		&cfg.Auth.RecordBackend, "authRecordBackend", "bolt",
		"event backend type, currently only 'bolt'")

	flag.StringVar(
		&cfg.Auth.RecordBackendConfig, "authRecordBackendConfig", "",
		"event backend-specific configuration string")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "unix",
				Addr:    "/tmp/teleport.auth.sock",
			}, &cfg.Auth.HTTPAddr),
		"authHTTPAddr", "Auth Server HTTP API listening address")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "127.0.0.1:33000",
			}, &cfg.Auth.SSHAddr),
		"authSSHAddr", "Auth Server SSH tunnel API listening address")

	flag.StringVar(
		&cfg.Auth.Domain, "authDomain", "",
		"authentication server domain name, e.g. example.com")

	flag.StringVar(
		&cfg.SSH.Token, "sshToken", "",
		"one time provisioning token for SSH node to register with authority")

	// CP role options
	flag.BoolVar(&cfg.CP.Enabled, "cp", false,
		"enable Control Panel endpoint")

	flag.StringVar(
		&cfg.CP.AssetsDir, "cpAssetsDir", "",
		"path to control panel assets")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "127.0.0.1:33002",
			}, &cfg.CP.Addr),
		"cpAddr", "CP server web listening address")

	flag.StringVar(
		&cfg.CP.Domain, "cpDomain", "",
		"control panel domain to serve, e.g. example.com")

	// Outbound tunnel role options
	flag.BoolVar(&cfg.Tun.Enabled, "tun", false, "enable outbound tunnel")

	flag.Var(
		utils.NewNetAddrVal(
			utils.NetAddr{
				Network: "tcp",
				Addr:    "127.0.0.1:33006",
			}, &cfg.Tun.SrvAddr),
		"tunSrvAddr", "tun agent dial address")

	flag.StringVar(
		&cfg.Tun.Token, "tunToken", "",
		"one time provisioning token for tun agent to register with authority")

	flag.Parse()

	// some variables can be set via environment variables
	// TODO(klizhentas) - implement
	if os.Getenv("TELEPORT_SSH_TOKEN") != "" {
		cfg.SSH.Token = os.Getenv("TELEPORT_SSH_TOKEN")
	}

	if os.Getenv("TELEPORT_TUN_TOKEN") != "" {
		cfg.Tun.Token = os.Getenv("TELEPORT_TUN_TOKEN")
	}

	srv, err := service.NewTeleport(cfg)
	if err != nil {
		fmt.Printf("error starting teleport: %v\n", err)
		return
	}

	if err := srv.Start(); err != nil {
		log.Errorf("teleport failed to start with error: %v", err)
		return
	}
	srv.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	// "bufio"
	"bufio"
	"fmt"
	"github.com/dimfeld/gocache"
	"github.com/dimfeld/goconfig"
	"github.com/dimfeld/httppath"
	"github.com/dimfeld/httptreemux"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"sync"
	"time"
)

var (
	logger      *log.Logger
	debugLogger *log.Logger
	debugMode   bool
	config      *Config
)

func debugf(format string, args ...interface{}) {
	if debugMode {
		debugLogger.Printf(format, args...)
	}
}

func debug(args ...interface{}) {
	if debugMode {
		debugLogger.Println(args...)
	}
}
func catchSIGINT(f func(), quit bool) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			logger.Println("SIGINT received...")
			f()
			if quit {
				os.Exit(1)
			}
		}
	}()
}

type GlobalData struct {
	*sync.RWMutex

	// General cache
	cache    gocache.Cache
	memCache gocache.Cache

	archive   ArchiveSpecList
	templates *template.Template
}

type Config struct {
	IndexPosts          int
	PostsDir            string
	DataDir             string
	CacheDir            string
	TagsPath            string
	TagsPageReverseSort bool
	LogFile             string
	LogPrefix           string
	DebugMode           bool
	Domain              string
	Port                int
}

type simpleBlogHandler func(*GlobalData, http.ResponseWriter, *http.Request, map[string]string)

func handlerWrapper(handler simpleBlogHandler, globalData *GlobalData) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, urlParams map[string]string) {
		logger.Printf("%s %s", r.Method, r.RequestURI)
		startTime := time.Now()
		handler(globalData, w, r, urlParams)
		endTime := time.Now()
		duration := endTime.Sub(startTime)
		logger.Printf("   Handled in %d us", duration/time.Microsecond)
	}
}

func fileWrapper(filename string, handler httptreemux.HandlerFunc) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, urlParams map[string]string) {
		urlParams["file"] = filename
		handler(w, r, urlParams)
	}
}

func filePrefixWrapper(prefix string, handler httptreemux.HandlerFunc) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, urlParams map[string]string) {
		urlParams["file"] = filepath.Join(prefix, httppath.Clean(urlParams["file"]))
		handler(w, r, urlParams)
	}
}

func isDirectory(dirPath string) bool {
	stat, err := os.Stat(dirPath)
	if err != nil || !stat.IsDir() {
		return false
	}
	return true
}

func setup() (router *httptreemux.TreeMux, cleanup func()) {
	config = &Config{Port: 80}
	confFile := os.Getenv("SIMPLEBLOG_CONFFILE")
	if len(os.Args) > 1 {
		confFile = os.Args[1]
	}

	if confFile == "" {
		confFile = "simpleblog.conf"
	}

	var confReader io.Reader = os.Stdin
	var err error
	if confFile != "-" {
		// Load from stdin
		confReader, err = os.Open(confFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error loading config: %s\n", err)
			os.Exit(1)
		}
	}

	err = goconfig.Load(config, confReader, "SIMPLEBLOG")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading config: %s\n", err)
		os.Exit(1)
	}

	debugMode = config.DebugMode
	if config.Port != 80 {
		config.Domain = fmt.Sprintf("%s:%d", config.Domain, config.Port)
	}

	logFile, err := os.OpenFile(config.LogFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0640)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not open log file %s\n", config.LogFile)
		os.Exit(1)
	}

	logBuffer := bufio.NewWriter(logFile)

	closer := func() {
		logger.Println("Shutting down...")
		logBuffer.Flush()
		logFile.Sync()
		logFile.Close()
	}

	var logWriter io.Writer = logBuffer
	if debugMode {
		// In debug mode, use unbuffered logging so that they come out right away.
		logWriter = logFile
	}

	logger = log.New(logWriter, config.LogPrefix, log.LstdFlags)
	debugLogger = log.New(logWriter, "DEBUG ", log.LstdFlags)
	logger.Println("Starting...")

	diskCache, err := gocache.NewDiskCache(config.CacheDir)
	if err != nil {
		logger.Fatal("Could not create disk cache in", config.CacheDir)
	}

	if !isDirectory(config.DataDir) {
		logger.Fatal("Could not find data directory", config.DataDir)
	}

	if !isDirectory(config.PostsDir) {
		logger.Fatal("Could not find posts directory", config.PostsDir)
	}

	if !isDirectory(filepath.Join(config.DataDir, "assets")) {
		logger.Fatal("Could not find assets directory", filepath.Join(config.DataDir, "assets"))
	}

	if !isDirectory(filepath.Join(config.DataDir, "images")) {
		logger.Fatal("Could not find assets directory", filepath.Join(config.DataDir, "images"))
	}

	// Large memory cache uses 64 MiB at most, with the largest object being 8 MiB.
	largeObjectLimit := 8 * 1024 * 1024
	largeMemCache := gocache.NewMemoryCache(64*1024*1024, largeObjectLimit)
	// Small memory cache uses 16 MiB at most, with the largest object being 16KiB.
	smallObjectLimit := 16 * 1024
	smallMemCache := gocache.NewMemoryCache(16*1024*1024, smallObjectLimit)

	// Create a split cache, putting all objects smaller than 16 KiB into the small cache.
	// This split cache prevents a few large objects from evicting all the smaller objects.
	memCache := gocache.NewSplitSize(
		gocache.SplitSizeChild{MaxSize: smallObjectLimit, Cache: smallMemCache},
		gocache.SplitSizeChild{MaxSize: largeObjectLimit, Cache: largeMemCache})

	multiLevelCache := gocache.MultiLevel{0: memCache, 1: diskCache}

	templates, err := createTemplates()
	if err != nil {
		logger.Fatal("Error parsing template:", err.Error())
	}

	os.Remove(config.TagsPath)
	globalData := &GlobalData{
		RWMutex:   &sync.RWMutex{},
		cache:     multiLevelCache,
		memCache:  memCache,
		templates: templates,
	}

	archive, err := NewArchiveSpecList(config.PostsDir)
	if err != nil {
		logger.Fatal("Could not create archive list: ", err)
	}
	globalData.archive = archive

	go watchFiles(globalData)

	router = httptreemux.New()
	router.PanicHandler = httptreemux.ShowErrorsPanicHandler

	router.GET("/", handlerWrapper(indexHandler, globalData))
	router.GET("/:year/:month/", handlerWrapper(archiveHandler, globalData))
	router.GET("/:year/:month/:post", handlerWrapper(postHandler, globalData))

	router.GET("/images/*file", filePrefixWrapper("images",
		handlerWrapper(staticNoCompressHandler, globalData)))
	router.GET("/assets/*file", filePrefixWrapper("assets",
		handlerWrapper(staticCompressHandler, globalData)))

	router.GET("/tag/:tag", handlerWrapper(tagHandler, globalData))
	// No pagination yet.
	//router.GET("/tag/:tag/:page", handlerWrapper(tagHandler, globalData))

	router.GET("/:page", handlerWrapper(pageHandler, globalData))
	router.GET("/favicon.ico", fileWrapper("assets/favicon.ico",
		handlerWrapper(staticCompressHandler, globalData)))
	router.GET("/feed", handlerWrapper(atomHandler, globalData))

	return router, closer
}

func main() {
	router, closer := setup()

	catchSIGINT(closer, true)
	defer closer()

	http.ListenAndServe(fmt.Sprintf(":%d", config.Port), router)
}
<commit_msg>Configurable cache size<commit_after>package main

import (
	// "bufio"
	"bufio"
	"fmt"
	"github.com/dimfeld/gocache"
	"github.com/dimfeld/goconfig"
	"github.com/dimfeld/httppath"
	"github.com/dimfeld/httptreemux"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"sync"
	"time"
)

var (
	logger      *log.Logger
	debugLogger *log.Logger
	debugMode   bool
	config      *Config
)

func debugf(format string, args ...interface{}) {
	if debugMode {
		debugLogger.Printf(format, args...)
	}
}

func debug(args ...interface{}) {
	if debugMode {
		debugLogger.Println(args...)
	}
}
func catchSIGINT(f func(), quit bool) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			logger.Println("SIGINT received...")
			f()
			if quit {
				os.Exit(1)
			}
		}
	}()
}

type GlobalData struct {
	*sync.RWMutex

	// General cache
	cache    gocache.Cache
	memCache gocache.Cache

	archive   ArchiveSpecList
	templates *template.Template
}

type Config struct {
	IndexPosts          int
	PostsDir            string
	DataDir             string
	CacheDir            string
	TagsPath            string
	TagsPageReverseSort bool
	LogFile             string
	LogPrefix           string
	DebugMode           bool
	Domain              string
	Port                int

	LargeMemCacheLimit       int
	SmallMemCacheLimit       int
	LargeMemCacheObjectLimit int
	SmallMemCacheObjectLimit int
}

type simpleBlogHandler func(*GlobalData, http.ResponseWriter, *http.Request, map[string]string)

func handlerWrapper(handler simpleBlogHandler, globalData *GlobalData) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, urlParams map[string]string) {
		logger.Printf("%s %s", r.Method, r.RequestURI)
		startTime := time.Now()
		handler(globalData, w, r, urlParams)
		endTime := time.Now()
		duration := endTime.Sub(startTime)
		logger.Printf("   Handled in %d us", duration/time.Microsecond)
	}
}

func fileWrapper(filename string, handler httptreemux.HandlerFunc) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, urlParams map[string]string) {
		urlParams["file"] = filename
		handler(w, r, urlParams)
	}
}

func filePrefixWrapper(prefix string, handler httptreemux.HandlerFunc) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, urlParams map[string]string) {
		urlParams["file"] = filepath.Join(prefix, httppath.Clean(urlParams["file"]))
		handler(w, r, urlParams)
	}
}

func isDirectory(dirPath string) bool {
	stat, err := os.Stat(dirPath)
	if err != nil || !stat.IsDir() {
		return false
	}
	return true
}

func setup() (router *httptreemux.TreeMux, cleanup func()) {
	config = &Config{
		Port: 80,
		// Large memory cache uses 64 MiB at most, with the largest object being 8 MiB.
		LargeMemCacheLimit:       64 * 1024 * 1024,
		LargeMemCacheObjectLimit: 8 * 1024 * 1024,
		// Small memory cache uses 16 MiB at most, with the largest object being 16KiB.
		SmallMemCacheLimit:       16 * 1024 * 1024,
		SmallMemCacheObjectLimit: 16 * 1024,
	}
	confFile := os.Getenv("SIMPLEBLOG_CONFFILE")
	if len(os.Args) > 1 {
		confFile = os.Args[1]
	}

	if confFile == "" {
		confFile = "simpleblog.conf"
	}

	var confReader io.Reader = os.Stdin
	var err error
	if confFile != "-" {
		// Load from stdin
		confReader, err = os.Open(confFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error loading config: %s\n", err)
			os.Exit(1)
		}
	}

	err = goconfig.Load(config, confReader, "SIMPLEBLOG")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading config: %s\n", err)
		os.Exit(1)
	}

	debugMode = config.DebugMode
	if config.Port != 80 {
		config.Domain = fmt.Sprintf("%s:%d", config.Domain, config.Port)
	}

	logFile, err := os.OpenFile(config.LogFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0640)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not open log file %s\n", config.LogFile)
		os.Exit(1)
	}

	logBuffer := bufio.NewWriter(logFile)

	closer := func() {
		logger.Println("Shutting down...")
		logBuffer.Flush()
		logFile.Sync()
		logFile.Close()
	}

	var logWriter io.Writer = logBuffer
	if debugMode {
		// In debug mode, use unbuffered logging so that they come out right away.
		logWriter = logFile
	}

	logger = log.New(logWriter, config.LogPrefix, log.LstdFlags)
	debugLogger = log.New(logWriter, "DEBUG ", log.LstdFlags)
	logger.Println("Starting...")

	diskCache, err := gocache.NewDiskCache(config.CacheDir)
	if err != nil {
		logger.Fatal("Could not create disk cache in", config.CacheDir)
	}

	if !isDirectory(config.DataDir) {
		logger.Fatal("Could not find data directory", config.DataDir)
	}

	if !isDirectory(config.PostsDir) {
		logger.Fatal("Could not find posts directory", config.PostsDir)
	}

	if !isDirectory(filepath.Join(config.DataDir, "assets")) {
		logger.Fatal("Could not find assets directory", filepath.Join(config.DataDir, "assets"))
	}

	if !isDirectory(filepath.Join(config.DataDir, "images")) {
		logger.Fatal("Could not find assets directory", filepath.Join(config.DataDir, "images"))
	}

	largeObjectLimit := config.LargeMemCacheObjectLimit
	largeMemCache := gocache.NewMemoryCache(
		config.LargeMemCacheLimit, largeObjectLimit)

	smallObjectLimit := config.SmallMemCacheObjectLimit
	smallMemCache := gocache.NewMemoryCache(
		config.SmallMemCacheLimit, smallObjectLimit)

	// Create a split cache, putting all objects smaller than 16 KiB into the small cache.
	// This split cache prevents a few large objects from evicting all the smaller objects.
	memCache := gocache.NewSplitSize(
		gocache.SplitSizeChild{MaxSize: smallObjectLimit, Cache: smallMemCache},
		gocache.SplitSizeChild{MaxSize: largeObjectLimit, Cache: largeMemCache})

	multiLevelCache := gocache.MultiLevel{0: memCache, 1: diskCache}

	templates, err := createTemplates()
	if err != nil {
		logger.Fatal("Error parsing template:", err.Error())
	}

	os.Remove(config.TagsPath)
	globalData := &GlobalData{
		RWMutex:   &sync.RWMutex{},
		cache:     multiLevelCache,
		memCache:  memCache,
		templates: templates,
	}

	archive, err := NewArchiveSpecList(config.PostsDir)
	if err != nil {
		logger.Fatal("Could not create archive list: ", err)
	}
	globalData.archive = archive

	go watchFiles(globalData)

	router = httptreemux.New()
	router.PanicHandler = httptreemux.ShowErrorsPanicHandler

	router.GET("/", handlerWrapper(indexHandler, globalData))
	router.GET("/:year/:month/", handlerWrapper(archiveHandler, globalData))
	router.GET("/:year/:month/:post", handlerWrapper(postHandler, globalData))

	router.GET("/images/*file", filePrefixWrapper("images",
		handlerWrapper(staticNoCompressHandler, globalData)))
	router.GET("/assets/*file", filePrefixWrapper("assets",
		handlerWrapper(staticCompressHandler, globalData)))

	router.GET("/tag/:tag", handlerWrapper(tagHandler, globalData))
	// No pagination yet.
	//router.GET("/tag/:tag/:page", handlerWrapper(tagHandler, globalData))

	router.GET("/:page", handlerWrapper(pageHandler, globalData))
	router.GET("/favicon.ico", fileWrapper("assets/favicon.ico",
		handlerWrapper(staticCompressHandler, globalData)))
	router.GET("/feed", handlerWrapper(atomHandler, globalData))

	return router, closer
}

func main() {
	router, closer := setup()

	catchSIGINT(closer, true)
	defer closer()

	http.ListenAndServe(fmt.Sprintf(":%d", config.Port), router)
}
<|endoftext|>


<commit_before>package gash

type SimpleHash struct {
    items [][]KvPair
    capacity int
}

func Create(capacity int) SimpleHash {
    table := SimpleHash{}
    table.capacity = capacity
    table.items = make([][]KvPair, capacity)
    for index := range table.items {
        table.items[index] = []KvPair{}
    }

    return table
}

func (table SimpleHash) Insert(k string, v interface{}) {
    index := Djb2(k) % table.capacity
    item := KvPair{k, v}

    isSet := false
    for searchIndex, pair := range table.items[index] {
        if pair.Key == k {
            table.items[index][searchIndex].Value = v
            isSet = true
            break
        }
    }
    if (!isSet) {
        table.items[index] = append(table.items[index], item)
    }
}

func (table SimpleHash) Find(k string) interface{} {
    index := Djb2(k) % table.capacity
    for _, pair := range table.items[index] {
        if pair.Key == k {
            return pair
        }
    }
    return nil
}

func (table SimpleHash) Remove(k string) {
    index := Djb2(k) % table.capacity
    for searchIndex, pair := range table.items[index] {
        if pair.Key == k {
            table.items[index] = append(table.items[index][:searchIndex],
                                        table.items[index][searchIndex+1:]...)
            break
        }
    }
}
<commit_msg>Add module description<commit_after>package gash

// A simple chained hash.

type SimpleHash struct {
    items [][]KvPair
    capacity int
}

func Create(capacity int) SimpleHash {
    table := SimpleHash{}
    table.capacity = capacity
    table.items = make([][]KvPair, capacity)
    for index := range table.items {
        table.items[index] = []KvPair{}
    }

    return table
}

func (table SimpleHash) Insert(k string, v interface{}) {
    index := Djb2(k) % table.capacity
    item := KvPair{k, v}

    isSet := false
    for searchIndex, pair := range table.items[index] {
        if pair.Key == k {
            table.items[index][searchIndex].Value = v
            isSet = true
            break
        }
    }
    if (!isSet) {
        table.items[index] = append(table.items[index], item)
    }
}

func (table SimpleHash) Find(k string) interface{} {
    index := Djb2(k) % table.capacity
    for _, pair := range table.items[index] {
        if pair.Key == k {
            return pair
        }
    }
    return nil
}

func (table SimpleHash) Remove(k string) {
    index := Djb2(k) % table.capacity
    for searchIndex, pair := range table.items[index] {
        if pair.Key == k {
            table.items[index] = append(table.items[index][:searchIndex],
                                        table.items[index][searchIndex+1:]...)
            break
        }
    }
}
<|endoftext|>


<commit_before>package main

import(
	"math"
	"fmt"
	"errors"
	"math/rand"
	"time"
	"sync"
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
)

type Result struct{
	Person Person
	Workload Workload
}

func (r Result) String()string{
	return fmt.Sprintf("<Result %v %s>", r.Person, r.Workload)
}

type Person struct{
	Vg float64 // Value/grade
	Gt float64 // Grade/time
	La float64 // Look ahead
	Vt float64 // Value/time
	P float64 // Rememberance
	B float64 // Beta
	D float64 // Delta

	Semester Semester // The schedule for the semester
	Assignments []Assignment // The assignment due
	WorkHours map[Assignment]int // The number of hours worked on assignments.
}

func (p Person) String() string{
	return fmt.Sprintf("<Person Vg:%f Gt:%f La:%f Vt:%f P:%f B:%f D:%f",
		p.Vg, p.Gt, p.La, p.Vt, p.P, p.B, p.D)
}

type Assignment struct{
	TotalGrade int // The total grade units this assignment is graded out of.
	DateDue int // The date due in numbers of day into the semester
	Name string
}

func (a Assignment) String() string{
	return fmt.Sprintf("",a.TotalGrade, a.DateDue, a.Name)
}

type Semester struct{
	Days int // The number of days
	Day int // The current day
	Weights map[int]float64 // A mapping of days to the attitude weight.
	Allowed map[int]int // A maping of days to hours available to work.
}

type Work struct{
	Day int
	Hours int
	Assignment Assignment
}

func (w Work) String() string{
	return fmt.Sprintf("<W Day=%d Hours=%d Assignment=%v",w.Day, w.Hours, w.Assignment)
}

func IntFactorial(i int)(int, error){
	if i > 20|| i < 1{return 0, errors.New("Factorial out of range")}
	if i == 1{
		return 1, nil
	}
	res, _ := IntFactorial(i-1)
	return i * res, nil
}

// Represents a possible ammount to work in the future.
type Workload struct{
	Days []Work
	Hours map[Assignment]int
}

func (w Workload) String() string {
	return fmt.Sprintf("<WL days=%v hours=%v>" ,w.Days, w.Hours)
}

// Generate workloads starting on the current day.
func (p *Person) Workloads() []Workload {

	workloads := make([]Workload,1)
	// Create base workload
	workloads[0] = *new(Workload)
	workloads[0].Days = make([]Work, 0)
	workloads[0].Hours = make(map[Assignment]int)
	for _,assignment := range(p.Assignments){
		workloads[0].Hours[assignment] = 0
	}
	// For each day until the look ahead day.
	for i := 0; float64(i)
= (p.Semester.Day + i){
					// For every time that does not put you over the limit.
					for t:=1; basewl.Hours[assignment]+
						t+p.WorkHours[assignment] <=
						assignment.TotalGrade &&
						t <= p.Semester.Allowed[p.Semester.Day + i]; t++{

						wl := *new(Workload)
						day := *new(Work)
						day.Assignment = assignment
						day.Hours = t
						day.Day = i + p.Semester.Day
						wl.Days = append(basewl.Days,day)
						wl.Hours = make(map[Assignment]int)
						for k, v := range(basewl.Hours){
							wl.Hours[k]=v
						}
						wl.Hours[assignment] = basewl.Hours[assignment] + t
						newloads = append(newloads,wl)
					}
				}
			}
			wl := *new(Workload)
			day := *new(Work)
			day.Assignment = Assignment{}
			day.Hours = 0
			day.Day = i + p.Semester.Day
			wl.Days = append(basewl.Days,day)
			wl.Hours = make(map[Assignment]int)
			for k,v := range(basewl.Hours){
				wl.Hours[k]=v
			}
			newloads = append(newloads, wl)
		}

		workloads = newloads
	}
	return workloads
}

// The discount function for this individual
// If d is 0 then the function is 1, otherwise it is B * (D ^ d)
func (p *Person) Discount(d int) float64{
	if d == 0 {
		return 1
	}
	return p.B * math.Pow(p.D, float64(d))
}

// Returns the expected utility of the given work.
func (p Person) WorkUtility(work Work) float64{
	utility := (float64(work.Hours) * p.Vg *p.Gt *p.B * p.Discount(work.Assignment.DateDue - p.Semester.Day))
	cost := (float64(work.Hours) * p.Semester.Weights[work.Day] * p.Discount(work.Day - p.Semester.Day))
	return utility -cost
}

// Returns the expected utility of the given workload evaluated on the current day.
func (p Person) Utility(workload Workload) float64{
	utility := float64(0)
	for _, work := range(workload.Days){
		if work.Assignment.TotalGrade != 0{
			utility += p.WorkUtility(work)
		}
	}
	return utility
}

func (p Person) Simulate(results chan Result){
	var globalWorkload Workload
	globalWorkload.Hours = make(map[Assignment]int)

	for p.Semester.Day
 maxUtility{
				maxUtility = utility
				choices = make([]Workload, 0)
				choices = append(choices, workload)
			}
		}

		var choice Workload
		if len(choices) > 1{
			choice = choices[rand.Intn(len(choices))]
		}else{
			choice = choices[0]
		}
		if len(choice.Days) > 0 {
			globalWorkload.Days = append(globalWorkload.Days, choice.Days[0])
			p.WorkHours[choice.Days[0].Assignment] += choice.Days[0].Hours
			globalWorkload.Hours[choice.Days[0].Assignment] += choice.Days[0].Hours
		}
	}

	results <- Result{Person:p, Workload:globalWorkload}
}

func main(){
	fmt.Println("Simulating people... This is most likely not going to work.")

	people := make([]Person, 0)
	rand.Seed(time.Now().Unix())

	assignments := make([]Assignment, 0)
	assignments = append(assignments, Assignment{DateDue:30,TotalGrade: 20, Name:"Assignment 1"})
	assignments = append(assignments, Assignment{DateDue:30,TotalGrade: 20, Name:"Assignment 2"})
	numPeople := 100
	numDays := 30
	for p:=0; p< numPeople; p++{
		// Create person
		Vg := rand.NormFloat64() * 1 + 10
		Gt := rand.NormFloat64() * 0.25 + 2
		B := rand.NormFloat64() * 0.2 + 0.5
		D := rand.NormFloat64() * 0.1 + 0.8
		if D > 1.0{
			D = 1
		}
		person := Person{Vg:Vg, Gt:Gt, La: 7, Vt: 1, P:1, B:B, D:D, WorkHours:make(map[Assignment]int)}
		semester := Semester{Days: 30,Day:0,Weights:make(map[int]float64), Allowed:make(map[int]int)}
		person.Semester = semester
		for i := 0; i <= 30; i++{
			person.Semester.Weights[i] = float64(i)*0.025
			person.Semester.Allowed[i] = 3
			if i > 25 {
				person.Semester.Allowed[i] = 2
				person.Semester.Weights[i] += float64(i)*0.025
			}
		}

		person.Assignments = assignments

		people = append(people, person)
	}

	results := make(chan Result)
	inChan := make(chan Person)

	numRoutines := 10
	var wg sync.WaitGroup
	for i := 0; i < numRoutines; i++{
		wg.Add(1)
		go func(people chan Person, results chan Result){
			for person := range(people){
				person.Simulate(results)
			}
			wg.Done()
		}(inChan, results)
	}

	//Wait for the channel simulations to finish then close the results chan
	go func(){
		wg.Wait()
		close(results)
	}()

	// Keep placing people in the channel
	go func(people []Person, inChan chan Person){
		for _,person := range people{
			inChan <- person
		}
		close(inChan)
	}(people, inChan)

	frequency := make(map[Assignment]plotter.XYs, len(assignments))
	grades := make(map[Assignment]int)

	for _, Assignment := range assignments{
		frequency[Assignment] = make(plotter.XYs,numDays+1)
		for i := 0;i <= numDays; i++{
			frequency[Assignment][i].X = float64(i)
			frequency[Assignment][i].Y = 0
		}
	}
	for res := range results{
		res := res
		fmt.Printf("Results: %v \n\t%v\n", res.Person, res.Workload)
		for n,day := range res.Workload.Days{
			if day.Assignment.TotalGrade != 0{
				frequency[day.Assignment][n].Y += float64(day.Hours)
			}
		}
		for Assignment, Grade := range res.Workload.Hours{
			grades[Assignment] = grades[Assignment] + Grade
		}
	}

	p, err := plot.New()
	if err != nil {
			panic(err)
	}
	p.Title.Text = "Frequency"
	p.Y.Label.Text = "Heights"
	p.Add(plotter.NewGrid())

	for assignment,hour := range grades{
		fmt.Printf("Average %s: %f\n", assignment, float64(hour)/float64(numPeople))
	}

	i := 0
	for assignment, xys := range frequency{
		lpline,lppoints, err := plotter.NewLinePoints(xys)
		if err != nil{
			panic(err)
		}
		lpline.Color = plotutil.Color(i)
		lpline.Dashes = plotutil.Dashes(i)
		lppoints.Color = plotutil.Color(i)
		lppoints.Shape = plotutil.Shape(i)
		i++
		p.Add(lpline)
		p.Add(lppoints)
		p.Legend.Add(assignment.Name, lpline, lppoints)
	}
	p.Legend.Top = true
	p.X.Min=0
	p.X.Max=float64(numDays)
	if err := p.Save(5, 3, "barchart.png"); err != nil {
		panic(err)
	}

}
<commit_msg>Last commit before trying out d3<commit_after>package main

import(
	"math"
	"fmt"
	"errors"
	"math/rand"
	"time"
	"sync"
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
)

type Result struct{
	Person Person
	Workload Workload
}

func (r Result) String()string{
	return fmt.Sprintf("<Result %v %s>", r.Person, r.Workload)
}

type Person struct{
	Vg float64 // Value/grade
	Gt float64 // Grade/time
	La float64 // Look ahead
	Vt float64 // Value/time
	P float64 // Rememberance
	B float64 // Beta
	D float64 // Delta

	Semester Semester // The schedule for the semester
	Assignments []Assignment // The assignment due
	WorkHours map[Assignment]int // The number of hours worked on assignments.
}

func (p Person) String() string{
	return fmt.Sprintf("<Person Vg:%f Gt:%f La:%f Vt:%f P:%f B:%f D:%f",
		p.Vg, p.Gt, p.La, p.Vt, p.P, p.B, p.D)
}

type Assignment struct{
	TotalGrade int // The total grade units this assignment is graded out of.
	DateDue int // The date due in numbers of day into the semester
	Name string
}

func (a Assignment) String() string{
	return fmt.Sprintf("",a.TotalGrade, a.DateDue, a.Name)
}

type Semester struct{
	Days int // The number of days
	Day int // The current day
	Weights map[int]float64 // A mapping of days to the attitude weight.
	Allowed map[int]int // A maping of days to hours available to work.
}

type Work struct{
	Day int
	Hours int
	Assignment Assignment
}

func (w Work) String() string{
	return fmt.Sprintf("<W Day=%d Hours=%d Assignment=%v",w.Day, w.Hours, w.Assignment)
}

func IntFactorial(i int)(int, error){
	if i > 20|| i < 1{return 0, errors.New("Factorial out of range")}
	if i == 1{
		return 1, nil
	}
	res, _ := IntFactorial(i-1)
	return i * res, nil
}

// Represents a possible ammount to work in the future.
type Workload struct{
	Days []Work
	Hours map[Assignment]int
}

func (w Workload) String() string {
	return fmt.Sprintf("<WL days=%v hours=%v>" ,w.Days, w.Hours)
}

// Generate workloads starting on the current day.
func (p *Person) Workloads() []Workload {

	workloads := make([]Workload,1)
	// Create base workload
	workloads[0] = *new(Workload)
	workloads[0].Days = make([]Work, 0)
	workloads[0].Hours = make(map[Assignment]int)
	for _,assignment := range(p.Assignments){
		workloads[0].Hours[assignment] = 0
	}
	// For each day until the look ahead day.
	for i := 0; float64(i)
= (p.Semester.Day + i){
					// For every time that does not put you over the limit.
					for t:=1; basewl.Hours[assignment]+
						t+p.WorkHours[assignment] <=
						assignment.TotalGrade &&
						t <= p.Semester.Allowed[p.Semester.Day + i]; t++{

						wl := *new(Workload)
						day := *new(Work)
						day.Assignment = assignment
						day.Hours = t
						day.Day = i + p.Semester.Day
						wl.Days = append(basewl.Days,day)
						wl.Hours = make(map[Assignment]int)
						for k, v := range(basewl.Hours){
							wl.Hours[k]=v
						}
						wl.Hours[assignment] = basewl.Hours[assignment] + t
						newloads = append(newloads,wl)
					}
				}
			}
			wl := *new(Workload)
			day := *new(Work)
			day.Assignment = Assignment{}
			day.Hours = 0
			day.Day = i + p.Semester.Day
			wl.Days = append(basewl.Days,day)
			wl.Hours = make(map[Assignment]int)
			for k,v := range(basewl.Hours){
				wl.Hours[k]=v
			}
			newloads = append(newloads, wl)
		}

		workloads = newloads
	}
	return workloads
}

// The discount function for this individual
// If d is 0 then the function is 1, otherwise it is B * (D ^ d)
func (p *Person) Discount(d int) float64{
	if d == 0 {
		return 1
	}
	return p.B * math.Pow(p.D, float64(d))
}

// Returns the expected utility of the given work.
func (p Person) WorkUtility(work Work) float64{
	utility := (float64(work.Hours) * p.Vg *p.Gt *p.B * p.Discount(work.Assignment.DateDue - p.Semester.Day))
	cost := (float64(work.Hours) * p.Semester.Weights[work.Day] * p.Discount(work.Day - p.Semester.Day))
	return utility -cost
}

// Returns the expected utility of the given workload evaluated on the current day.
func (p Person) Utility(workload Workload) float64{
	utility := float64(0)
	for _, work := range(workload.Days){
		if work.Assignment.TotalGrade != 0{
			utility += p.WorkUtility(work)
		}
	}
	return utility
}

func (p Person) Simulate(results chan Result){
	var globalWorkload Workload
	globalWorkload.Hours = make(map[Assignment]int)

	for p.Semester.Day
 maxUtility{
				maxUtility = utility
				choices = make([]Workload, 0)
				choices = append(choices, workload)
			}
		}

		var choice Workload
		if len(choices) > 1{
			choice = choices[rand.Intn(len(choices))]
		}else{
			choice = choices[0]
		}
		if len(choice.Days) > 0 {
			globalWorkload.Days = append(globalWorkload.Days, choice.Days[0])
			p.WorkHours[choice.Days[0].Assignment] += choice.Days[0].Hours
			globalWorkload.Hours[choice.Days[0].Assignment] += choice.Days[0].Hours
		}
	}

	results <- Result{Person:p, Workload:globalWorkload}
}

func main(){
	fmt.Println("Simulating people... This is most likely not going to work.")

	people := make([]Person, 0)
	rand.Seed(time.Now().Unix())

	assignments := make([]Assignment, 0)
	assignments = append(assignments, Assignment{DateDue:30,TotalGrade: 20, Name:"Assignment 1"})
	assignments = append(assignments, Assignment{DateDue:30,TotalGrade: 20, Name:"Assignment 2"})
	numPeople := 100
	numDays := 30
	for p:=0; p< numPeople; p++{
		// Create person
		Vg := rand.NormFloat64() * 1 + 10
		Gt := rand.NormFloat64() * 0.25 + 2
		B := rand.NormFloat64() * 0.2 + 0.5
		D := rand.NormFloat64() * 0.1 + 0.8
		if D > 1.0{
			D = 1
		}
		person := Person{Vg:Vg, Gt:Gt, La: 7, Vt: 1, P:1, B:B, D:D, WorkHours:make(map[Assignment]int)}
		semester := Semester{Days: 30,Day:0,Weights:make(map[int]float64), Allowed:make(map[int]int)}
		person.Semester = semester
		for i := 0; i <= 30; i++{
			person.Semester.Weights[i] = float64(i)*0.025
			person.Semester.Allowed[i] = 3
			if i > 25 {
				person.Semester.Allowed[i] = 2
				person.Semester.Weights[i] += float64(i)*0.025
			}
		}

		person.Assignments = assignments

		people = append(people, person)
	}

	results := make(chan Result)
	inChan := make(chan Person)

	numRoutines := 10
	var wg sync.WaitGroup
	for i := 0; i < numRoutines; i++{
		wg.Add(1)
		go func(people chan Person, results chan Result){
			for person := range(people){
				person.Simulate(results)
			}
			wg.Done()
		}(inChan, results)
	}

	//Wait for the channel simulations to finish then close the results chan
	go func(){
		wg.Wait()
		close(results)
	}()

	// Keep placing people in the channel
	go func(people []Person, inChan chan Person){
		for _,person := range people{
			inChan <- person
		}
		close(inChan)
	}(people, inChan)

	frequency := make(map[Assignment]plotter.XYs, len(assignments))
	grades := make(map[Assignment]int)
	gradeDistribution := make(plotter.XYs,101)

	for i := 0;i<= 100;i++{
		gradeDistribution[i].X = float64(i)
		gradeDistribution[i].Y = 0
	}

	for _, Assignment := range assignments{
		frequency[Assignment] = make(plotter.XYs,numDays+1)
		for i := 0;i <= numDays; i++{
			frequency[Assignment][i].X = float64(i)
			frequency[Assignment][i].Y = 0
		}
	}
	z :=0
	for res := range results{
		res := res
		fmt.Printf("Results: %v \n\t%v\n", res.Person, res.Workload)
		for n,day := range res.Workload.Days{
			if day.Assignment.TotalGrade != 0{
				frequency[day.Assignment][n].Y += float64(day.Hours)
			}
		}
		for Assignment, Grade := range res.Workload.Hours{
			if Assignment.Name != ""{
				grade := int((float64(Grade)/(float64(Assignment.TotalGrade))*float64(100)))
				gradeDistribution[grade].Y = gradeDistribution[grade].Y + float64(1)
				grades[Assignment] = grades[Assignment] + Grade
			}
		}
		z+=1
	}
	wt, err := plot.New()
	if err != nil {
			panic(err)
	}
	wt.Title.Text = "Distribution of work hours vs time."
	wt.Y.Label.Text = "Work Hours"
	wt.X.Label.Text = "Time"
	wt.Add(plotter.NewGrid())

	for assignment,hour := range grades{
		fmt.Printf("Average %s: %f\n", assignment, float64(hour)/float64(numPeople))
	}

	i := 0
	for assignment, xys := range frequency{
		lpline,lppoints, err := plotter.NewLinePoints(xys)
		if err != nil{
			panic(err)
		}
		lpline.Color = plotutil.Color(i)
		lpline.Dashes = plotutil.Dashes(i)
		lppoints.Color = plotutil.Color(i)
		lppoints.Shape = plotutil.Shape(i)
		i++
		wt.Add(lpline)
		wt.Add(lppoints)
		wt.Legend.Add(assignment.Name, lpline, lppoints)
	}
	wt.Legend.Top = true
	wt.X.Min=0
	wt.X.Max=float64(numDays)
	if err := wt.Save(5, 3, "workDistribution.png"); err != nil {
		panic(err)
	}

	gt, err := plot.New()
	if err != nil {
			panic(err)
	}
	gt.Title.Text = "Distribution of grades"
	gt.Y.Label.Text = "Frequency"
	gt.X.Label.Text = "Grades"
	gt.Add(plotter.NewGrid())

	h, err := plotter.NewHistogram(gradeDistribution, 10)
	if err != nil{
		panic(err)
	}
	h.FillColor = plotutil.Color(1)
	gt.Add(h)

	gt.Legend.Top = true
	gt.Y.Min=0
	gt.Y.Tick.Marker = func(min, max float64) []plot.Tick {
		const suggestedTicks = 3
		delta := 1
		for (max-min)/float64(delta) < float64(suggestedTicks) {
			delta += 10
		}
		ticks := make([]plot.Tick, 0)
		for i := int(min); i<= int(max); i+= delta{
			ticks = append(ticks, plot.Tick{Value:float64(i),Label:fmt.Sprintf("%d",i)})
		}
		return ticks
	}
	gt.X.Min=0
	gt.X.Max=float64(100)
	if err := gt.Save(5, 3, "gradeDistribution.png"); err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>package upload

import (
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"github.com/qiniu/qshell/v2/iqshell/storage/object"
	"github.com/qiniu/qshell/v2/iqshell/storage/object/batch"
	"os"
)

type serverChecker struct {
	Bucket     string
	Key        string
	FilePath   string // 本地文件存储位置
	CheckExist bool   // 检查服务端是否已存在
	CheckHash  bool   // 是否检查 hash, 检查是会对比服务端文件 hash
	CheckSize  bool   // 是否检查文件大小，检查是会对比服务端文件大小
	FileSize   int64  // 文件大小，核对文件大小时使用
}

func (c *serverChecker) check() (exist, match bool, err *data.CodeError) {
	fileServerStatus, err := object.Status(object.StatusApiInfo{
		Bucket: c.Bucket,
		Key:    c.Key,
	})
	if err != nil {
		err = data.NewEmptyError().AppendDescF("get file status, %s", err)
		return false, false, err
	}

	checkHash := c.CheckHash
	if checkHash && utils.IsNetworkSource(c.FilePath) {
		checkHash = false
		log.WarningF("network resource doesn't support check hash: %s", c.FilePath)
	}

	if checkHash {
		return c.checkHash(fileServerStatus.OperationResult)
	} else if c.CheckSize {
		return c.checkServerSize(fileServerStatus.OperationResult)
	} else {
		//return true, true, nil
		return c.checkServerSize(fileServerStatus.OperationResult)
	}
}

func (c *serverChecker) checkHash(fileServerStatus batch.OperationResult) (exist bool, match bool, err *data.CodeError) {
	file, oErr := os.Open(c.FilePath)
	if oErr != nil {
		return true, false, data.NewEmptyError().AppendDescF("check hash: open local file:%s error, %s", c.FilePath, oErr)
	}
	defer func() {
		if e := file.Close(); e != nil {
			log.ErrorF("check hash: close file:%s error:%v", c.FilePath, e)
		}
	}()

	localHash := ""
	if utils.IsSignByEtagV2(fileServerStatus.Hash) {
		localHash, err = utils.EtagV2(file, fileServerStatus.Parts)
		if err != nil {
			return true, false, data.NewEmptyError().AppendDescF("check hash: get etag v2:%s error, %v", c.FilePath, err)
		}
	} else {
		localHash, err = utils.EtagV1(file)
		if err != nil {
			log.ErrorF("====== %v hash:%+v", err, localHash)
			return true, false, data.NewEmptyError().AppendDescF("check hash: get etag v1:%s error, %v", c.FilePath, err)
		}
	}

	if localHash == fileServerStatus.Hash {
		return true, true, nil
	} else {
		log.WarningF("File:%s exist at [%s:%s], but hash not match[%s|%s]",
			c.FilePath, c.Bucket, c.Key, localHash, fileServerStatus.Hash)
		return true, false, nil
	}
}

func (c *serverChecker) checkServerSize(fileServerStatus batch.OperationResult) (bool, bool, *data.CodeError) {
	if c.FileSize == fileServerStatus.FSize {
		return true, true, nil
	} else {
		log.WarningF("File:%s exist at [%s:%s], but size not match[%d|%d]",
			c.FilePath, c.Bucket, c.Key, c.FileSize, fileServerStatus.FSize)
		return true, false, nil
	}
}
<commit_msg>delete useless code<commit_after>package upload

import (
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"github.com/qiniu/qshell/v2/iqshell/storage/object"
	"github.com/qiniu/qshell/v2/iqshell/storage/object/batch"
	"os"
)

type serverChecker struct {
	Bucket     string
	Key        string
	FilePath   string // 本地文件存储位置
	CheckExist bool   // 检查服务端是否已存在
	CheckHash  bool   // 是否检查 hash, 检查是会对比服务端文件 hash
	CheckSize  bool   // 是否检查文件大小，检查是会对比服务端文件大小
	FileSize   int64  // 文件大小，核对文件大小时使用
}

func (c *serverChecker) check() (exist, match bool, err *data.CodeError) {
	fileServerStatus, err := object.Status(object.StatusApiInfo{
		Bucket: c.Bucket,
		Key:    c.Key,
	})
	if err != nil {
		err = data.NewEmptyError().AppendDescF("get file status, %s", err)
		return false, false, err
	}

	checkHash := c.CheckHash
	if checkHash && utils.IsNetworkSource(c.FilePath) {
		checkHash = false
		log.WarningF("network resource doesn't support check hash: %s", c.FilePath)
	}

	if checkHash {
		return c.checkHash(fileServerStatus.OperationResult)
	} else if c.CheckSize {
		return c.checkServerSize(fileServerStatus.OperationResult)
	} else {
		//return true, true, nil
		return c.checkServerSize(fileServerStatus.OperationResult)
	}
}

func (c *serverChecker) checkHash(fileServerStatus batch.OperationResult) (exist bool, match bool, err *data.CodeError) {
	file, oErr := os.Open(c.FilePath)
	if oErr != nil {
		return true, false, data.NewEmptyError().AppendDescF("check hash: open local file:%s error, %s", c.FilePath, oErr)
	}
	defer func() {
		if e := file.Close(); e != nil {
			log.ErrorF("check hash: close file:%s error:%v", c.FilePath, e)
		}
	}()

	localHash := ""
	if utils.IsSignByEtagV2(fileServerStatus.Hash) {
		localHash, err = utils.EtagV2(file, fileServerStatus.Parts)
		if err != nil {
			return true, false, data.NewEmptyError().AppendDescF("check hash: get etag v2:%s error, %v", c.FilePath, err)
		}
	} else {
		localHash, err = utils.EtagV1(file)
		if err != nil {
			return true, false, data.NewEmptyError().AppendDescF("check hash: get etag v1:%s error, %v", c.FilePath, err)
		}
	}

	if localHash == fileServerStatus.Hash {
		return true, true, nil
	} else {
		log.WarningF("File:%s exist at [%s:%s], but hash not match[%s|%s]",
			c.FilePath, c.Bucket, c.Key, localHash, fileServerStatus.Hash)
		return true, false, nil
	}
}

func (c *serverChecker) checkServerSize(fileServerStatus batch.OperationResult) (bool, bool, *data.CodeError) {
	if c.FileSize == fileServerStatus.FSize {
		return true, true, nil
	} else {
		log.WarningF("File:%s exist at [%s:%s], but size not match[%d|%d]",
			c.FilePath, c.Bucket, c.Key, c.FileSize, fileServerStatus.FSize)
		return true, false, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/rsync"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
)

func daemonStorageMount(s *state.State) error {
	var storageBackups string
	var storageImages string
	err := s.Node.Transaction(func(tx *db.NodeTx) error {
		nodeConfig, err := node.ConfigLoad(tx)
		if err != nil {
			return err
		}

		storageBackups = nodeConfig.StorageBackupsVolume()
		storageImages = nodeConfig.StorageImagesVolume()

		return nil
	})
	if err != nil {
		return err
	}

	mount := func(storageType string, source string) error {
		// Parse the source.
		fields := strings.Split(source, "/")
		if len(fields) != 2 {
			return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
		}

		poolName := fields[0]
		volumeName := fields[1]

		pool, err := storagePools.GetPoolByName(s, poolName)
		if err != nil {
			return err
		}

		// Mount volume.
		_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
		if err != nil {
			return errors.Wrapf(err, "Failed to mount storage volume %q", source)
		}

		return nil
	}

	if storageBackups != "" {
		err := mount("backups", storageBackups)
		if err != nil {
			return errors.Wrap(err, "Failed to mount backups storage")
		}
	}

	if storageImages != "" {
		err := mount("images", storageImages)
		if err != nil {
			return errors.Wrap(err, "Failed to mount images storage")
		}
	}

	return nil
}

func daemonStorageUsed(s *state.State, poolName string, volumeName string) (bool, error) {
	var storageBackups string
	var storageImages string
	err := s.Node.Transaction(func(tx *db.NodeTx) error {
		nodeConfig, err := node.ConfigLoad(tx)
		if err != nil {
			return err
		}

		storageBackups = nodeConfig.StorageBackupsVolume()
		storageImages = nodeConfig.StorageImagesVolume()

		return nil
	})
	if err != nil {
		return false, err
	}

	fullName := fmt.Sprintf("%s/%s", poolName, volumeName)
	if storageBackups == fullName || storageImages == fullName {
		return true, nil
	}

	return false, nil
}

func daemonStorageValidate(s *state.State, target string) error {
	// Check syntax.
	if target == "" {
		return nil
	}

	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	// Validate pool exists.
	poolID, dbPool, err := s.Cluster.StoragePoolGet(poolName)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage pool %q", poolName)
	}

	// Validate pool driver (can't be CEPH or CEPHFS).
	if dbPool.Driver == "ceph" || dbPool.Driver == "cephfs" {
		return fmt.Errorf("Server storage volumes cannot be stored on Ceph")
	}

	// Confirm volume exists.
	_, _, err = s.Cluster.StoragePoolNodeVolumeGetType(volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume %q", target)
	}

	snapshots, err := s.Cluster.StoragePoolVolumeSnapshotsGetType(volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume snapshots %q", target)
	}

	if len(snapshots) != 0 {
		return fmt.Errorf("Storage volumes for use by LXD itself cannot have snapshots")
	}

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	ourMount, err := pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}
	if ourMount {
		defer pool.UnmountCustomVolume(project.Default, volumeName, nil)
	}

	// Validate volume is empty (ignore lost+found).
	mountpoint := shared.VarPath("storage-pools", poolName, "custom", volumeName)

	entries, err := ioutil.ReadDir(mountpoint)
	if err != nil {
		return errors.Wrapf(err, "Failed to list %q", mountpoint)
	}

	for _, entry := range entries {
		entryName := entry.Name()

		if entryName == "lost+found" {
			continue
		}

		return fmt.Errorf("Storage volume %q isn't empty", target)
	}

	return nil
}

func daemonStorageMove(s *state.State, storageType string, target string) error {
	destPath := shared.VarPath(storageType)

	// Track down the current storage.
	var sourcePool string
	var sourceVolume string

	sourcePath, err := os.Readlink(destPath)
	if err != nil {
		sourcePath = destPath
	} else {
		fields := strings.Split(sourcePath, "/")
		sourcePool = fields[len(fields)-3]
		sourceVolume = fields[len(fields)-1]
	}

	moveContent := func(source string, target string) error {
		// Copy the content.
		_, err := rsync.LocalCopy(source, target, "", false)
		if err != nil {
			return err
		}

		// Remove the source content.
		entries, err := ioutil.ReadDir(source)
		if err != nil {
			return err
		}

		for _, entry := range entries {
			err := os.RemoveAll(filepath.Join(source, entry.Name()))
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Deal with unsetting.
	if target == "" {
		// Things already look correct.
		if sourcePath == destPath {
			return nil
		}

		// Remove the symlink.
		err = os.Remove(destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to delete storage symlink at %q", destPath)
		}

		// Re-create as a directory.
		err = os.MkdirAll(destPath, 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed to create directory %q", destPath)
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		_, err = pool.UnmountCustomVolume(project.Default, sourceVolume, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolume)
		}

		return nil
	}

	// Parse the target.
	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}

	// Set ownership & mode.
	mountpoint := shared.VarPath("storage-pools", poolName, "custom", volumeName)
	destPath = mountpoint

	err = os.Chmod(mountpoint, 0700)
	if err != nil {
		return errors.Wrapf(err, "Failed to set permissions on %q", mountpoint)
	}

	err = os.Chown(mountpoint, 0, 0)
	if err != nil {
		return errors.Wrapf(err, "Failed to set ownership on %q", mountpoint)
	}

	// Handle changes.
	if sourcePath != shared.VarPath(storageType) {
		// Remove the symlink.
		err := os.Remove(shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to remove the new symlink at %q", shared.VarPath(storageType))
		}

		// Create the new symlink.
		err = os.Symlink(destPath, shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		_, err = pool.UnmountCustomVolume(project.Default, sourceVolume, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolume)
		}

		return nil
	}

	sourcePath = shared.VarPath(storageType) + ".temp"

	// Rename the existing storage.
	err = os.Rename(shared.VarPath(storageType), sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to rename existing storage %q", shared.VarPath(storageType))
	}

	// Create the new symlink.
	err = os.Symlink(destPath, shared.VarPath(storageType))
	if err != nil {
		return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
	}

	// Move the data across.
	err = moveContent(sourcePath, destPath)
	if err != nil {
		return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
	}

	// Remove the old data.
	err = os.RemoveAll(sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to cleanup old directory %q", sourcePath)
	}

	return nil
}
<commit_msg>lxd/daemon/storage: Adds support for custom volume projects<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/rsync"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	storageDrivers "github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared"
)

func daemonStorageMount(s *state.State) error {
	var storageBackups string
	var storageImages string
	err := s.Node.Transaction(func(tx *db.NodeTx) error {
		nodeConfig, err := node.ConfigLoad(tx)
		if err != nil {
			return err
		}

		storageBackups = nodeConfig.StorageBackupsVolume()
		storageImages = nodeConfig.StorageImagesVolume()

		return nil
	})
	if err != nil {
		return err
	}

	mount := func(storageType string, source string) error {
		// Parse the source.
		fields := strings.Split(source, "/")
		if len(fields) != 2 {
			return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
		}

		poolName := fields[0]
		volumeName := fields[1]

		pool, err := storagePools.GetPoolByName(s, poolName)
		if err != nil {
			return err
		}

		// Mount volume.
		_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
		if err != nil {
			return errors.Wrapf(err, "Failed to mount storage volume %q", source)
		}

		return nil
	}

	if storageBackups != "" {
		err := mount("backups", storageBackups)
		if err != nil {
			return errors.Wrap(err, "Failed to mount backups storage")
		}
	}

	if storageImages != "" {
		err := mount("images", storageImages)
		if err != nil {
			return errors.Wrap(err, "Failed to mount images storage")
		}
	}

	return nil
}

func daemonStorageUsed(s *state.State, poolName string, volumeName string) (bool, error) {
	var storageBackups string
	var storageImages string
	err := s.Node.Transaction(func(tx *db.NodeTx) error {
		nodeConfig, err := node.ConfigLoad(tx)
		if err != nil {
			return err
		}

		storageBackups = nodeConfig.StorageBackupsVolume()
		storageImages = nodeConfig.StorageImagesVolume()

		return nil
	})
	if err != nil {
		return false, err
	}

	fullName := fmt.Sprintf("%s/%s", poolName, volumeName)
	if storageBackups == fullName || storageImages == fullName {
		return true, nil
	}

	return false, nil
}

func daemonStorageValidate(s *state.State, target string) error {
	// Check syntax.
	if target == "" {
		return nil
	}

	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	// Validate pool exists.
	poolID, dbPool, err := s.Cluster.StoragePoolGet(poolName)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage pool %q", poolName)
	}

	// Validate pool driver (can't be CEPH or CEPHFS).
	if dbPool.Driver == "ceph" || dbPool.Driver == "cephfs" {
		return fmt.Errorf("Server storage volumes cannot be stored on Ceph")
	}

	// Confirm volume exists.
	_, _, err = s.Cluster.StoragePoolNodeVolumeGetTypeByProject(project.Default, volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume %q", target)
	}

	snapshots, err := s.Cluster.StoragePoolVolumeSnapshotsGetType(project.Default, volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume snapshots %q", target)
	}

	if len(snapshots) != 0 {
		return fmt.Errorf("Storage volumes for use by LXD itself cannot have snapshots")
	}

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	ourMount, err := pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}
	if ourMount {
		defer pool.UnmountCustomVolume(project.Default, volumeName, nil)
	}

	// Validate volume is empty (ignore lost+found).
	volStorageName := project.StorageVolume(project.Default, volumeName)
	mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName)

	entries, err := ioutil.ReadDir(mountpoint)
	if err != nil {
		return errors.Wrapf(err, "Failed to list %q", mountpoint)
	}

	for _, entry := range entries {
		entryName := entry.Name()

		if entryName == "lost+found" {
			continue
		}

		return fmt.Errorf("Storage volume %q isn't empty", target)
	}

	return nil
}

func daemonStorageMove(s *state.State, storageType string, target string) error {
	destPath := shared.VarPath(storageType)

	// Track down the current storage.
	var sourcePool string
	var sourceVolume string

	sourcePath, err := os.Readlink(destPath)
	if err != nil {
		sourcePath = destPath
	} else {
		fields := strings.Split(sourcePath, "/")
		sourcePool = fields[len(fields)-3]
		sourceVolume = fields[len(fields)-1]
	}

	moveContent := func(source string, target string) error {
		// Copy the content.
		_, err := rsync.LocalCopy(source, target, "", false)
		if err != nil {
			return err
		}

		// Remove the source content.
		entries, err := ioutil.ReadDir(source)
		if err != nil {
			return err
		}

		for _, entry := range entries {
			err := os.RemoveAll(filepath.Join(source, entry.Name()))
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Deal with unsetting.
	if target == "" {
		// Things already look correct.
		if sourcePath == destPath {
			return nil
		}

		// Remove the symlink.
		err = os.Remove(destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to delete storage symlink at %q", destPath)
		}

		// Re-create as a directory.
		err = os.MkdirAll(destPath, 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed to create directory %q", destPath)
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		projectName, sourceVolumeName := project.StorageVolumeParts(sourceVolume)
		_, err = pool.UnmountCustomVolume(projectName, sourceVolumeName, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolumeName)
		}

		return nil
	}

	// Parse the target.
	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}

	// Set ownership & mode.
	volStorageName := project.StorageVolume(project.Default, volumeName)
	mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName)
	destPath = mountpoint

	err = os.Chmod(mountpoint, 0700)
	if err != nil {
		return errors.Wrapf(err, "Failed to set permissions on %q", mountpoint)
	}

	err = os.Chown(mountpoint, 0, 0)
	if err != nil {
		return errors.Wrapf(err, "Failed to set ownership on %q", mountpoint)
	}

	// Handle changes.
	if sourcePath != shared.VarPath(storageType) {
		// Remove the symlink.
		err := os.Remove(shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to remove the new symlink at %q", shared.VarPath(storageType))
		}

		// Create the new symlink.
		err = os.Symlink(destPath, shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		_, err = pool.UnmountCustomVolume(project.Default, sourceVolume, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolume)
		}

		return nil
	}

	sourcePath = shared.VarPath(storageType) + ".temp"

	// Rename the existing storage.
	err = os.Rename(shared.VarPath(storageType), sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to rename existing storage %q", shared.VarPath(storageType))
	}

	// Create the new symlink.
	err = os.Symlink(destPath, shared.VarPath(storageType))
	if err != nil {
		return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
	}

	// Move the data across.
	err = moveContent(sourcePath, destPath)
	if err != nil {
		return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
	}

	// Remove the old data.
	err = os.RemoveAll(sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to cleanup old directory %q", sourcePath)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/query"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

func doProfileUpdate(d *Daemon, project, name string, id int64, profile *api.Profile, req api.ProfilePut) error {
	// Sanity checks
	err := instance.ValidConfig(d.os, req.Config, true, false)
	if err != nil {
		return err
	}

	// Validate instance devices with ProfileValidationName name to indicate profile validation.
	// At this point we don't know the instance type, so just use Container type for validation.
	err = instance.ValidDevices(d.State(), d.cluster, instancetype.Container, instance.ProfileValidationName, deviceConfig.NewDevices(req.Devices), false)
	if err != nil {
		return err
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query containers associated with profile '%s'", name)
	}

	// Check if the root device is supposed to be changed or removed.
	oldProfileRootDiskDeviceKey, oldProfileRootDiskDevice, _ := shared.GetRootDiskDevice(profile.Devices)
	_, newProfileRootDiskDevice, _ := shared.GetRootDiskDevice(req.Devices)
	if len(containers) > 0 && oldProfileRootDiskDevice["pool"] != "" && newProfileRootDiskDevice["pool"] == "" || (oldProfileRootDiskDevice["pool"] != newProfileRootDiskDevice["pool"]) {
		// Check for containers using the device
		for _, container := range containers {
			// Check if the device is locally overridden
			k, v, _ := shared.GetRootDiskDevice(container.Devices.CloneNative())
			if k != "" && v["pool"] != "" {
				continue
			}

			// Check what profile the device comes from
			profiles := container.Profiles
			for i := len(profiles) - 1; i >= 0; i-- {
				_, profile, err := d.cluster.ProfileGet("default", profiles[i])
				if err != nil {
					return err
				}

				// Check if we find a match for the device
				_, ok := profile.Devices[oldProfileRootDiskDeviceKey]
				if ok {
					// Found the profile
					if profiles[i] == name {
						// If it's the current profile, then we can't modify that root device
						return fmt.Errorf("At least one container relies on this profile's root disk device")
					} else {
						// If it's not, then move on to the next container
						break
					}
				}
			}
		}
	}

	// Update the database
	err = query.Retry(func() error {
		tx, err := d.cluster.Begin()
		if err != nil {
			return err
		}

		if profile.Description != req.Description {
			err = db.ProfileDescriptionUpdate(tx, id, req.Description)
			if err != nil {
				tx.Rollback()
				return err
			}
		}

		// Optimize for description-only changes
		if reflect.DeepEqual(profile.Config, req.Config) && reflect.DeepEqual(profile.Devices, req.Devices) {
			err = db.TxCommit(tx)
			if err != nil {
				return err
			}

			return nil
		}

		err = db.ProfileConfigClear(tx, id)
		if err != nil {
			tx.Rollback()
			return err
		}

		err = db.ProfileConfigAdd(tx, id, req.Config)
		if err != nil {
			tx.Rollback()
			return err
		}

		err = db.DevicesAdd(tx, "profile", id, deviceConfig.NewDevices(req.Devices))
		if err != nil {
			tx.Rollback()
			return err
		}

		err = db.TxCommit(tx)
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Update all the containers on this node using the profile. Must be
	// done after db.TxCommit due to DB lock.
	nodeName := ""
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.NodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}
	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, profile.ProfilePut, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Like doProfileUpdate but does not update the database, since it was already
// updated by doProfileUpdate itself, called on the notifying node.
func doProfileUpdateCluster(d *Daemon, project, name string, old api.ProfilePut) error {
	nodeName := ""
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.NodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query containers associated with profile '%s'", name)
	}

	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, old, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Profile update of a single container.
func doProfileUpdateContainer(d *Daemon, name string, old api.ProfilePut, nodeName string, args db.InstanceArgs) error {
	if args.Node != "" && args.Node != nodeName {
		// No-op, this container does not belong to this node.
		return nil
	}

	profiles, err := d.cluster.ProfilesGet(args.Project, args.Profiles)
	if err != nil {
		return err
	}
	for i, profileName := range args.Profiles {
		if profileName == name {
			// Use the old config and devices.
			profiles[i].Config = old.Config
			profiles[i].Devices = old.Devices
			break
		}
	}

	c := containerLXCInstantiate(d.State(), args)

	c.expandConfig(profiles)
	c.expandDevices(profiles)

	return c.Update(db.InstanceArgs{
		Architecture: c.Architecture(),
		Config:       c.LocalConfig(),
		Description:  c.Description(),
		Devices:      c.LocalDevices(),
		Ephemeral:    c.IsEphemeral(),
		Profiles:     c.Profiles(),
		Project:      c.Project(),
		Type:         c.Type(),
		Snapshot:     c.IsSnapshot(),
	}, true)
}

// Query the db for information about containers associated with the given
// profile.
func getProfileContainersInfo(cluster *db.Cluster, project, profile string) ([]db.InstanceArgs, error) {
	// Query the db for information about containers associated with the
	// given profile.
	names, err := cluster.ProfileContainersGet(project, profile)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to query containers with profile '%s'", profile)
	}

	containers := []db.InstanceArgs{}
	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		for ctProject, ctNames := range names {
			for _, ctName := range ctNames {
				container, err := tx.InstanceGet(ctProject, ctName)
				if err != nil {
					return err
				}

				containers = append(containers, db.ContainerToArgs(container))
			}
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to fetch containers")
	}

	return containers, nil
}
<commit_msg>lxd/profiles/utils: db.InstanceToArgs usage<commit_after>package main

import (
	"fmt"
	"reflect"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/query"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

func doProfileUpdate(d *Daemon, project, name string, id int64, profile *api.Profile, req api.ProfilePut) error {
	// Sanity checks
	err := instance.ValidConfig(d.os, req.Config, true, false)
	if err != nil {
		return err
	}

	// Validate instance devices with ProfileValidationName name to indicate profile validation.
	// At this point we don't know the instance type, so just use Container type for validation.
	err = instance.ValidDevices(d.State(), d.cluster, instancetype.Container, instance.ProfileValidationName, deviceConfig.NewDevices(req.Devices), false)
	if err != nil {
		return err
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query containers associated with profile '%s'", name)
	}

	// Check if the root device is supposed to be changed or removed.
	oldProfileRootDiskDeviceKey, oldProfileRootDiskDevice, _ := shared.GetRootDiskDevice(profile.Devices)
	_, newProfileRootDiskDevice, _ := shared.GetRootDiskDevice(req.Devices)
	if len(containers) > 0 && oldProfileRootDiskDevice["pool"] != "" && newProfileRootDiskDevice["pool"] == "" || (oldProfileRootDiskDevice["pool"] != newProfileRootDiskDevice["pool"]) {
		// Check for containers using the device
		for _, container := range containers {
			// Check if the device is locally overridden
			k, v, _ := shared.GetRootDiskDevice(container.Devices.CloneNative())
			if k != "" && v["pool"] != "" {
				continue
			}

			// Check what profile the device comes from
			profiles := container.Profiles
			for i := len(profiles) - 1; i >= 0; i-- {
				_, profile, err := d.cluster.ProfileGet("default", profiles[i])
				if err != nil {
					return err
				}

				// Check if we find a match for the device
				_, ok := profile.Devices[oldProfileRootDiskDeviceKey]
				if ok {
					// Found the profile
					if profiles[i] == name {
						// If it's the current profile, then we can't modify that root device
						return fmt.Errorf("At least one container relies on this profile's root disk device")
					} else {
						// If it's not, then move on to the next container
						break
					}
				}
			}
		}
	}

	// Update the database
	err = query.Retry(func() error {
		tx, err := d.cluster.Begin()
		if err != nil {
			return err
		}

		if profile.Description != req.Description {
			err = db.ProfileDescriptionUpdate(tx, id, req.Description)
			if err != nil {
				tx.Rollback()
				return err
			}
		}

		// Optimize for description-only changes
		if reflect.DeepEqual(profile.Config, req.Config) && reflect.DeepEqual(profile.Devices, req.Devices) {
			err = db.TxCommit(tx)
			if err != nil {
				return err
			}

			return nil
		}

		err = db.ProfileConfigClear(tx, id)
		if err != nil {
			tx.Rollback()
			return err
		}

		err = db.ProfileConfigAdd(tx, id, req.Config)
		if err != nil {
			tx.Rollback()
			return err
		}

		err = db.DevicesAdd(tx, "profile", id, deviceConfig.NewDevices(req.Devices))
		if err != nil {
			tx.Rollback()
			return err
		}

		err = db.TxCommit(tx)
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Update all the containers on this node using the profile. Must be
	// done after db.TxCommit due to DB lock.
	nodeName := ""
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.NodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}
	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, profile.ProfilePut, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Like doProfileUpdate but does not update the database, since it was already
// updated by doProfileUpdate itself, called on the notifying node.
func doProfileUpdateCluster(d *Daemon, project, name string, old api.ProfilePut) error {
	nodeName := ""
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.NodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query containers associated with profile '%s'", name)
	}

	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, old, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Profile update of a single container.
func doProfileUpdateContainer(d *Daemon, name string, old api.ProfilePut, nodeName string, args db.InstanceArgs) error {
	if args.Node != "" && args.Node != nodeName {
		// No-op, this container does not belong to this node.
		return nil
	}

	profiles, err := d.cluster.ProfilesGet(args.Project, args.Profiles)
	if err != nil {
		return err
	}
	for i, profileName := range args.Profiles {
		if profileName == name {
			// Use the old config and devices.
			profiles[i].Config = old.Config
			profiles[i].Devices = old.Devices
			break
		}
	}

	c := containerLXCInstantiate(d.State(), args)

	c.expandConfig(profiles)
	c.expandDevices(profiles)

	return c.Update(db.InstanceArgs{
		Architecture: c.Architecture(),
		Config:       c.LocalConfig(),
		Description:  c.Description(),
		Devices:      c.LocalDevices(),
		Ephemeral:    c.IsEphemeral(),
		Profiles:     c.Profiles(),
		Project:      c.Project(),
		Type:         c.Type(),
		Snapshot:     c.IsSnapshot(),
	}, true)
}

// Query the db for information about containers associated with the given
// profile.
func getProfileContainersInfo(cluster *db.Cluster, project, profile string) ([]db.InstanceArgs, error) {
	// Query the db for information about containers associated with the
	// given profile.
	names, err := cluster.ProfileContainersGet(project, profile)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to query containers with profile '%s'", profile)
	}

	containers := []db.InstanceArgs{}
	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		for ctProject, ctNames := range names {
			for _, ctName := range ctNames {
				container, err := tx.InstanceGet(ctProject, ctName)
				if err != nil {
					return err
				}

				containers = append(containers, db.InstanceToArgs(container))
			}
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to fetch containers")
	}

	return containers, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pgsql

import (
	"database/sql"
	"time"

	"github.com/coreos/clair/database"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
)

func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
	if feature.Name == "" {
		return 0, cerrors.NewBadRequestError("could not find/insert invalid Feature")
	}

	// Do cache lookup.
	if pgSQL.cache != nil {
		promCacheQueriesTotal.WithLabelValues("feature").Inc()
		id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Name)
		if found {
			promCacheHitsTotal.WithLabelValues("feature").Inc()
			return id.(int), nil
		}
	}

	// We do `defer observeQueryTime` here because we don't want to observe cached features.
	defer observeQueryTime("insertFeature", "all", time.Now())

	// Find or create Namespace.
	namespaceID, err := pgSQL.insertNamespace(feature.Namespace)
	if err != nil {
		return 0, err
	}

	// Find or create Feature.
	var id int
	err = pgSQL.QueryRow(getQuery("soi_feature"), feature.Name, namespaceID).Scan(&id)
	if err != nil {
		return 0, handleError("soi_feature", err)
	}

	if pgSQL.cache != nil {
		pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Name, id)
	}

	return id, nil
}

func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) (id int, err error) {
	if featureVersion.Version.String() == "" {
		return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion")
	}

	// Do cache lookup.
	cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String()
	if pgSQL.cache != nil {
		promCacheQueriesTotal.WithLabelValues("featureversion").Inc()
		id, found := pgSQL.cache.Get(cacheIndex)
		if found {
			promCacheHitsTotal.WithLabelValues("featureversion").Inc()
			return id.(int), nil
		}
	}

	// We do `defer observeQueryTime` here because we don't want to observe cached featureversions.
	defer observeQueryTime("insertFeatureVersion", "all", time.Now())

	// Find or create Feature first.
	t := time.Now()
	featureID, err := pgSQL.insertFeature(featureVersion.Feature)
	observeQueryTime("insertFeatureVersion", "insertFeature", t)

	if err != nil {
		return 0, err
	}

	featureVersion.Feature.ID = featureID

	// Begin transaction.
	tx, err := pgSQL.Begin()
	if err != nil {
		tx.Rollback()
		return 0, handleError("insertFeatureVersion.Begin()", err)
	}

	// Lock Vulnerability_Affects_FeatureVersion exclusively.
	// We want to prevent InsertVulnerability to modify it.
	promConcurrentLockVAFV.Inc()
	defer promConcurrentLockVAFV.Dec()
	t = time.Now()
	_, err = tx.Exec(getQuery("l_vulnerability_affects_featureversion"))
	observeQueryTime("insertFeatureVersion", "lock", t)

	if err != nil {
		tx.Rollback()
		return 0, handleError("insertFeatureVersion.l_vulnerability_affects_featureversion", err)
	}

	// Find or create FeatureVersion.
	var newOrExisting string

	t = time.Now()
	err = tx.QueryRow(getQuery("soi_featureversion"), featureID, &featureVersion.Version).
		Scan(&newOrExisting, &featureVersion.ID)
	observeQueryTime("insertFeatureVersion", "soi_featureversion", t)

	if err != nil {
		tx.Rollback()
		return 0, handleError("soi_featureversion", err)
	}

	if newOrExisting == "exi" {
		// That featureVersion already exists, return its id.
		tx.Commit()
		return featureVersion.ID, nil
	}

	// Link the new FeatureVersion with every vulnerabilities that affect it, by inserting in
	// Vulnerability_Affects_FeatureVersion.
	t = time.Now()
	err = linkFeatureVersionToVulnerabilities(tx, featureVersion)
	observeQueryTime("insertFeatureVersion", "linkFeatureVersionToVulnerabilities", t)

	if err != nil {
		tx.Rollback()
		return 0, err
	}

	// Commit transaction.
	err = tx.Commit()
	if err != nil {
		return 0, handleError("insertFeatureVersion.Commit()", err)
	}

	if pgSQL.cache != nil {
		pgSQL.cache.Add(cacheIndex, featureVersion.ID)
	}

	return featureVersion.ID, nil
}

// TODO(Quentin-M): Batch me
func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) {
	IDs := make([]int, 0, len(featureVersions))

	for i := 0; i < len(featureVersions); i++ {
		id, err := pgSQL.insertFeatureVersion(featureVersions[i])
		if err != nil {
			return IDs, err
		}
		IDs = append(IDs, id)
	}

	return IDs, nil
}

type vulnerabilityAffectsFeatureVersion struct {
	vulnerabilityID int
	fixedInID       int
	fixedInVersion  types.Version
}

func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.FeatureVersion) error {
	// Select every vulnerability and the fixed version that affect this Feature.
	// TODO(Quentin-M): LIMIT
	rows, err := tx.Query(getQuery("s_vulnerability_fixedin_feature"), featureVersion.Feature.ID)
	if err != nil {
		return handleError("s_vulnerability_fixedin_feature", err)
	}
	defer rows.Close()

	var affects []vulnerabilityAffectsFeatureVersion
	for rows.Next() {
		var affect vulnerabilityAffectsFeatureVersion

		err := rows.Scan(&affect.fixedInID, &affect.vulnerabilityID, &affect.fixedInVersion)
		if err != nil {
			return handleError("s_vulnerability_fixedin_feature.Scan()", err)
		}

		if featureVersion.Version.Compare(affect.fixedInVersion) < 0 {
			// The version of the FeatureVersion we are inserting is lower than the fixed version on this
			// Vulnerability, thus, this FeatureVersion is affected by it.
			affects = append(affects, affect)
		}
	}
	if err = rows.Err(); err != nil {
		return handleError("s_vulnerability_fixedin_feature.Rows()", err)
	}
	rows.Close()

	// Insert into Vulnerability_Affects_FeatureVersion.
	for _, affect := range affects {
		// TODO(Quentin-M): Batch me.
		_, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), affect.vulnerabilityID,
			featureVersion.ID, affect.fixedInID)
		if err != nil {
			return handleError("i_vulnerability_affects_featureversion", err)
		}
	}

	return nil
}
<commit_msg>database: cache feature version upon lookup<commit_after>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pgsql

import (
	"database/sql"
	"time"

	"github.com/coreos/clair/database"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
)

func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
	if feature.Name == "" {
		return 0, cerrors.NewBadRequestError("could not find/insert invalid Feature")
	}

	// Do cache lookup.
	if pgSQL.cache != nil {
		promCacheQueriesTotal.WithLabelValues("feature").Inc()
		id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Name)
		if found {
			promCacheHitsTotal.WithLabelValues("feature").Inc()
			return id.(int), nil
		}
	}

	// We do `defer observeQueryTime` here because we don't want to observe cached features.
	defer observeQueryTime("insertFeature", "all", time.Now())

	// Find or create Namespace.
	namespaceID, err := pgSQL.insertNamespace(feature.Namespace)
	if err != nil {
		return 0, err
	}

	// Find or create Feature.
	var id int
	err = pgSQL.QueryRow(getQuery("soi_feature"), feature.Name, namespaceID).Scan(&id)
	if err != nil {
		return 0, handleError("soi_feature", err)
	}

	if pgSQL.cache != nil {
		pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Name, id)
	}

	return id, nil
}

func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) (id int, err error) {
	if featureVersion.Version.String() == "" {
		return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion")
	}

	// Do cache lookup.
	cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String()
	if pgSQL.cache != nil {
		promCacheQueriesTotal.WithLabelValues("featureversion").Inc()
		id, found := pgSQL.cache.Get(cacheIndex)
		if found {
			promCacheHitsTotal.WithLabelValues("featureversion").Inc()
			return id.(int), nil
		}
	}

	// We do `defer observeQueryTime` here because we don't want to observe cached featureversions.
	defer observeQueryTime("insertFeatureVersion", "all", time.Now())

	// Find or create Feature first.
	t := time.Now()
	featureID, err := pgSQL.insertFeature(featureVersion.Feature)
	observeQueryTime("insertFeatureVersion", "insertFeature", t)

	if err != nil {
		return 0, err
	}

	featureVersion.Feature.ID = featureID

	// Begin transaction.
	tx, err := pgSQL.Begin()
	if err != nil {
		tx.Rollback()
		return 0, handleError("insertFeatureVersion.Begin()", err)
	}

	// Lock Vulnerability_Affects_FeatureVersion exclusively.
	// We want to prevent InsertVulnerability to modify it.
	promConcurrentLockVAFV.Inc()
	defer promConcurrentLockVAFV.Dec()
	t = time.Now()
	_, err = tx.Exec(getQuery("l_vulnerability_affects_featureversion"))
	observeQueryTime("insertFeatureVersion", "lock", t)

	if err != nil {
		tx.Rollback()
		return 0, handleError("insertFeatureVersion.l_vulnerability_affects_featureversion", err)
	}

	// Find or create FeatureVersion.
	var newOrExisting string

	t = time.Now()
	err = tx.QueryRow(getQuery("soi_featureversion"), featureID, &featureVersion.Version).
		Scan(&newOrExisting, &featureVersion.ID)
	observeQueryTime("insertFeatureVersion", "soi_featureversion", t)

	if err != nil {
		tx.Rollback()
		return 0, handleError("soi_featureversion", err)
	}

	if newOrExisting == "exi" {
		// That featureVersion already exists, return its id.
		tx.Commit()

		if pgSQL.cache != nil {
			pgSQL.cache.Add(cacheIndex, featureVersion.ID)
		}

		return featureVersion.ID, nil
	}

	// Link the new FeatureVersion with every vulnerabilities that affect it, by inserting in
	// Vulnerability_Affects_FeatureVersion.
	t = time.Now()
	err = linkFeatureVersionToVulnerabilities(tx, featureVersion)
	observeQueryTime("insertFeatureVersion", "linkFeatureVersionToVulnerabilities", t)

	if err != nil {
		tx.Rollback()
		return 0, err
	}

	// Commit transaction.
	err = tx.Commit()
	if err != nil {
		return 0, handleError("insertFeatureVersion.Commit()", err)
	}

	if pgSQL.cache != nil {
		pgSQL.cache.Add(cacheIndex, featureVersion.ID)
	}

	return featureVersion.ID, nil
}

// TODO(Quentin-M): Batch me
func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) {
	IDs := make([]int, 0, len(featureVersions))

	for i := 0; i < len(featureVersions); i++ {
		id, err := pgSQL.insertFeatureVersion(featureVersions[i])
		if err != nil {
			return IDs, err
		}
		IDs = append(IDs, id)
	}

	return IDs, nil
}

type vulnerabilityAffectsFeatureVersion struct {
	vulnerabilityID int
	fixedInID       int
	fixedInVersion  types.Version
}

func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.FeatureVersion) error {
	// Select every vulnerability and the fixed version that affect this Feature.
	// TODO(Quentin-M): LIMIT
	rows, err := tx.Query(getQuery("s_vulnerability_fixedin_feature"), featureVersion.Feature.ID)
	if err != nil {
		return handleError("s_vulnerability_fixedin_feature", err)
	}
	defer rows.Close()

	var affects []vulnerabilityAffectsFeatureVersion
	for rows.Next() {
		var affect vulnerabilityAffectsFeatureVersion

		err := rows.Scan(&affect.fixedInID, &affect.vulnerabilityID, &affect.fixedInVersion)
		if err != nil {
			return handleError("s_vulnerability_fixedin_feature.Scan()", err)
		}

		if featureVersion.Version.Compare(affect.fixedInVersion) < 0 {
			// The version of the FeatureVersion we are inserting is lower than the fixed version on this
			// Vulnerability, thus, this FeatureVersion is affected by it.
			affects = append(affects, affect)
		}
	}
	if err = rows.Err(); err != nil {
		return handleError("s_vulnerability_fixedin_feature.Rows()", err)
	}
	rows.Close()

	// Insert into Vulnerability_Affects_FeatureVersion.
	for _, affect := range affects {
		// TODO(Quentin-M): Batch me.
		_, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), affect.vulnerabilityID,
			featureVersion.ID, affect.fixedInID)
		if err != nil {
			return handleError("i_vulnerability_affects_featureversion", err)
		}
	}

	return nil
}
<|endoftext|>