<commit_before>package serverlog

import (
	"fmt"
	"github.com/mgutz/ansi"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"
	"syscall"
	"time"
)

var (
	logToConsol = false // should log to consol
	logToFile   = false // should log to file
	logDir      = ""    // directory in which to store logfiles
	maxDays     = -1    // total number of logfiles at any time
	// if exeeded will delete oldest

	//log queue
	logChan     = make(chan logItem, 100)
	logNameChan = make(chan string, 2)
	killChan    = make(chan bool, 1)

	//colour functions
	startupColourFunc = ansi.ColorFunc("green+b:black")
	fatalColourFunc   = ansi.ColorFunc("red+b:black")
	generalColourFunc = ansi.ColorFunc("blue+b:black")
	warningColourFunc = ansi.ColorFunc("yellow+b:black")
)

// The clolour type enumerator
const (
	startupColour int = iota
	fatalColour
	generalColour
	warningColour
)

// Init initialises the srvlog package. if either consoleLog or fileLog
// is true it will start the logger in another gorutine ready to log
func Init(consolLog, fileLog bool, maxLogDays int, pathToLogDir string) {
	logToConsol = consolLog
	logToFile = fileLog
	logDir = pathToLogDir
	maxDays = maxLogDays

	if logToFile {
		// make sure log directory exists
		info, err := os.Stat(logDir)
		if err != nil {
			if os.IsNotExist(err) {
				log.Fatalln("The directory specified to serverlog does not exist.")
			}
			log.Fatalln(err)
		}
		if !info.IsDir() {
			log.Fatalln("The path specified to serverlog is not a directory.")
		}

		// make sure have premissions to log directory
		err = syscall.Access(logDir, syscall.O_RDWR)
		if err != nil {
			log.Fatalln("Serverlog needs read and write premissions to the specified directory.")
		}

		// manage logfile names and number of logfiles at any one time
		// only needed if logging to files
		go logFileOverseer()
	}

	go listen()
}

// Startup used to log the startup message
// example "Startup("Server listening on port:", PORT)"
func Startup(args ...interface{}) {
	logItem := logItem{
		prefix:           "STARTUP:",
		prefixColourFunc: startupColour,
		content:          args,
	}
	logChan <- logItem
}

// Fatal is used to log something a server killing circumstance
// same as log.Fatalln()
// This will terminate the process with an exit code of 1
func Fatal(args ...interface{}) {
	logItem := logItem{
		prefix:           "FATAL:  ",
		prefixColourFunc: fatalColour,
		content:          args,
	}
	logChan <- logItem
}

// General is used to log general stuff
func General(args ...interface{}) {
	logItem := logItem{
		prefix:           "GENERAL:",
		prefixColourFunc: generalColour,
		content:          args,
	}
	logChan <- logItem
}

// Warning is used to log warnings
func Warning(args ...interface{}) {
	logItem := logItem{
		prefix:           "WARNING:",
		prefixColourFunc: warningColour,
		content:          args,
	}
	logChan <- logItem
}

// Kill will terminate the listener and logFileOverseer
func Kill() {
	killChan <- false
	logToConsol = false
	logToFile = false
}

// logitem is the struct passed to the logger function
type logItem struct {
	prefix           string
	prefixColourFunc int
	content          []interface{}
}

// listen is the listener which runs in its own gorutine and logs messages
func listen() {
	currentLogPath := path.Join(logDir, time.Now().Format("02-01-2006-{csrv}.log"))

	for {
		select {
		case item := <-logChan:
			writeToConsole(item)
			writeToFile(item, currentLogPath)
			if item.prefixColourFunc == fatalColour {
				os.Exit(1)
			}
		case newLogPath := <-logNameChan:
			currentLogPath = newLogPath
		case <-killChan:
			return
		}
	}
}

// writeToFile logs a message to the logfile
func writeToFile(item logItem, logPath string) {
	if logToFile {
		file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
		defer file.Close()
		if err != nil {
			log.Fatalln(err)
		}

		line := time.Now().Format("15:04:05") + " " + item.prefix + " " + fmt.Sprintln(item.content)
		_, err = file.WriteString(line)
		if err != nil {
			log.Fatalln(err)
		}
	}
}

// writeToConsole logs a message to the console
func writeToConsole(item logItem) {
	if logToConsol {
		fmt.Print(time.Now().Format("15:04:05"), colourInText(item.prefix, item.prefixColourFunc), " ")
		fmt.Println(item.content...)
	}
}

// logFileOverseer makes sure that there are never more logfiles for each day than
// maxLogDays. also makes sure to update the listener as to new logfile names
// for each day.
func logFileOverseer() {
	for {
		in24Hr := time.Now().AddDate(0, 0, 1) // AddDate is used in case the next day is next month or year
		tomorrow := time.Date(in24Hr.Year(), in24Hr.Month(), in24Hr.Day(), 0, 0, 0, 0, in24Hr.Location())
		timeToWait := tomorrow.Sub(time.Now())
		newDay := time.After(timeToWait)

		select {
		case <-newDay:
			// tell listener new logfile name.
			logNameChan <- path.Join(logDir, time.Now().Format("02-01-2006-{csrv}.log"))

			if maxDays > 0 {
				files, err := ioutil.ReadDir(logDir)
				if err != nil {
					Warning("Server log failed to read from log directory :", err)
					break
				}

				logs := make([]string, 0, maxDays*2)
				index := 0
				for _, file := range files {
					if strings.Contains(file.Name(), "{csrv}") {
						logs[index] = file.Name()
						index++
					}
				}
				sort.Strings(logs)

				numberLogsLeft := len(logs)
				for i := 0; (numberLogsLeft > maxDays) && (i < len(logs)); i++ {
					logToDelete := path.Join(logDir, logs[i])
					err := os.Remove(logToDelete)
					if err != nil {
						Warning("Server log failed to delete logfile :", logToDelete, ":", err)
						break
					}
					numberLogsLeft--
				}
			}
		case <-killChan:
			return
		}
	}
}

// colourInText returns the text coloured in based on the colour func enumerator
func colourInText(text string, colourFunc int) string {
	switch colourFunc {
	case startupColour:
		return startupColourFunc(text)
	case fatalColour:
		return fatalColourFunc(text)
	case generalColour:
		return generalColourFunc(text)
	case warningColour:
		return warningColourFunc(text)
	default:
		return text
	}
}
<commit_msg>Fixed bug where there would always be 1 more logfile than specified<commit_after>package serverlog

import (
	"fmt"
	"github.com/mgutz/ansi"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"
	"syscall"
	"time"
)

var (
	logToConsol = false // should log to consol
	logToFile   = false // should log to file
	logDir      = ""    // directory in which to store logfiles
	maxDays     = -1    // total number of logfiles at any time
	// if exeeded will delete oldest

	//log queue
	logChan     = make(chan logItem, 100)
	logNameChan = make(chan string, 2)
	killChan    = make(chan bool, 1)

	//colour functions
	startupColourFunc = ansi.ColorFunc("green+b:black")
	fatalColourFunc   = ansi.ColorFunc("red+b:black")
	generalColourFunc = ansi.ColorFunc("blue+b:black")
	warningColourFunc = ansi.ColorFunc("yellow+b:black")
)

// The clolour type enumerator
const (
	startupColour int = iota
	fatalColour
	generalColour
	warningColour
)

// Init initialises the srvlog package. if either consoleLog or fileLog
// is true it will start the logger in another gorutine ready to log
func Init(consolLog, fileLog bool, maxLogDays int, pathToLogDir string) {
	logToConsol = consolLog
	logToFile = fileLog
	logDir = pathToLogDir
	maxDays = maxLogDays

	if logToFile {
		// make sure log directory exists
		info, err := os.Stat(logDir)
		if err != nil {
			if os.IsNotExist(err) {
				log.Fatalln("The directory specified to serverlog does not exist.")
			}
			log.Fatalln(err)
		}
		if !info.IsDir() {
			log.Fatalln("The path specified to serverlog is not a directory.")
		}

		// make sure have premissions to log directory
		err = syscall.Access(logDir, syscall.O_RDWR)
		if err != nil {
			log.Fatalln("Serverlog needs read and write premissions to the specified directory.")
		}

		// manage logfile names and number of logfiles at any one time
		// only needed if logging to files
		go logFileOverseer()
	}

	go listen()
}

// Startup used to log the startup message
// example "Startup("Server listening on port:", PORT)"
func Startup(args ...interface{}) {
	logItem := logItem{
		prefix:           "STARTUP:",
		prefixColourFunc: startupColour,
		content:          args,
	}
	logChan <- logItem
}

// Fatal is used to log something a server killing circumstance
// same as log.Fatalln()
// This will terminate the process with an exit code of 1
func Fatal(args ...interface{}) {
	logItem := logItem{
		prefix:           "FATAL:  ",
		prefixColourFunc: fatalColour,
		content:          args,
	}
	logChan <- logItem
}

// General is used to log general stuff
func General(args ...interface{}) {
	logItem := logItem{
		prefix:           "GENERAL:",
		prefixColourFunc: generalColour,
		content:          args,
	}
	logChan <- logItem
}

// Warning is used to log warnings
func Warning(args ...interface{}) {
	logItem := logItem{
		prefix:           "WARNING:",
		prefixColourFunc: warningColour,
		content:          args,
	}
	logChan <- logItem
}

// Kill will terminate the listener and logFileOverseer
func Kill() {
	killChan <- false
	logToConsol = false
	logToFile = false
}

// logitem is the struct passed to the logger function
type logItem struct {
	prefix           string
	prefixColourFunc int
	content          []interface{}
}

// listen is the listener which runs in its own gorutine and logs messages
func listen() {
	currentLogPath := path.Join(logDir, time.Now().Format("02-01-2006-{csrv}.log"))

	for {
		select {
		case item := <-logChan:
			writeToConsole(item)
			writeToFile(item, currentLogPath)
			if item.prefixColourFunc == fatalColour {
				os.Exit(1)
			}
		case newLogPath := <-logNameChan:
			currentLogPath = newLogPath
		case <-killChan:
			return
		}
	}
}

// writeToFile logs a message to the logfile
func writeToFile(item logItem, logPath string) {
	if logToFile {
		file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
		defer file.Close()
		if err != nil {
			log.Fatalln(err)
		}

		line := time.Now().Format("15:04:05") + " " + item.prefix + " " + fmt.Sprintln(item.content)
		_, err = file.WriteString(line)
		if err != nil {
			log.Fatalln(err)
		}
	}
}

// writeToConsole logs a message to the console
func writeToConsole(item logItem) {
	if logToConsol {
		fmt.Print(time.Now().Format("15:04:05"), colourInText(item.prefix, item.prefixColourFunc), " ")
		fmt.Println(item.content...)
	}
}

// logFileOverseer makes sure that there are never more logfiles for each day than
// maxLogDays. also makes sure to update the listener as to new logfile names
// for each day.
func logFileOverseer() {
	for {
		in24Hr := time.Now().AddDate(0, 0, 1) // AddDate is used in case the next day is next month or year
		tomorrow := time.Date(in24Hr.Year(), in24Hr.Month(), in24Hr.Day(), 0, 0, 0, 0, in24Hr.Location())
		timeToWait := tomorrow.Sub(time.Now())
		newDay := time.After(timeToWait)

		select {
		case <-newDay:
			newLogFile := path.Join(logDir, tomorrow.Format("02-01-2006-{csrv}.log"))

			// create new logfile
			_, err := os.Create(newLogFile)
			if err != nil {
				Warning("Serverlog failed to create new logfile:", newLogFile, ":", err)
				break
			}

			// tell listener new logfile name.
			logNameChan <- newLogFile

			if maxDays > 0 {
				files, err := ioutil.ReadDir(logDir)
				if err != nil {
					Warning("Server log failed to read from log directory :", err)
					break
				}

				logs := make([]string, 0, maxDays*2)
				index := 0
				for _, file := range files {
					if strings.Contains(file.Name(), "{csrv}") {
						logs[index] = file.Name()
						index++
					}
				}
				sort.Strings(logs)

				numberLogsLeft := len(logs)
				for i := 0; (numberLogsLeft > maxDays) && (i < len(logs)); i++ {
					logToDelete := path.Join(logDir, logs[i])
					err := os.Remove(logToDelete)
					if err != nil {
						Warning("Server log failed to delete logfile :", logToDelete, ":", err)
						break
					}
					numberLogsLeft--
				}
			}
		case <-killChan:
			return
		}
	}
}

// colourInText returns the text coloured in based on the colour func enumerator
func colourInText(text string, colourFunc int) string {
	switch colourFunc {
	case startupColour:
		return startupColourFunc(text)
	case fatalColour:
		return fatalColourFunc(text)
	case generalColour:
		return generalColourFunc(text)
	case warningColour:
		return warningColourFunc(text)
	default:
		return text
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/PuerkitoBio/gocrawl"
	"github.com/PuerkitoBio/goquery"

	"gopkg.in/yaml.v2"
)

type config struct {
	BaseURL  string             `yaml:"baseURL"`
	Formats  map[string]format  `yaml:"formats"`
	Elements map[string]Element `yaml:"elements"`
}

type format struct {
	ID       string `yaml:"ext"`
	Loc      string `yaml:"loc"`
	BasePath string `yaml:"basepath,omitempty"`
}

type Element struct {
	ID     string   `yaml:"id"`
	File   string   `yaml:"file,omitempty"`
	Meta   bool     `yaml:"meta,omitempty"`
	Name   string   `yaml:"name"`
	Files  []string `yaml:"files,omitempty"`
	Parent string   `yaml:"parent,omitempty"`
}

type ElementSlice map[string]Element

func (e ElementSlice) Generate(myConfig *config) ([]byte, error) {

	myConfig.Elements = e
	return yaml.Marshal(myConfig)
}

type Ext struct {
	*gocrawl.DefaultExtender
	Elements ElementSlice
}

func (e *Ext) parse_geofabrik(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	var thisElement Element
	downloadMain := doc.Find("div#download-main")
	parent, haveParent := doc.Find("p a").Attr("href")
	if haveParent && strings.Index(parent, "http://www.geofabrik.de/") == -1 {
		parent = parent[0 : len(parent)-5] // remove ".html"
		if parent == "index" {             //first level
			parent = ""
		} else {
			temp := strings.Split(parent, "/")
			parent = temp[len(temp)-1]
		}
		thisElement.Parent = parent
		for element := range downloadMain.Nodes {
			singleElement := downloadMain.Eq(element)
			name := singleElement.Find("h2").Text()
			thisElement.Name = name
			index := 0
			li := singleElement.Find("li")
			for el := range li.Nodes {
				myel := li.Eq(el)
				linkval, link := myel.Find("a").Attr("href")
				if link {
					switch index {
					case 0: // osm.pbf
						thisElement.ID = linkval[0 : len(linkval)-15]
						thisElement.Files = append(thisElement.Files, "osm.pbf")
					case 1: // shp.zip
						thisElement.Files = append(thisElement.Files, "shp.zip")
					case 2: // osm.bz2
						thisElement.Files = append(thisElement.Files, "osm.bz2")
					case 3: // osh.pbf
						thisElement.Files = append(thisElement.Files, "osh.pbf")
					case 4: // poly
						thisElement.Files = append(thisElement.Files, "poly")
					case 5: //-updates
						thisElement.Files = append(thisElement.Files, "state")
					}
				}
				index++
			}
		}
		if len(thisElement.Files) == 0 {
			thisElement.Meta = true
		}
		//Exceptions!
		// Only Georgia (EU and US)
		if thisElement.ID == "georgia" {
			thisElement.File = "georgia"
			if thisElement.Parent == "europe" {
				thisElement.Name = "Georgia (Europe country)"
				thisElement.ID = "georgia-eu"
			} else {
				thisElement.Name = "Georgia (US State)"
				thisElement.ID = "georgia-us"
			}
		}
		e.Elements[thisElement.ID] = thisElement
	}
	return nil, true
}

func (e *Ext) mergeElement(element *Element) {
	if cE, ok := e.Elements[element.ID]; ok {
		//cE := &e.Elements[element.ID]

		if cE.Parent != element.Parent {
			panic(fmt.Sprintln("Error! : Parent mismatch!"))
		}
		cE.Files = append(cE.Files, element.Files...)
		if len(cE.Files) == 0 {
			cE.Meta = true
		} else {
			cE.Meta = false
		}
		e.Elements[element.ID] = cE
	} else {
		e.Elements[element.ID] = *element
	}
}

func (e *Ext) parse_dosmfr(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	//var thisElement Element
	parent := doc.Find("h1").Text()
	fmt.Println(parent)

	list := doc.Find("table tr")
	for line := range list.Nodes {
		singleElement := list.Eq(line)
		lien := singleElement.Find("a")
		//index := 0
		for aa := range lien.Nodes {
			var element Element
			element.Parent = parent
			a := lien.Eq(aa)
			vallink, link := a.Attr("href")
			if link {
				if strings.Index(vallink, "/") != -1 {
					// /!\ meta! or levelup
					if vallink[0:1] != "/" {
						//meta
						// looking if already in e.Elements
						element.Meta = true
						t := strings.Split(vallink, "/")
						element.ID = t[len(t)-1]
					}
				}
			}
			e.mergeElement(&element)
		}
	}
	return nil, true
}

func (e *Ext) Visit(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	fmt.Printf("Visit: %s\n", ctx.URL())
	switch ctx.URL().Host {
	case "download.geofabrik.de":
		return e.parse_geofabrik(ctx, res, doc)
	case "download.openstreetmap.fr":
		return e.parse_dosmfr(ctx, res, doc)
	default:
		panic(fmt.Sprintln("Panic! " + ctx.URL().Host + " is not supported!"))
		return nil, true
	}

}

func (e *Ext) Filter(ctx *gocrawl.URLContext, isVisited bool) bool {
	if isVisited {
		return false
	}
	if len(ctx.URL().RawQuery) != 0 {
		return false
	} else if strings.Index(ctx.URL().Path, "newshapes.html") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "technical.html") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "robots.txt") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "replication") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "cgi-bin") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".pdf") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".pbf") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".poly") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".bz2") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".zip") != -1 {
		return false
	} else if ctx.URL().Path[len(ctx.URL().Path)-1:] == "/" {
		return true
	} else if strings.Index(ctx.URL().Path, ".html") != -1 {
		return true
		//	} else if ctx.URL().Path[len(ctx.URL().Path)-8:] == "-updates" {
		//		return false
		//	} else {
		//		return false
	}
	return false
}

func generate(url string, fname string, myConfig *config) {
	ext := &Ext{&gocrawl.DefaultExtender{}, make(map[string]Element)}
	// Set custom options
	opts := gocrawl.NewOptions(ext)
	opts.CrawlDelay = 100 * time.Millisecond
	opts.LogFlags = gocrawl.LogError
  //	opts.LogFlags = gocrawl.LogAll
	opts.SameHostOnly = true //false
	opts.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36"
	opts.MaxVisits = 15000

	file := gocrawl.NewCrawlerWithOptions(opts)
	file.Run(url)
	out, _ := ext.Elements.Generate(myConfig)
	filename, _ := filepath.Abs(fname)
	err := ioutil.WriteFile(filename, out, 0644)
	if err != nil {
		log.Fatalln(" File error: %v ", err)
		os.Exit(1)
	}
}

func main() {

	//Generate geofabrik.yml
	var geofabrik config
	geofabrik.BaseURL = "http://download.geofabrik.de"
	geofabrik.Formats = make(map[string]format)
	geofabrik.Formats["osh.pbf"] = format{ID: "osh.pbf", Loc: ".osh.pbf"}
	geofabrik.Formats["osm.bz2"] = format{ID: "osm.bz2", Loc: "-latest.osm.bz2"}
	geofabrik.Formats["osm.pbf"] = format{ID: "osm.pbf", Loc: "-latest.osm.pbf"}
	geofabrik.Formats["poly"] = format{ID: "poly", Loc: ".poly"}
	geofabrik.Formats["state"] = format{ID: "state", Loc: ".state"}
	geofabrik.Formats["shp.zip"] = format{ID: "shp.zip", Loc: "-latest-free.shp.zip"}
	generate("http://download.geofabrik.de/", "geofabrik.yml", &geofabrik)
	var myConfig config
	myConfig.BaseURL = "http://download.openstreetmap.fr/extracts"
	myConfig.Formats = make(map[string]format)
	myConfig.Formats["osm.pbf"] = format{ID: "osm.pbf", Loc: ".osm.pbf"}
	myConfig.Formats["poly"] = format{ID: "poly", Loc: ".poly", BasePath: "../polygons/"}
	myConfig.Formats["state"] = format{ID: "state", Loc: ".state.txt"}
	//generate("http://download.openstreetmap.fr/", "openstreetmap_fr.yml", &myConfig)
}
<commit_msg>style: Linting<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/PuerkitoBio/gocrawl"
	"github.com/PuerkitoBio/goquery"

	"gopkg.in/yaml.v2"
)

type config struct {
	BaseURL  string             `yaml:"baseURL"`
	Formats  map[string]format  `yaml:"formats"`
	Elements map[string]Element `yaml:"elements"`
}

type format struct {
	ID       string `yaml:"ext"`
	Loc      string `yaml:"loc"`
	BasePath string `yaml:"basepath,omitempty"`
}

// Element define file path, name ...
type Element struct {
	ID     string   `yaml:"id"`
	File   string   `yaml:"file,omitempty"`
	Meta   bool     `yaml:"meta,omitempty"`
	Name   string   `yaml:"name"`
	Files  []string `yaml:"files,omitempty"`
	Parent string   `yaml:"parent,omitempty"`
}

// ElementSlice contain all Elements
type ElementSlice map[string]Element

// Generate make the slice which contain all Elements
func (e ElementSlice) Generate(myConfig *config) ([]byte, error) {

	myConfig.Elements = e
	return yaml.Marshal(myConfig)
}

// Ext simple struct for managing ElementSlice and crawler
type Ext struct {
	*gocrawl.DefaultExtender
	Elements ElementSlice
}

func (e *Ext) parseGeofabrik(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	var thisElement Element
	downloadMain := doc.Find("div#download-main")
	parent, haveParent := doc.Find("p a").Attr("href")
	if haveParent && strings.Index(parent, "http://www.geofabrik.de/") == -1 {
		parent = parent[0 : len(parent)-5] // remove ".html"
		if parent == "index" {             //first level
			parent = ""
		} else {
			temp := strings.Split(parent, "/")
			parent = temp[len(temp)-1]
		}
		thisElement.Parent = parent
		for element := range downloadMain.Nodes {
			singleElement := downloadMain.Eq(element)
			name := singleElement.Find("h2").Text()
			thisElement.Name = name
			index := 0
			li := singleElement.Find("li")
			for el := range li.Nodes {
				myel := li.Eq(el)
				linkval, link := myel.Find("a").Attr("href")
				if link {
					switch index {
					case 0: // osm.pbf
						thisElement.ID = linkval[0 : len(linkval)-15]
						thisElement.Files = append(thisElement.Files, "osm.pbf")
					case 1: // shp.zip
						thisElement.Files = append(thisElement.Files, "shp.zip")
					case 2: // osm.bz2
						thisElement.Files = append(thisElement.Files, "osm.bz2")
					case 3: // osh.pbf
						thisElement.Files = append(thisElement.Files, "osh.pbf")
					case 4: // poly
						thisElement.Files = append(thisElement.Files, "poly")
					case 5: //-updates
						thisElement.Files = append(thisElement.Files, "state")
					}
				}
				index++
			}
		}
		if len(thisElement.Files) == 0 {
			thisElement.Meta = true
		}
		//Exceptions!
		// Only Georgia (EU and US)
		if thisElement.ID == "georgia" {
			thisElement.File = "georgia"
			if thisElement.Parent == "europe" {
				thisElement.Name = "Georgia (Europe country)"
				thisElement.ID = "georgia-eu"
			} else {
				thisElement.Name = "Georgia (US State)"
				thisElement.ID = "georgia-us"
			}
		}
		e.Elements[thisElement.ID] = thisElement
	}
	return nil, true
}

func (e *Ext) mergeElement(element *Element) {
	if cE, ok := e.Elements[element.ID]; ok {
		//cE := &e.Elements[element.ID]

		if cE.Parent != element.Parent {
			panic(fmt.Sprintln("Error! : Parent mismatch!"))
		}
		cE.Files = append(cE.Files, element.Files...)
		if len(cE.Files) == 0 {
			cE.Meta = true
		} else {
			cE.Meta = false
		}
		e.Elements[element.ID] = cE
	} else {
		e.Elements[element.ID] = *element
	}
}

func (e *Ext) parseOSMfr(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	//var thisElement Element
	parent := doc.Find("h1").Text()
	fmt.Println(parent)

	list := doc.Find("table tr")
	for line := range list.Nodes {
		singleElement := list.Eq(line)
		lien := singleElement.Find("a")
		//index := 0
		for aa := range lien.Nodes {
			var element Element
			element.Parent = parent
			a := lien.Eq(aa)
			vallink, link := a.Attr("href")
			if link {
				if strings.Index(vallink, "/") != -1 {
					// /!\ meta! or levelup
					if vallink[0:1] != "/" {
						//meta
						// looking if already in e.Elements
						element.Meta = true
						t := strings.Split(vallink, "/")
						element.ID = t[len(t)-1]
					}
				}
			}
			e.mergeElement(&element)
		}
	}
	return nil, true
}

// Visit launch right crawler
func (e *Ext) Visit(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	fmt.Printf("Visit: %s\n", ctx.URL())
	switch ctx.URL().Host {
	case "download.geofabrik.de":
		return e.parseGeofabrik(ctx, res, doc)
	case "download.openstreetmap.fr":
		return e.parseOSMfr(ctx, res, doc)
	default:
		panic(fmt.Sprintln("Panic! " + ctx.URL().Host + " is not supported!"))
	}

}

// Filter remove non needed urls.
func (e *Ext) Filter(ctx *gocrawl.URLContext, isVisited bool) bool {
	if isVisited {
		return false
	}
	if len(ctx.URL().RawQuery) != 0 {
		return false
		// TODO: refactorize? Use config file?
	} else if strings.Index(ctx.URL().Path, "newshapes.html") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "technical.html") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "robots.txt") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "replication") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, "cgi-bin") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".pdf") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".pbf") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".poly") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".bz2") != -1 {
		return false
	} else if strings.Index(ctx.URL().Path, ".zip") != -1 {
		return false
	} else if ctx.URL().Path[len(ctx.URL().Path)-1:] == "/" {
		return true
	} else if strings.Index(ctx.URL().Path, ".html") != -1 {
		return true
		//	} else if ctx.URL().Path[len(ctx.URL().Path)-8:] == "-updates" {
		//		return false
		//	} else {
		//		return false
	}
	return false
}

func generate(url string, fname string, myConfig *config) {
	ext := &Ext{&gocrawl.DefaultExtender{}, make(map[string]Element)}
	// Set custom options
	opts := gocrawl.NewOptions(ext)
	opts.CrawlDelay = 100 * time.Millisecond
	opts.LogFlags = gocrawl.LogError
	//	opts.LogFlags = gocrawl.LogAll
	opts.SameHostOnly = true //false
	opts.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36"
	opts.MaxVisits = 15000

	file := gocrawl.NewCrawlerWithOptions(opts)
	file.Run(url)
	out, _ := ext.Elements.Generate(myConfig)
	filename, _ := filepath.Abs(fname)
	err := ioutil.WriteFile(filename, out, 0644)
	if err != nil {
		log.Fatalln(" File error: %v ", err)
		os.Exit(1)
	}
}

func main() {

	//Generate geofabrik.yml
	var geofabrik config
	geofabrik.BaseURL = "http://download.geofabrik.de"
	geofabrik.Formats = make(map[string]format)
	geofabrik.Formats["osh.pbf"] = format{ID: "osh.pbf", Loc: ".osh.pbf"}
	geofabrik.Formats["osm.bz2"] = format{ID: "osm.bz2", Loc: "-latest.osm.bz2"}
	geofabrik.Formats["osm.pbf"] = format{ID: "osm.pbf", Loc: "-latest.osm.pbf"}
	geofabrik.Formats["poly"] = format{ID: "poly", Loc: ".poly"}
	geofabrik.Formats["state"] = format{ID: "state", Loc: ".state"}
	geofabrik.Formats["shp.zip"] = format{ID: "shp.zip", Loc: "-latest-free.shp.zip"}
	generate("http://download.geofabrik.de/", "geofabrik.yml", &geofabrik)
	var myConfig config
	myConfig.BaseURL = "http://download.openstreetmap.fr/extracts"
	myConfig.Formats = make(map[string]format)
	myConfig.Formats["osm.pbf"] = format{ID: "osm.pbf", Loc: ".osm.pbf"}
	myConfig.Formats["poly"] = format{ID: "poly", Loc: ".poly", BasePath: "../polygons/"}
	myConfig.Formats["state"] = format{ID: "state", Loc: ".state.txt"}
	//generate("http://download.openstreetmap.fr/", "openstreetmap_fr.yml", &myConfig)
}
<|endoftext|>


<commit_before>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package declarative

import (
	"github.com/lxn/walk"
)

type Dialog struct {
	AssignTo           **walk.Dialog
	Name               string
	Disabled           bool
	Hidden             bool
	Font               Font
	MinSize            Size
	MaxSize            Size
	ContextMenuActions []*walk.Action
	Title              string
	Size               Size
	DataBinder         DataBinder
	Layout             Layout
	Children           []Widget
	DefaultButton      **walk.PushButton
	CancelButton       **walk.PushButton
}

func (d Dialog) Create(owner walk.RootWidget) error {
	w, err := walk.NewDialog(owner)
	if err != nil {
		return err
	}

	tlwi := topLevelWindowInfo{
		Name:               d.Name,
		Disabled:           d.Disabled,
		Hidden:             d.Hidden,
		Font:               d.Font,
		MinSize:            d.MinSize,
		MaxSize:            d.MaxSize,
		ContextMenuActions: d.ContextMenuActions,
		DataBinder:         d.DataBinder,
		Layout:             d.Layout,
		Children:           d.Children,
	}

	var db *walk.DataBinder
	if d.DataBinder.AssignTo == nil {
		d.DataBinder.AssignTo = &db
	}

	return InitWidget(tlwi, w, func() error {
		if err := w.SetTitle(d.Title); err != nil {
			return err
		}

		if err := w.SetSize(d.Size.toW()); err != nil {
			return err
		}

		if d.DefaultButton != nil {
			if err := w.SetDefaultButton(*d.DefaultButton); err != nil {
				return err
			}

			db := *d.DataBinder.AssignTo
			if db != nil {
				db.CanSubmitChanged().Attach(func() {
					(*d.DefaultButton).SetEnabled(db.CanSubmit())
				})
			}
		}
		if d.CancelButton != nil {
			if err := w.SetCancelButton(*d.CancelButton); err != nil {
				return err
			}
		}

		if d.AssignTo != nil {
			*d.AssignTo = w
		}

		return nil
	})
}
<commit_msg>declarative/Dialog: Add Run method<commit_after>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package declarative

import (
	"github.com/lxn/walk"
)

type Dialog struct {
	AssignTo           **walk.Dialog
	Name               string
	Disabled           bool
	Hidden             bool
	Font               Font
	MinSize            Size
	MaxSize            Size
	ContextMenuActions []*walk.Action
	Title              string
	Size               Size
	DataBinder         DataBinder
	Layout             Layout
	Children           []Widget
	DefaultButton      **walk.PushButton
	CancelButton       **walk.PushButton
}

func (d Dialog) Create(owner walk.RootWidget) error {
	w, err := walk.NewDialog(owner)
	if err != nil {
		return err
	}

	tlwi := topLevelWindowInfo{
		Name:               d.Name,
		Disabled:           d.Disabled,
		Hidden:             d.Hidden,
		Font:               d.Font,
		MinSize:            d.MinSize,
		MaxSize:            d.MaxSize,
		ContextMenuActions: d.ContextMenuActions,
		DataBinder:         d.DataBinder,
		Layout:             d.Layout,
		Children:           d.Children,
	}

	var db *walk.DataBinder
	if d.DataBinder.AssignTo == nil {
		d.DataBinder.AssignTo = &db
	}

	return InitWidget(tlwi, w, func() error {
		if err := w.SetTitle(d.Title); err != nil {
			return err
		}

		if err := w.SetSize(d.Size.toW()); err != nil {
			return err
		}

		if d.DefaultButton != nil {
			if err := w.SetDefaultButton(*d.DefaultButton); err != nil {
				return err
			}

			db := *d.DataBinder.AssignTo
			if db != nil {
				db.CanSubmitChanged().Attach(func() {
					(*d.DefaultButton).SetEnabled(db.CanSubmit())
				})
			}
		}
		if d.CancelButton != nil {
			if err := w.SetCancelButton(*d.CancelButton); err != nil {
				return err
			}
		}

		if d.AssignTo != nil {
			*d.AssignTo = w
		}

		return nil
	})
}

func (d Dialog) Run(owner walk.RootWidget) (int, error) {
	var w *walk.Dialog

	if d.AssignTo == nil {
		d.AssignTo = &w
	}

	if err := d.Create(owner); err != nil {
		return 0, err
	}

	return (*d.AssignTo).Run(), nil
}
<|endoftext|>


<commit_before>package constants

const (
	BOSHURL        = "https://s3.amazonaws.com/bbl-precompiled-bosh-releases/release-bosh-257.15-on-ubuntu-trusty-stemcell-3263.7.tgz"
	BOSHSHA1       = ""
	BOSHAWSCPIURL  = "https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=60"
	BOSHAWSCPISHA1 = "8e40a9ff892204007889037f094a1b0d23777058"
	StemcellURL    = "https://d26ekeud912fhb.cloudfront.net/bosh-stemcell/aws/light-bosh-stemcell-3263.7-aws-xen-hvm-ubuntu-trusty-go_agent.tgz"
	StemcellSHA1   = "6f09d8ed6514a22eaafefa97dcd916f1fc357631"
)
<commit_msg>Update constants<commit_after>package constants

const (
	BOSHURL        = "https://s3.amazonaws.com/bbl-precompiled-bosh-releases/release-bosh-257.15-on-ubuntu-trusty-stemcell-3263.7.1.tgz"
	BOSHSHA1       = "e72406ddbfc4ebb7e389544c0c48802780a54f1c"
	BOSHAWSCPIURL  = "https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=60"
	BOSHAWSCPISHA1 = "8e40a9ff892204007889037f094a1b0d23777058"
	StemcellURL    = "https://d26ekeud912fhb.cloudfront.net/bosh-stemcell/aws/light-bosh-stemcell-3263.7-aws-xen-hvm-ubuntu-trusty-go_agent.tgz"
	StemcellSHA1   = "6f09d8ed6514a22eaafefa97dcd916f1fc357631"
)
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"strconv"

	"github.com/knative/pkg/apis"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/util/intstr"
)

// Validate inspects and validates ClusterIngress object.
func (ci *ClusterIngress) Validate() *apis.FieldError {
	return ci.Spec.Validate().ViaField("spec")
}

// Validate inspects and validates IngressSpec object.
func (spec *IngressSpec) Validate() *apis.FieldError {
	// Spec must not be empty.
	if equality.Semantic.DeepEqual(spec, &IngressSpec{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// Spec must have at least one rule.
	if len(spec.Rules) == 0 {
		all = all.Also(apis.ErrMissingField("rules"))
	}
	// Validate each rule.
	for idx, rule := range spec.Rules {
		all = all.Also(rule.Validate().ViaFieldIndex("rules", idx))
	}
	// TLS settings are optional.  However, all provided settings should be valid.
	for idx, tls := range spec.TLS {
		all = all.Also(tls.Validate().ViaFieldIndex("tls", idx))
	}
	return all
}

// Validate inspects and validates ClusterIngressRule object.
func (r *ClusterIngressRule) Validate() *apis.FieldError {
	// Provided rule must not be empty.
	if equality.Semantic.DeepEqual(r, &ClusterIngressRule{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	if r.HTTP == nil {
		all = all.Also(apis.ErrMissingField("http"))
	} else {
		all = all.Also(r.HTTP.Validate().ViaField("http"))
	}
	return all
}

// Validate inspects and validates HTTPClusterIngressRuleValue object.
func (h *HTTPClusterIngressRuleValue) Validate() *apis.FieldError {
	if len(h.Paths) == 0 {
		return apis.ErrMissingField("paths")
	}
	var all *apis.FieldError
	for idx, path := range h.Paths {
		all = all.Also(path.Validate().ViaFieldIndex("paths", idx))
	}
	return all
}

// Validate inspects and validates HTTPClusterIngressPath object.
func (h HTTPClusterIngressPath) Validate() *apis.FieldError {
	// Provided rule must not be empty.
	if equality.Semantic.DeepEqual(h, HTTPClusterIngressPath{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// Must provide as least one split.
	if len(h.Splits) == 0 {
		all = all.Also(apis.ErrMissingField("splits"))
	} else {
		totalPct := 0
		for idx, split := range h.Splits {
			if err := split.Validate(); err != nil {
				return err.ViaFieldIndex("splits", idx)
			}
			totalPct += split.Percent
		}
		// If a single split is provided we allow missing Percent, and
		// interpret as 100%.
		if len(h.Splits) == 1 && totalPct == 0 {
			totalPct = 100
		}
		// Total traffic split percentage must sum up to 100%.
		if totalPct != 100 {
			all = all.Also(&apis.FieldError{
				Message: "Traffic split percentage must total to 100, but was " + strconv.Itoa(totalPct),
				Paths:   []string{"splits"},
			})
		}
	}
	if h.Retries != nil {
		all = all.Also(h.Retries.Validate().ViaField("retries"))
	}
	return all
}

// Validate inspects and validates HTTPClusterIngressPath object.
func (s ClusterIngressBackendSplit) Validate() *apis.FieldError {
	// Must not be empty.
	if equality.Semantic.DeepEqual(s, ClusterIngressBackendSplit{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// Percent must be between 0 and 100.
	if s.Percent < 0 || s.Percent > 100 {
		all = all.Also(apis.ErrInvalidValue(strconv.Itoa(s.Percent), "percent"))
	}
	return all.Also(s.ClusterIngressBackend.Validate())
}

// Validate inspects the fields of the type ClusterIngressBackend
// to determine if they are valid.
func (b ClusterIngressBackend) Validate() *apis.FieldError {
	// Must not be empty.
	if equality.Semantic.DeepEqual(b, ClusterIngressBackend{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	if b.ServiceNamespace == "" {
		all = all.Also(apis.ErrMissingField("serviceNamespace"))
	}
	if b.ServiceName == "" {
		all = all.Also(apis.ErrMissingField("serviceName"))
	}
	if equality.Semantic.DeepEqual(b.ServicePort, intstr.IntOrString{}) {
		all = all.Also(apis.ErrMissingField("servicePort"))
	}
	return all
}

// Validate inspects and validates HTTPRetry object.
func (r *HTTPRetry) Validate() *apis.FieldError {
	// Attempts must be greater than 0.
	if r.Attempts < 0 {
		return apis.ErrInvalidValue(strconv.Itoa(r.Attempts), "attempts")
	}
	return nil
}

// Validate inspects and validates ClusterIngressTLS object.
func (t *ClusterIngressTLS) Validate() *apis.FieldError {
	// Provided TLS setting must not be empty.
	if equality.Semantic.DeepEqual(t, &ClusterIngressTLS{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// SecretName and SecretNamespace must not be empty.
	if t.SecretName == "" {
		all = all.Also(apis.ErrMissingField("secretName"))
	}
	if t.SecretNamespace == "" {
		all = all.Also(apis.ErrMissingField("secretNamespace"))
	}
	return all
}
<commit_msg>use the new err invalid array value helper (#3063)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"strconv"

	"github.com/knative/pkg/apis"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/util/intstr"
)

// Validate inspects and validates ClusterIngress object.
func (ci *ClusterIngress) Validate() *apis.FieldError {
	return ci.Spec.Validate().ViaField("spec")
}

// Validate inspects and validates IngressSpec object.
func (spec *IngressSpec) Validate() *apis.FieldError {
	// Spec must not be empty.
	if equality.Semantic.DeepEqual(spec, &IngressSpec{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// Spec must have at least one rule.
	if len(spec.Rules) == 0 {
		all = all.Also(apis.ErrMissingField("rules"))
	}
	// Validate each rule.
	for idx, rule := range spec.Rules {
		all = all.Also(rule.Validate().ViaFieldIndex("rules", idx))
	}
	// TLS settings are optional.  However, all provided settings should be valid.
	for idx, tls := range spec.TLS {
		all = all.Also(tls.Validate().ViaFieldIndex("tls", idx))
	}
	return all
}

// Validate inspects and validates ClusterIngressRule object.
func (r *ClusterIngressRule) Validate() *apis.FieldError {
	// Provided rule must not be empty.
	if equality.Semantic.DeepEqual(r, &ClusterIngressRule{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	if r.HTTP == nil {
		all = all.Also(apis.ErrMissingField("http"))
	} else {
		all = all.Also(r.HTTP.Validate().ViaField("http"))
	}
	return all
}

// Validate inspects and validates HTTPClusterIngressRuleValue object.
func (h *HTTPClusterIngressRuleValue) Validate() *apis.FieldError {
	if len(h.Paths) == 0 {
		return apis.ErrMissingField("paths")
	}
	var all *apis.FieldError
	for idx, path := range h.Paths {
		all = all.Also(path.Validate().ViaFieldIndex("paths", idx))
	}
	return all
}

// Validate inspects and validates HTTPClusterIngressPath object.
func (h HTTPClusterIngressPath) Validate() *apis.FieldError {
	// Provided rule must not be empty.
	if equality.Semantic.DeepEqual(h, HTTPClusterIngressPath{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// Must provide as least one split.
	if len(h.Splits) == 0 {
		all = all.Also(apis.ErrMissingField("splits"))
	} else {
		totalPct := 0
		for idx, split := range h.Splits {
			if err := split.Validate(); err != nil {
				return err.ViaFieldIndex("splits", idx)
			}
			totalPct += split.Percent
		}
		// If a single split is provided we allow missing Percent, and
		// interpret as 100%.
		if len(h.Splits) == 1 && totalPct == 0 {
			totalPct = 100
		} else if totalPct != 100 {
			// Total traffic split percentage must sum up to 100%.
			all = all.Also(&apis.FieldError{
				Message: "Traffic split percentage must total to 100, but was " + strconv.Itoa(totalPct),
				Paths:   []string{"splits"},
			})
		}
	}
	if h.Retries != nil {
		all = all.Also(h.Retries.Validate().ViaField("retries"))
	}
	return all
}

// Validate inspects and validates HTTPClusterIngressPath object.
func (s ClusterIngressBackendSplit) Validate() *apis.FieldError {
	// Must not be empty.
	if equality.Semantic.DeepEqual(s, ClusterIngressBackendSplit{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// Percent must be between 0 and 100.
	if s.Percent < 0 || s.Percent > 100 {
		all = all.Also(apis.ErrInvalidValue(strconv.Itoa(s.Percent), "percent"))
	}
	return all.Also(s.ClusterIngressBackend.Validate())
}

// Validate inspects the fields of the type ClusterIngressBackend
// to determine if they are valid.
func (b ClusterIngressBackend) Validate() *apis.FieldError {
	// Must not be empty.
	if equality.Semantic.DeepEqual(b, ClusterIngressBackend{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	if b.ServiceNamespace == "" {
		all = all.Also(apis.ErrMissingField("serviceNamespace"))
	}
	if b.ServiceName == "" {
		all = all.Also(apis.ErrMissingField("serviceName"))
	}
	if equality.Semantic.DeepEqual(b.ServicePort, intstr.IntOrString{}) {
		all = all.Also(apis.ErrMissingField("servicePort"))
	}
	return all
}

// Validate inspects and validates HTTPRetry object.
func (r *HTTPRetry) Validate() *apis.FieldError {
	// Attempts must be greater than 0.
	if r.Attempts < 0 {
		return apis.ErrInvalidValue(strconv.Itoa(r.Attempts), "attempts")
	}
	return nil
}

// Validate inspects and validates ClusterIngressTLS object.
func (t *ClusterIngressTLS) Validate() *apis.FieldError {
	// Provided TLS setting must not be empty.
	if equality.Semantic.DeepEqual(t, &ClusterIngressTLS{}) {
		return apis.ErrMissingField(apis.CurrentField)
	}
	var all *apis.FieldError
	// SecretName and SecretNamespace must not be empty.
	if t.SecretName == "" {
		all = all.Also(apis.ErrMissingField("secretName"))
	}
	if t.SecretNamespace == "" {
		all = all.Also(apis.ErrMissingField("secretNamespace"))
	}
	return all
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package scheme

import (
	os "os"

	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
	announced "k8s.io/apimachinery/pkg/apimachinery/announced"
	registered "k8s.io/apimachinery/pkg/apimachinery/registered"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
)

var Scheme = runtime.NewScheme()
var Codecs = serializer.NewCodecFactory(Scheme)
var ParameterCodec = runtime.NewParameterCodec(Scheme)

var Registry = registered.NewOrDie(os.Getenv("KUBE_API_VERSIONS"))
var GroupFactoryRegistry = make(announced.APIGroupFactoryRegistry)

func init() {
	v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
	Install(GroupFactoryRegistry, Registry, Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	apiextensions.Install(groupFactoryRegistry, registry, scheme)
}
<commit_msg>generated<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package scheme

import (
	os "os"

	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
	registered "k8s.io/apimachinery/pkg/apimachinery/registered"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
)

var Scheme = runtime.NewScheme()
var Codecs = serializer.NewCodecFactory(Scheme)
var ParameterCodec = runtime.NewParameterCodec(Scheme)

var Registry = registered.NewOrDie(os.Getenv("KUBE_API_VERSIONS"))

func init() {
	v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
	Install(Registry, Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	apiextensions.Install(registry, scheme)
}
<|endoftext|>


<commit_before>//go:build linux || darwin
// +build linux darwin

package command

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/mount/unmount"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/hanwen/go-fuse/v2/fuse"
	"google.golang.org/grpc/reflection"
	"net"
	"net/http"
	"os"
	"os/user"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/grace"
)

func runMount(cmd *Command, args []string) bool {

	if *mountOptions.debug {
		go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil)
	}

	grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
	if *mountReadRetryTime < time.Second {
		*mountReadRetryTime = time.Second
	}
	util.RetryWaitTime = *mountReadRetryTime

	umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64)
	if umaskErr != nil {
		fmt.Printf("can not parse umask %s", *mountOptions.umaskString)
		return false
	}

	if len(args) > 0 {
		return false
	}

	return RunMount(&mountOptions, os.FileMode(umask))
}

func RunMount(option *MountOptions, umask os.FileMode) bool {

	// basic checks
	chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
	if chunkSizeLimitMB <= 0 {
		fmt.Printf("Please specify a reasonable buffer size.")
		return false
	}

	// try to connect to filer
	filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses()
	util.LoadConfiguration("security", false)
	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
	var cipher bool
	var err error
	for i := 0; i < 10; i++ {
		err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
			if err != nil {
				return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err)
			}
			cipher = resp.Cipher
			return nil
		})
		if err != nil {
			glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
			glog.V(0).Infof("wait for %d seconds ...", i+1)
			time.Sleep(time.Duration(i+1) * time.Second)
		}
	}
	if err != nil {
		glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
		return true
	}

	filerMountRootPath := *option.filerMountRootPath

	// clean up mount point
	dir := util.ResolvePath(*option.dir)
	if dir == "" {
		fmt.Printf("Please specify the mount directory via \"-dir\"")
		return false
	}

	unmount.Unmount(dir)

	// start on local unix socket
	if *option.localSocket == "" {
		mountDirHash := util.HashToInt32([]byte(dir))
		if mountDirHash < 0 {
			mountDirHash = -mountDirHash
		}
		*option.localSocket = fmt.Sprintf("/tmp/seaweefs-mount-%d.sock", mountDirHash)
		if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
			glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
		}
	}
	montSocketListener, err := net.Listen("unix", *option.localSocket)
	if err != nil {
		glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
	}

	// detect mount folder mode
	if *option.dirAutoCreate {
		os.MkdirAll(dir, os.FileMode(0777)&^umask)
	}
	fileInfo, err := os.Stat(dir)

	// collect uid, gid
	uid, gid := uint32(0), uint32(0)
	mountMode := os.ModeDir | 0777
	if err == nil {
		mountMode = os.ModeDir | os.FileMode(0777)&^umask
		uid, gid = util.GetFileUidGid(fileInfo)
		fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, mountMode)
	} else {
		fmt.Printf("can not stat %s\n", dir)
		return false
	}

	// detect uid, gid
	if uid == 0 {
		if u, err := user.Current(); err == nil {
			if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
				uid = uint32(parsedId)
			}
			if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
				gid = uint32(parsedId)
			}
			fmt.Printf("current uid=%d gid=%d\n", uid, gid)
		}
	}

	// mapping uid, gid
	uidGidMapper, err := meta_cache.NewUidGidMapper(*option.uidMap, *option.gidMap)
	if err != nil {
		fmt.Printf("failed to parse %s %s: %v\n", *option.uidMap, *option.gidMap, err)
		return false
	}

	// Ensure target mount point availability
	if isValid := checkMountPointAvailable(dir); !isValid {
		glog.Fatalf("Expected mount to still be active, target mount point: %s, please check!", dir)
		return true
	}

	serverFriendlyName := strings.ReplaceAll(*option.filer, ",", "+")

	// mount fuse
	fuseMountOptions := &fuse.MountOptions{
		AllowOther:               *option.allowOthers,
		Options:                  nil,
		MaxBackground:            128,
		MaxWrite:                 1024 * 1024 * 2,
		MaxReadAhead:             1024 * 1024 * 2,
		IgnoreSecurityLabels:     false,
		RememberInodes:           false,
		FsName:                   serverFriendlyName + ":" + filerMountRootPath,
		Name:                     "seaweedfs",
		SingleThreaded:           false,
		DisableXAttrs:            false,
		Debug:                    *option.debug,
		EnableLocks:              false,
		ExplicitDataCacheControl: false,
		DirectMount:              true,
		DirectMountFlags:         0,
		//SyncRead:                 false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
		//EnableAcl:                true,
	}
	if *option.nonempty {
		fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
	}
	if *option.readOnly {
		if runtime.GOOS == "darwin" {
			fuseMountOptions.Options = append(fuseMountOptions.Options, "rdonly")
		} else {
			fuseMountOptions.Options = append(fuseMountOptions.Options, "ro")
		}
	}
	if runtime.GOOS == "darwin" {
		// https://github-wiki-see.page/m/macfuse/macfuse/wiki/Mount-Options
		ioSizeMB := 1
		for ioSizeMB*2 <= *option.chunkSizeLimitMB && ioSizeMB*2 <= 32 {
			ioSizeMB *= 2
		}
		fuseMountOptions.Options = append(fuseMountOptions.Options, "daemon_timeout=600")
		fuseMountOptions.Options = append(fuseMountOptions.Options, "noapplexattr")
		// fuseMountOptions.Options = append(fuseMountOptions.Options, "novncache") // need to test effectiveness
		fuseMountOptions.Options = append(fuseMountOptions.Options, "slow_statfs")
		fuseMountOptions.Options = append(fuseMountOptions.Options, "volname="+serverFriendlyName)
		fuseMountOptions.Options = append(fuseMountOptions.Options, fmt.Sprintf("iosize=%d", ioSizeMB*1024*1024))
	}

	// find mount point
	mountRoot := filerMountRootPath
	if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
		mountRoot = mountRoot[0 : len(mountRoot)-1]
	}

	seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{
		MountDirectory:     dir,
		FilerAddresses:     filerAddresses,
		GrpcDialOption:     grpcDialOption,
		FilerMountRootPath: mountRoot,
		Collection:         *option.collection,
		Replication:        *option.replication,
		TtlSec:             int32(*option.ttlSec),
		DiskType:           types.ToDiskType(*option.diskType),
		ChunkSizeLimit:     int64(chunkSizeLimitMB) * 1024 * 1024,
		ConcurrentWriters:  *option.concurrentWriters,
		CacheDir:           *option.cacheDir,
		CacheSizeMB:        *option.cacheSizeMB,
		DataCenter:         *option.dataCenter,
		Quota:              int64(*option.collectionQuota) * 1024 * 1024,
		MountUid:           uid,
		MountGid:           gid,
		MountMode:          mountMode,
		MountCtime:         fileInfo.ModTime(),
		MountMtime:         time.Now(),
		Umask:              umask,
		VolumeServerAccess: *mountOptions.volumeServerAccess,
		Cipher:             cipher,
		UidGidMapper:       uidGidMapper,
	})

	server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
	if err != nil {
		glog.Fatalf("Mount fail: %v", err)
	}
	grace.OnInterrupt(func() {
		unmount.Unmount(dir)
	})

	grpcS := pb.NewGrpcServer()
	mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem)
	reflection.Register(grpcS)
	go grpcS.Serve(montSocketListener)

	seaweedFileSystem.StartBackgroundTasks()

	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)

	server.Serve()

	return true
}
<commit_msg>mount: remove leftover socket file<commit_after>//go:build linux || darwin
// +build linux darwin

package command

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/mount/unmount"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/hanwen/go-fuse/v2/fuse"
	"google.golang.org/grpc/reflection"
	"net"
	"net/http"
	"os"
	"os/user"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/grace"
)

func runMount(cmd *Command, args []string) bool {

	if *mountOptions.debug {
		go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil)
	}

	grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
	if *mountReadRetryTime < time.Second {
		*mountReadRetryTime = time.Second
	}
	util.RetryWaitTime = *mountReadRetryTime

	umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64)
	if umaskErr != nil {
		fmt.Printf("can not parse umask %s", *mountOptions.umaskString)
		return false
	}

	if len(args) > 0 {
		return false
	}

	return RunMount(&mountOptions, os.FileMode(umask))
}

func RunMount(option *MountOptions, umask os.FileMode) bool {

	// basic checks
	chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
	if chunkSizeLimitMB <= 0 {
		fmt.Printf("Please specify a reasonable buffer size.")
		return false
	}

	// try to connect to filer
	filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses()
	util.LoadConfiguration("security", false)
	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
	var cipher bool
	var err error
	for i := 0; i < 10; i++ {
		err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
			if err != nil {
				return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err)
			}
			cipher = resp.Cipher
			return nil
		})
		if err != nil {
			glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
			glog.V(0).Infof("wait for %d seconds ...", i+1)
			time.Sleep(time.Duration(i+1) * time.Second)
		}
	}
	if err != nil {
		glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
		return true
	}

	filerMountRootPath := *option.filerMountRootPath

	// clean up mount point
	dir := util.ResolvePath(*option.dir)
	if dir == "" {
		fmt.Printf("Please specify the mount directory via \"-dir\"")
		return false
	}

	unmount.Unmount(dir)

	// start on local unix socket
	if *option.localSocket == "" {
		mountDirHash := util.HashToInt32([]byte(dir))
		if mountDirHash < 0 {
			mountDirHash = -mountDirHash
		}
		*option.localSocket = fmt.Sprintf("/tmp/seaweefs-mount-%d.sock", mountDirHash)
	}
	if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
		glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
	}
	montSocketListener, err := net.Listen("unix", *option.localSocket)
	if err != nil {
		glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
	}

	// detect mount folder mode
	if *option.dirAutoCreate {
		os.MkdirAll(dir, os.FileMode(0777)&^umask)
	}
	fileInfo, err := os.Stat(dir)

	// collect uid, gid
	uid, gid := uint32(0), uint32(0)
	mountMode := os.ModeDir | 0777
	if err == nil {
		mountMode = os.ModeDir | os.FileMode(0777)&^umask
		uid, gid = util.GetFileUidGid(fileInfo)
		fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, mountMode)
	} else {
		fmt.Printf("can not stat %s\n", dir)
		return false
	}

	// detect uid, gid
	if uid == 0 {
		if u, err := user.Current(); err == nil {
			if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
				uid = uint32(parsedId)
			}
			if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
				gid = uint32(parsedId)
			}
			fmt.Printf("current uid=%d gid=%d\n", uid, gid)
		}
	}

	// mapping uid, gid
	uidGidMapper, err := meta_cache.NewUidGidMapper(*option.uidMap, *option.gidMap)
	if err != nil {
		fmt.Printf("failed to parse %s %s: %v\n", *option.uidMap, *option.gidMap, err)
		return false
	}

	// Ensure target mount point availability
	if isValid := checkMountPointAvailable(dir); !isValid {
		glog.Fatalf("Expected mount to still be active, target mount point: %s, please check!", dir)
		return true
	}

	serverFriendlyName := strings.ReplaceAll(*option.filer, ",", "+")

	// mount fuse
	fuseMountOptions := &fuse.MountOptions{
		AllowOther:               *option.allowOthers,
		Options:                  nil,
		MaxBackground:            128,
		MaxWrite:                 1024 * 1024 * 2,
		MaxReadAhead:             1024 * 1024 * 2,
		IgnoreSecurityLabels:     false,
		RememberInodes:           false,
		FsName:                   serverFriendlyName + ":" + filerMountRootPath,
		Name:                     "seaweedfs",
		SingleThreaded:           false,
		DisableXAttrs:            false,
		Debug:                    *option.debug,
		EnableLocks:              false,
		ExplicitDataCacheControl: false,
		DirectMount:              true,
		DirectMountFlags:         0,
		//SyncRead:                 false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
		//EnableAcl:                true,
	}
	if *option.nonempty {
		fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
	}
	if *option.readOnly {
		if runtime.GOOS == "darwin" {
			fuseMountOptions.Options = append(fuseMountOptions.Options, "rdonly")
		} else {
			fuseMountOptions.Options = append(fuseMountOptions.Options, "ro")
		}
	}
	if runtime.GOOS == "darwin" {
		// https://github-wiki-see.page/m/macfuse/macfuse/wiki/Mount-Options
		ioSizeMB := 1
		for ioSizeMB*2 <= *option.chunkSizeLimitMB && ioSizeMB*2 <= 32 {
			ioSizeMB *= 2
		}
		fuseMountOptions.Options = append(fuseMountOptions.Options, "daemon_timeout=600")
		fuseMountOptions.Options = append(fuseMountOptions.Options, "noapplexattr")
		// fuseMountOptions.Options = append(fuseMountOptions.Options, "novncache") // need to test effectiveness
		fuseMountOptions.Options = append(fuseMountOptions.Options, "slow_statfs")
		fuseMountOptions.Options = append(fuseMountOptions.Options, "volname="+serverFriendlyName)
		fuseMountOptions.Options = append(fuseMountOptions.Options, fmt.Sprintf("iosize=%d", ioSizeMB*1024*1024))
	}

	// find mount point
	mountRoot := filerMountRootPath
	if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
		mountRoot = mountRoot[0 : len(mountRoot)-1]
	}

	seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{
		MountDirectory:     dir,
		FilerAddresses:     filerAddresses,
		GrpcDialOption:     grpcDialOption,
		FilerMountRootPath: mountRoot,
		Collection:         *option.collection,
		Replication:        *option.replication,
		TtlSec:             int32(*option.ttlSec),
		DiskType:           types.ToDiskType(*option.diskType),
		ChunkSizeLimit:     int64(chunkSizeLimitMB) * 1024 * 1024,
		ConcurrentWriters:  *option.concurrentWriters,
		CacheDir:           *option.cacheDir,
		CacheSizeMB:        *option.cacheSizeMB,
		DataCenter:         *option.dataCenter,
		Quota:              int64(*option.collectionQuota) * 1024 * 1024,
		MountUid:           uid,
		MountGid:           gid,
		MountMode:          mountMode,
		MountCtime:         fileInfo.ModTime(),
		MountMtime:         time.Now(),
		Umask:              umask,
		VolumeServerAccess: *mountOptions.volumeServerAccess,
		Cipher:             cipher,
		UidGidMapper:       uidGidMapper,
	})

	server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
	if err != nil {
		glog.Fatalf("Mount fail: %v", err)
	}
	grace.OnInterrupt(func() {
		unmount.Unmount(dir)
	})

	grpcS := pb.NewGrpcServer()
	mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem)
	reflection.Register(grpcS)
	go grpcS.Serve(montSocketListener)

	seaweedFileSystem.StartBackgroundTasks()

	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)

	server.Serve()

	return true
}
<|endoftext|>


<commit_before>package controller

import (
	"database/sql"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
	_ "github.com/lib/pq" // _ means to import only for its side-effects (initialization)
	"github.com/williamqliu/go-app/goweb/model"
)

// App : struct to expose references to the router and database
type App struct {
	Router *mux.Router
	DB     *sql.DB
}

// InitializeDB : connect to database
func (app *App) InitializeDB(user, password, dbname, dbhostname string) {
	connectionString := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, password, dbname)

	if len(dbhostname) > 0 {
		connectionString = fmt.Sprintf("user=%s password=%s dbname=%s host=%s sslmode=disable", user, password, dbname, dbhostname)
	}

	var err error
	app.DB, err = sql.Open("postgres", connectionString)
	if err != nil {
		log.Fatal(err)
	}
}

// InitializeTables : initialize database tables if not exists
func (app *App) InitializeTables() {
	fmt.Println("Initializing Table")
	if _, err := app.DB.Query(model.UserTableCreationQuery); err != nil {
		log.Fatal(err)
	}
}

// InitializeRoutes : func to initialize routes
func (app *App) InitializeRoutes() {
	app.Router = mux.NewRouter()
	app.Router.HandleFunc("/", indexHandler).Methods("GET")
	app.Router.HandleFunc("/hello", helloHandler).Methods("GET")
	app.Router.HandleFunc("/login", app.getLoginTemplate).Methods("GET")
	app.Router.HandleFunc("/users", app.getUsers).Methods("GET")
	app.Router.HandleFunc("/users", app.createUser).Methods("POST")
	app.Router.HandleFunc("/users/{id:[0-9]+}", app.getUser).Methods("GET")
	app.Router.HandleFunc("/users/{id:[0-9]+}", app.updateUser).Methods("PUT")
	app.Router.HandleFunc("/users/{id:[0-9]+}", app.deleteUser).Methods("DELETE")

	// Serve Static Files through web server instead of reverse-proxy
	var dir string
	flag.StringVar(&dir, "dir", "./static", "the directory to serve files from, defaults to current dir") // bind string to flag with key, value, and comment
	flag.Parse()                                                                                // call Parse() after flags are defined
	app.Router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir))))
}

// Run : func to start the main application at specifc port
func (app *App) Run(addr string) {
	//log.Fatal(http.ListenAndServe(":8080", app.Router))
	err := http.ListenAndServe(":8080", app.Router)
	if err != nil {
		log.Fatal(err)
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	tmpl, err := template.ParseFiles("./view/index.html")
	if err != nil {
		log.Fatal(err)
	}
	err = tmpl.Execute(w, nil)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Log: indexHandler request")
}

// helloHandler : func to parse form data (e.g. url with '/hello?abcd=12')
func helloHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello Handler") // send data to client side

	// print the parsed form on server side
	r.ParseForm()                       // parse arguments, e.g. /hello/?url_long=111&url_long=222
	fmt.Println("path", r.URL.Path)     // `/hello/`
	fmt.Println("scheme", r.URL.Scheme) // scheme
	fmt.Println("method", r.Method)     // method GET
	fmt.Println(r.Form["url_long"])     // [111 222]
	for k, v := range r.Form {
		fmt.Println("key:", k)                    // key: url_long
		fmt.Println("val:", strings.Join(v, " ")) // val: 111 222
	}
	log.Println("Log: helloHandler request")
}
<commit_msg>Fix ListenAndServe from static 8080 address to address passed in<commit_after>package controller

import (
	"database/sql"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
	_ "github.com/lib/pq" // _ means to import only for its side-effects (initialization)
	"github.com/williamqliu/go-app/goweb/model"
)

// App : struct to expose references to the router and database
type App struct {
	Router *mux.Router
	DB     *sql.DB
}

// InitializeDB : connect to database
func (app *App) InitializeDB(user, password, dbname, dbhostname string) {
	connectionString := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, password, dbname)

	if len(dbhostname) > 0 {
		connectionString = fmt.Sprintf("user=%s password=%s dbname=%s host=%s sslmode=disable", user, password, dbname, dbhostname)
	}

	var err error
	app.DB, err = sql.Open("postgres", connectionString)
	if err != nil {
		log.Fatal(err)
	}
}

// InitializeTables : initialize database tables if not exists
func (app *App) InitializeTables() {
	fmt.Println("Initializing Table")
	if _, err := app.DB.Query(model.UserTableCreationQuery); err != nil {
		log.Fatal(err)
	}
}

// InitializeRoutes : func to initialize routes
func (app *App) InitializeRoutes() {
	app.Router = mux.NewRouter()
	app.Router.HandleFunc("/", indexHandler).Methods("GET")
	app.Router.HandleFunc("/hello", helloHandler).Methods("GET")
	app.Router.HandleFunc("/login", app.getLoginTemplate).Methods("GET")
	app.Router.HandleFunc("/users", app.getUsers).Methods("GET")
	app.Router.HandleFunc("/users", app.createUser).Methods("POST")
	app.Router.HandleFunc("/users/{id:[0-9]+}", app.getUser).Methods("GET")
	app.Router.HandleFunc("/users/{id:[0-9]+}", app.updateUser).Methods("PUT")
	app.Router.HandleFunc("/users/{id:[0-9]+}", app.deleteUser).Methods("DELETE")

	// Serve Static Files through web server instead of reverse-proxy
	var dir string
	flag.StringVar(&dir, "dir", "./static", "the directory to serve files from, defaults to current dir") // bind string to flag with key, value, and comment
	flag.Parse()                                                                                // call Parse() after flags are defined
	app.Router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir))))
}

// Run : func to start the main application at specifc port
func (app *App) Run(addr string) {
	err := http.ListenAndServe(addr, app.Router)
	if err != nil {
		log.Fatal(err)
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	tmpl, err := template.ParseFiles("./view/index.html")
	if err != nil {
		log.Fatal(err)
	}
	err = tmpl.Execute(w, nil)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Log: indexHandler request")
}

// helloHandler : func to parse form data (e.g. url with '/hello?abcd=12')
func helloHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello Handler") // send data to client side

	// print the parsed form on server side
	r.ParseForm()                       // parse arguments, e.g. /hello/?url_long=111&url_long=222
	fmt.Println("path", r.URL.Path)     // `/hello/`
	fmt.Println("scheme", r.URL.Scheme) // scheme
	fmt.Println("method", r.Method)     // method GET
	fmt.Println(r.Form["url_long"])     // [111 222]
	for k, v := range r.Form {
		fmt.Println("key:", k)                    // key: url_long
		fmt.Println("val:", strings.Join(v, " ")) // val: 111 222
	}
	log.Println("Log: helloHandler request")
}
<|endoftext|>


<commit_before>package impl

import (
	"encoding/base64"
	"fmt"
	"github.com/bmatcuk/doublestar"
	"github.com/vitesse-ftian/dggo/vitessedata/proto/xdrive"
	"hash/fnv"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
	"vitessedata/plugin"
)

// DoRead servies XDrive read requests.   It read a ReadRequest from stdin and reply
// a sequence of PluginDataReply to stdout.   It should end the data stream with a
// trivial (Errcode == 0, but there is no data) message.

func DoRead(req xdrive.ReadRequest, rootpath string) error {

	// Check/validate frag info.  Again, not necessary, as xdriver server should always
	// fill in good value.
	if req.FragCnt <= 0 || req.FragId < 0 || req.FragId >= req.FragCnt {
		plugin.DbgLog("Invalid read req %v", req)
		plugin.DataReply(-3, fmt.Sprintf("Read request frag (%d, %d) is not valid.", req.FragId, req.FragCnt))
		return fmt.Errorf("Invalid read request")
	}

	//
	// Filter:
	// req may contains a list of Filters that got pushed down from XDrive server.
	// As per plugin protocol, plugin can ignore all of them if they choose to be
	// lazy.  See comments in csvhandler.go.
	//
	// All filters are derived from SQL (where clause).  There is a special kind of
	// filter called "QUERY", which allow users to send any query to plugin.
	//
	// var query string
	// for _, f := range req.Filter {
	// f cannot be nil
	//		if f.Op == "QUERY" {
	//			query = f.Args[0]
	//		}
	//	}

	// Glob:
	idx := strings.Index(req.Filespec.Path[1:], "/")
	path := req.Filespec.Path[idx+1:]
	path = filepath.Join(rootpath, path)
	plugin.DbgLog("path %s", path)
	flist, err := doublestar.Glob(path)
	if err != nil {
		plugin.DbgLogIfErr(err, "Glob failed.  %s", path)
		plugin.DataReply(-2, "rmgr glob failed: "+err.Error())
		return err
	}

	// There are many different ways to implement FragId/FragCnt.   Here we use filename.
	// All data within one file go to one fragid.  We determine which files this call
	// should serve.  Any deterministic scheme should work.  We use hash mod.
	myflist := []string{}
	for _, f := range flist {
		h := fnv.New32a()
		h.Write([]byte(f))
		hv := int32(h.Sum32())

		tmp := hv % req.FragCnt
		if tmp < 0 {
			tmp += req.FragCnt
		}

		if req.FragId == tmp {
			plugin.DbgLog("Frag: file %s hash to %d, match frag (%d, %d)", f, hv, req.FragId, req.FragCnt)
			myflist = append(myflist, f)
		} else {
			plugin.DbgLog("Frag: file %s hash to %d, does not match frag (%d, %d)", f, hv, req.FragId, req.FragCnt)
		}
	}

	// Return each file.
	plugin.DbgLog("fsplugin: path %s, frag (%d, %d) globed %v", path, req.FragId, req.FragCnt, myflist)

	// ls_file columns:
	//	dir,
	//	basename, size, mode, modtime, isdir,  as returned by lstat
	//	content_base64
	//
	needContent := false
	projcollist := req.Columnlist
	if len(projcollist) == 0 {
		projcollist = make([]string, len(req.Columndesc))
		for j, d := range req.Columndesc {
			projcollist[j] = d.Name
		}
	}

	for _, c := range projcollist {
		if c == "content_base64" {
			needContent = true
		}
	}

	// Now process each file.
	for _, f := range myflist {
		fi, err := os.Lstat(f)
		if err != nil {
			return err
		}

		content := ""
		contentNull := true
		if needContent && !fi.IsDir() {
			dat, err := ioutil.ReadFile(f)
			if err == nil {
				content = base64.StdEncoding.EncodeToString(dat)
				contentNull = false
			}
		}

		for col := 0; col < len(projcollist); col++ {
			xcol := new(xdrive.XCol)
			xcol.Colname = projcollist[col]
			xcol.Nrow = 1
			xcol.Nullmap = make([]bool, 1)
			xcol.Nullmap[0] = false

			switch xcol.Colname {
			case "dir":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = filepath.Dir(f)
			case "basename":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fi.Name()
			case "size":
				// BUG BUG: Deepgreen server has a bug, it messed up col type.
				// only str is safe.   Fix that!   And we will reenable this
				// int64 thing.
				// xcol.I64Data = make([]int64, 1)
				// xcol.I64Data[0] = fi.Size()
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fmt.Sprintf("%d", fi.Size())
			case "mode":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fi.Mode().String()
			case "modtime":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fi.ModTime().Format(time.RFC1123Z)
			case "isdir":
				// BUG BUG
				// xcol.I32Data = make([]int32, 1)
				xcol.Sdata = make([]string, 1)
				if fi.IsDir() {
					// xcol.I32Data[0] = 1
					xcol.Sdata[0] = "1"
				} else {
					// xcol.I32Data[0] = 0
					xcol.Sdata[0] = "0"
				}
			case "content_base64":
				xcol.Sdata = make([]string, 1)
				if contentNull {
					xcol.Nullmap[0] = true
				} else {
					xcol.Sdata[0] = content
				}
			}

			var colreply xdrive.XColDataReply
			colreply.Data = xcol
			err = plugin.ReplyXColData(colreply)
			if err != nil {
				plugin.DbgLogIfErr(err, "Write data col failed.")
				return err
			}
		}
	}

	// Done, write an end.
	var col xdrive.XColDataReply
	err = plugin.ReplyXColData(col)
	if err != nil {
		plugin.DbgLogIfErr(err, "Write data end failed.")
		return err
	}
	return nil
}
<commit_msg>Impl an xdrive_query fileglob filter.<commit_after>package impl

import (
	"encoding/base64"
	"fmt"
	"github.com/bmatcuk/doublestar"
	"github.com/vitesse-ftian/dggo/vitessedata/proto/xdrive"
	"hash/fnv"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
	"vitessedata/plugin"
)

// evaluate QUERY= filters.   All the filters are AND-ed.
func evalQuery(qstr []string, fn []string) []string {
	if len(qstr) == 0 {
		return fn
	}

	ret := make([]string, 0)
	for _, filename := range fn {
		ok := true
		for _, qf := range qstr {
			if len(qf) > 9 && qf[:9] == "fileglob=" {
				matched, err := doublestar.PathMatch(qf[9:], filename)
				if err != nil || !matched {
					ok = false
					break
				}
			}
		}

		if ok {
			ret = append(ret, filename)
		}
	}
	return ret
}

// DoRead servies XDrive read requests.   It read a ReadRequest from stdin and reply
// a sequence of PluginDataReply to stdout.   It should end the data stream with a
// trivial (Errcode == 0, but there is no data) message.
func DoRead(req xdrive.ReadRequest, rootpath string) error {

	// Check/validate frag info.  Again, not necessary, as xdriver server should always
	// fill in good value.
	if req.FragCnt <= 0 || req.FragId < 0 || req.FragId >= req.FragCnt {
		plugin.DbgLog("Invalid read req %v", req)
		plugin.DataReply(-3, fmt.Sprintf("Read request frag (%d, %d) is not valid.", req.FragId, req.FragCnt))
		return fmt.Errorf("Invalid read request")
	}

	//
	// Filter:
	// req may contains a list of Filters that got pushed down from XDrive server.
	// As per plugin protocol, plugin can ignore all of them if they choose to be
	// lazy.  See comments in csvhandler.go.
	//
	// All filters are derived from SQL (where clause).  There is a special kind of
	// filter called "QUERY", which allow users to send any query to plugin.
	//
	query := make([]string, 0)
	for _, f := range req.Filter {
		// f cannot be nil, we only do
		plugin.DbgLog("xdrive_query: %s, %s.", f.Op, f.Args[0])
		if f.Op == "QUERY" {
			query = append(query, f.Args[0])
		}
	}

	// Glob:
	idx := strings.Index(req.Filespec.Path[1:], "/")
	path := req.Filespec.Path[idx+1:]
	path = filepath.Join(rootpath, path)
	plugin.DbgLog("path %s", path)
	flist, err := doublestar.Glob(path)
	if err != nil {
		plugin.DbgLogIfErr(err, "Glob failed.  %s", path)
		plugin.DataReply(-2, "rmgr glob failed: "+err.Error())
		return err
	}

	flist = evalQuery(query, flist)

	// There are many different ways to implement FragId/FragCnt.   Here we use filename.
	// All data within one file go to one fragid.  We determine which files this call
	// should serve.  Any deterministic scheme should work.  We use hash mod.
	myflist := []string{}
	for _, f := range flist {
		h := fnv.New32a()
		h.Write([]byte(f))
		hv := int32(h.Sum32())

		tmp := hv % req.FragCnt
		if tmp < 0 {
			tmp += req.FragCnt
		}

		if req.FragId == tmp {
			// plugin.DbgLog("Frag: file %s hash to %d, match frag (%d, %d)", f, hv, req.FragId, req.FragCnt)
			myflist = append(myflist, f)
		} else {
			// plugin.DbgLog("Frag: file %s hash to %d, does not match frag (%d, %d)", f, hv, req.FragId, req.FragCnt)
		}
	}

	// Return each file.
	// plugin.DbgLog("fsplugin: path %s, frag (%d, %d) globed %v", path, req.FragId, req.FragCnt, myflist)

	// ls_file columns:
	//	dir,
	//	basename, size, mode, modtime, isdir,  as returned by lstat
	//	content_base64
	//
	needContent := false
	projcollist := req.Columnlist
	if len(projcollist) == 0 {
		projcollist = make([]string, len(req.Columndesc))
		for j, d := range req.Columndesc {
			projcollist[j] = d.Name
		}
	}

	for _, c := range projcollist {
		if c == "content_base64" {
			needContent = true
		}
	}

	// Now process each file.
	for _, f := range myflist {
		fi, err := os.Lstat(f)
		if err != nil {
			return err
		}

		content := ""
		contentNull := true
		if needContent && !fi.IsDir() {
			dat, err := ioutil.ReadFile(f)
			if err == nil {
				content = base64.StdEncoding.EncodeToString(dat)
				contentNull = false
			}
		}

		for col := 0; col < len(projcollist); col++ {
			xcol := new(xdrive.XCol)
			xcol.Colname = projcollist[col]
			xcol.Nrow = 1
			xcol.Nullmap = make([]bool, 1)
			xcol.Nullmap[0] = false

			switch xcol.Colname {
			case "dir":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = filepath.Dir(f)
			case "basename":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fi.Name()
			case "size":
				// BUG BUG: Deepgreen server has a bug, it messed up col type.
				// only str is safe.   Fix that!   And we will reenable this
				// int64 thing.
				// xcol.I64Data = make([]int64, 1)
				// xcol.I64Data[0] = fi.Size()
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fmt.Sprintf("%d", fi.Size())
			case "mode":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fi.Mode().String()
			case "modtime":
				xcol.Sdata = make([]string, 1)
				xcol.Sdata[0] = fi.ModTime().Format(time.RFC1123Z)
			case "isdir":
				// BUG BUG
				// xcol.I32Data = make([]int32, 1)
				xcol.Sdata = make([]string, 1)
				if fi.IsDir() {
					// xcol.I32Data[0] = 1
					xcol.Sdata[0] = "1"
				} else {
					// xcol.I32Data[0] = 0
					xcol.Sdata[0] = "0"
				}
			case "content_base64":
				xcol.Sdata = make([]string, 1)
				if contentNull {
					xcol.Nullmap[0] = true
				} else {
					xcol.Sdata[0] = content
				}
			}

			var colreply xdrive.XColDataReply
			colreply.Data = xcol
			err = plugin.ReplyXColData(colreply)
			if err != nil {
				plugin.DbgLogIfErr(err, "Write data col failed.")
				return err
			}
		}
	}

	// Done, write an end.
	var col xdrive.XColDataReply
	err = plugin.ReplyXColData(col)
	if err != nil {
		plugin.DbgLogIfErr(err, "Write data end failed.")
		return err
	}
	return nil
}
<|endoftext|>