<commit_before>// Copyright 2016 The astrogo Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/draw"
	"image/png"
	"log"
	"os"

	"github.com/astrogo/fitsio"

	"golang.org/x/exp/shiny/driver"
	"golang.org/x/exp/shiny/screen"
	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/lifecycle"
	"golang.org/x/mobile/event/paint"
	"golang.org/x/mobile/event/size"
)

type fileInfo struct {
	Name   string
	Images []image.Image
}

func main() {

	help := flag.Bool("help", false, "show help")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `view-fits - a FITS image viewer.

Usage of view-fits:
$ view-fits [file1 [file2 [...]]]

Examples:
$ view-fits astrogo/fitsio/testdata/file-img2-bitpix+08.fits
$ view-fits astrogo/fitsio/testdata/file-img2-bitpix*.fits

Controls:
- left/right arrows: switch to previous/next file
- up/down arrows:    switch to previous/next image in the current file
- r:                 reload/redisplay current image
- z:                 resize window to fit current image
- p:                 print current image to 'output.png'
- ?:                 show help
- q/ESC:             quit
`)
	}

	flag.Parse()

	if *help || len(os.Args) < 2 {
		flag.Usage()
		os.Exit(0)
	}

	log.SetFlags(0)
	log.SetPrefix("[view-fits] ")

	infos := processFiles()
	if len(infos) == 0 {
		log.Fatal("No image among given FITS files.")
	}

	type cursor struct {
		file int
		img  int
	}

	driver.Main(func(s screen.Screen) {

		// Number of files.
		nbFiles := len(infos)

		// Current displayed file and image in file.
		cur := cursor{file: 0, img: 0}

		// Building the main window.
		w, err := s.NewWindow(&screen.NewWindowOptions{})
		if err != nil {
			log.Fatal(err)
		}
		defer w.Release()

		// Building the screen buffer.
		b, err := s.NewBuffer(image.Point{})
		if err != nil {
			log.Fatal(err)
		}
		defer release(b)

		w.Fill(b.Bounds(), color.Black, draw.Src)
		w.Publish()

		repaint := true
		var (
			sz size.Event
			//bkg = color.Black
			bkg = color.RGBA{0xe0, 0xe0, 0xe0, 0xff} // Material Design "Grey 300"
		)

		for {
			switch e := w.NextEvent().(type) {
			default:
				// ignore

			case lifecycle.Event:
				switch {
				case e.From == lifecycle.StageVisible && e.To == lifecycle.StageFocused:
					repaint = true
				default:
					repaint = false
				}
				if repaint {
					w.Send(paint.Event{})
				}

			case key.Event:
				switch e.Code {
				case key.CodeEscape, key.CodeQ:
					return

				case key.CodeSlash:
					if e.Direction == key.DirPress && e.Modifiers&key.ModShift != 0 {
						flag.Usage()
						continue
					}

				case key.CodeRightArrow:
					if e.Direction == key.DirPress {
						repaint = true
						if cur.file < nbFiles-1 {
							cur.file++
						} else {
							cur.file = 0
						}
						cur.img = 0
						log.Printf("file:   %v\n", infos[cur.file].Name)
						log.Printf("images: %d\n", len(infos[cur.file].Images))
					}

				case key.CodeLeftArrow:
					if e.Direction == key.DirPress {
						repaint = true
						if cur.file == 0 {
							cur.file = nbFiles - 1
						} else {
							cur.file--
						}
						cur.img = 0
						log.Printf("file:   %v\n", infos[cur.file].Name)
						log.Printf("images: %d\n", len(infos[cur.file].Images))
					}

				case key.CodeDownArrow:
					if e.Direction == key.DirPress {
						repaint = true
						nbImg := len(infos[cur.file].Images)
						if cur.img < nbImg-1 {
							cur.img++
						} else {
							cur.img = 0
						}
					}

				case key.CodeUpArrow:
					if e.Direction == key.DirPress {
						repaint = true
						nbImg := len(infos[cur.file].Images)
						if cur.img == 0 {
							cur.img = nbImg - 1
						} else {
							cur.img--
						}
					}

				case key.CodeR:
					if e.Direction == key.DirPress {
						repaint = true
					}

				case key.CodeZ:
					if e.Direction == key.DirPress {
						// resize to current image
						// TODO(sbinet)
						repaint = true
					}

				case key.CodeP:
					if e.Direction != key.DirPress {
						continue
					}
					out, err := os.Create("output.png")
					if err != nil {
						log.Fatalf("error printing image: %v\n", err)
					}
					defer out.Close()
					err = png.Encode(out, infos[cur.file].Images[cur.img])
					if err != nil {
						log.Fatalf("error printing image: %v\n", err)
					}
					err = out.Close()
					if err != nil {
						log.Fatalf("error printing image: %v\n", err)
					}
					log.Printf("printed current image to [%s]\n", out.Name())
				}

				if repaint {
					w.Send(paint.Event{})
				}

			case size.Event:
				sz = e

			case paint.Event:
				if !repaint {
					continue
				}
				repaint = false
				img := infos[cur.file].Images[cur.img]

				release(b)
				b, err = s.NewBuffer(img.Bounds().Size())
				if err != nil {
					log.Fatal(err)
				}
				defer release(b)

				draw.Draw(b.RGBA(), b.Bounds(), img, image.Point{}, draw.Src)

				w.Fill(sz.Bounds(), bkg, draw.Src)
				w.Upload(image.Point{}, b, img.Bounds())
				w.Publish()
			}

		}

	})
}

func processFiles() []fileInfo {
	infos := make([]fileInfo, 0, len(flag.Args()))
	// Parsing input files.
	for _, fname := range flag.Args() {

		finfo := fileInfo{Name: fname}

		// Opening the file.
		r, err := os.Open(fname)
		if err != nil {
			log.Fatal("Can not open the input file: %s.", err)
		}
		defer r.Close()

		// Opening the FITS file.
		f, err := fitsio.Open(r)
		if err != nil {
			log.Fatal("Can not open the FITS input file: %s.", err)
		}
		defer f.Close()

		// Getting the file HDUs.
		hdus := f.HDUs()
		for _, hdu := range hdus {
			// Getting the header informations.
			header := hdu.Header()
			axes := header.Axes()

			// Discarding HDU with no axes.
			if len(axes) != 0 {
				if hdu, ok := hdu.(fitsio.Image); ok {
					img := hdu.Image()
					if img != nil {
						finfo.Images = append(finfo.Images, img)
					}
				}
			}
		}

		if len(finfo.Images) > 0 {
			infos = append(infos, finfo)
		}
	}

	return infos
}

type releaser interface {
	Release()
}

func release(r releaser) {
	if r != nil {
		r.Release()
	}
}
<commit_msg>cmd/view-fits: give an initial window+buffer size<commit_after>// Copyright 2016 The astrogo Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/draw"
	"image/png"
	"log"
	"os"

	"github.com/astrogo/fitsio"

	"golang.org/x/exp/shiny/driver"
	"golang.org/x/exp/shiny/screen"
	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/lifecycle"
	"golang.org/x/mobile/event/paint"
	"golang.org/x/mobile/event/size"
)

type fileInfo struct {
	Name   string
	Images []image.Image
}

func main() {

	help := flag.Bool("help", false, "show help")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `view-fits - a FITS image viewer.

Usage of view-fits:
$ view-fits [file1 [file2 [...]]]

Examples:
$ view-fits astrogo/fitsio/testdata/file-img2-bitpix+08.fits
$ view-fits astrogo/fitsio/testdata/file-img2-bitpix*.fits

Controls:
- left/right arrows: switch to previous/next file
- up/down arrows:    switch to previous/next image in the current file
- r:                 reload/redisplay current image
- z:                 resize window to fit current image
- p:                 print current image to 'output.png'
- ?:                 show help
- q/ESC:             quit
`)
	}

	flag.Parse()

	if *help || len(os.Args) < 2 {
		flag.Usage()
		os.Exit(0)
	}

	log.SetFlags(0)
	log.SetPrefix("[view-fits] ")

	infos := processFiles()
	if len(infos) == 0 {
		log.Fatal("No image among given FITS files.")
	}

	type cursor struct {
		file int
		img  int
	}

	driver.Main(func(s screen.Screen) {

		// Number of files.
		nbFiles := len(infos)

		// Current displayed file and image in file.
		cur := cursor{file: 0, img: 0}

		// Building the main window.
		w, err := s.NewWindow(&screen.NewWindowOptions{
			Width:  500,
			Height: 500,
		})
		if err != nil {
			log.Fatal(err)
		}
		defer w.Release()

		// Building the screen buffer.
		b, err := s.NewBuffer(image.Point{500, 500})
		if err != nil {
			log.Fatal(err)
		}
		defer release(b)

		w.Fill(b.Bounds(), color.Black, draw.Src)
		w.Publish()

		repaint := true
		var (
			sz size.Event
			//bkg = color.Black
			bkg = color.RGBA{0xe0, 0xe0, 0xe0, 0xff} // Material Design "Grey 300"
		)

		for {
			switch e := w.NextEvent().(type) {
			default:
				// ignore

			case lifecycle.Event:
				switch {
				case e.From == lifecycle.StageVisible && e.To == lifecycle.StageFocused:
					repaint = true
				default:
					repaint = false
				}
				if repaint {
					w.Send(paint.Event{})
				}

			case key.Event:
				switch e.Code {
				case key.CodeEscape, key.CodeQ:
					return

				case key.CodeSlash:
					if e.Direction == key.DirPress && e.Modifiers&key.ModShift != 0 {
						flag.Usage()
						continue
					}

				case key.CodeRightArrow:
					if e.Direction == key.DirPress {
						repaint = true
						if cur.file < nbFiles-1 {
							cur.file++
						} else {
							cur.file = 0
						}
						cur.img = 0
						log.Printf("file:   %v\n", infos[cur.file].Name)
						log.Printf("images: %d\n", len(infos[cur.file].Images))
					}

				case key.CodeLeftArrow:
					if e.Direction == key.DirPress {
						repaint = true
						if cur.file == 0 {
							cur.file = nbFiles - 1
						} else {
							cur.file--
						}
						cur.img = 0
						log.Printf("file:   %v\n", infos[cur.file].Name)
						log.Printf("images: %d\n", len(infos[cur.file].Images))
					}

				case key.CodeDownArrow:
					if e.Direction == key.DirPress {
						repaint = true
						nbImg := len(infos[cur.file].Images)
						if cur.img < nbImg-1 {
							cur.img++
						} else {
							cur.img = 0
						}
					}

				case key.CodeUpArrow:
					if e.Direction == key.DirPress {
						repaint = true
						nbImg := len(infos[cur.file].Images)
						if cur.img == 0 {
							cur.img = nbImg - 1
						} else {
							cur.img--
						}
					}

				case key.CodeR:
					if e.Direction == key.DirPress {
						repaint = true
					}

				case key.CodeZ:
					if e.Direction == key.DirPress {
						// resize to current image
						// TODO(sbinet)
						repaint = true
					}

				case key.CodeP:
					if e.Direction != key.DirPress {
						continue
					}
					out, err := os.Create("output.png")
					if err != nil {
						log.Fatalf("error printing image: %v\n", err)
					}
					defer out.Close()
					err = png.Encode(out, infos[cur.file].Images[cur.img])
					if err != nil {
						log.Fatalf("error printing image: %v\n", err)
					}
					err = out.Close()
					if err != nil {
						log.Fatalf("error printing image: %v\n", err)
					}
					log.Printf("printed current image to [%s]\n", out.Name())
				}

				if repaint {
					w.Send(paint.Event{})
				}

			case size.Event:
				sz = e

			case paint.Event:
				if !repaint {
					continue
				}
				repaint = false
				img := infos[cur.file].Images[cur.img]

				release(b)
				b, err = s.NewBuffer(img.Bounds().Size())
				if err != nil {
					log.Fatal(err)
				}
				defer release(b)

				draw.Draw(b.RGBA(), b.Bounds(), img, image.Point{}, draw.Src)

				w.Fill(sz.Bounds(), bkg, draw.Src)
				w.Upload(image.Point{}, b, img.Bounds())
				w.Publish()
			}

		}

	})
}

func processFiles() []fileInfo {
	infos := make([]fileInfo, 0, len(flag.Args()))
	// Parsing input files.
	for _, fname := range flag.Args() {

		finfo := fileInfo{Name: fname}

		// Opening the file.
		r, err := os.Open(fname)
		if err != nil {
			log.Fatal("Can not open the input file: %s.", err)
		}
		defer r.Close()

		// Opening the FITS file.
		f, err := fitsio.Open(r)
		if err != nil {
			log.Fatal("Can not open the FITS input file: %s.", err)
		}
		defer f.Close()

		// Getting the file HDUs.
		hdus := f.HDUs()
		for _, hdu := range hdus {
			// Getting the header informations.
			header := hdu.Header()
			axes := header.Axes()

			// Discarding HDU with no axes.
			if len(axes) != 0 {
				if hdu, ok := hdu.(fitsio.Image); ok {
					img := hdu.Image()
					if img != nil {
						finfo.Images = append(finfo.Images, img)
					}
				}
			}
		}

		if len(finfo.Images) > 0 {
			infos = append(infos, finfo)
		}
	}

	return infos
}

type releaser interface {
	Release()
}

func release(r releaser) {
	if r != nil {
		r.Release()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"github.com/tidwall/gjson"
	"github.com/whosonfirst/globe" // for to make DrawPreparedPaths public
	"github.com/whosonfirst/go-whosonfirst-crawl"
	"github.com/whosonfirst/go-whosonfirst-csv"
	"github.com/whosonfirst/go-whosonfirst-uri"
	"image/color"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func DrawFeature(feature []byte, gl *globe.Globe) error {

	geom_type := gjson.GetBytes(feature, "geometry.type")

	if !geom_type.Exists() {
		return errors.New("Geometry is missing a type property")
	}

	coords := gjson.GetBytes(feature, "geometry.coordinates")

	if !coords.Exists() {
		return errors.New("Geometry is missing a coordinates property")
	}

	switch geom_type.String() {

	case "Point":

	/*
		lonlat := coords.Array()
		lat := lonlat[1].Float()
		lon := lonlat[0].Float()

		gl.DrawDot(lat, lon, 0.01, globe.Color(green))
	*/

	case "Polygon":

		paths := make([][]*globe.Point, 0)

		for _, ring := range coords.Array() {

			path := make([]*globe.Point, 0)

			for _, r := range ring.Array() {

				lonlat := r.Array()
				lat := lonlat[1].Float()
				lon := lonlat[0].Float()

				pt := globe.NewPoint(lat, lon)
				path = append(path, &pt)
			}

			paths = append(paths, path)
		}

		gl.DrawPaths(paths)

	case "MultiPolygon":
		// log.Println("Can't process MultiPolygon")

	default:
		return errors.New("Unsupported geometry type")
	}

	return nil
}

func DrawRow(path string, row map[string]string, g *globe.Globe, throttle chan bool) error {

	<-throttle

	defer func() {
		throttle <- true
	}()

	rel_path, ok := row["path"]

	if !ok {
		log.Println("Missing path")
		return nil
	}

	meta := filepath.Dir(path)
	root := filepath.Dir(meta)
	data := filepath.Join(root, "data")

	abs_path := filepath.Join(data, rel_path)

	fh, err := os.Open(abs_path)

	if err != nil {
		log.Fatal("failed to open %s, because %s\n", abs_path, err)
	}

	defer fh.Close()

	feature, err := ioutil.ReadAll(fh)

	if err != nil {
		log.Fatal("failed to read %s, because %s\n", abs_path, err)
	}

	return DrawFeature(feature, g)
}

func main() {

	outfile := flag.String("out", "", "Where to write globe")
	size := flag.Int("size", 1600, "The size of the globe (in pixels)")
	mode := flag.String("mode", "meta", "... (default is 'meta' for one or more meta files)")

	feature := flag.Bool("feature", false, "...")

	center := flag.String("center", "", "")
	center_lat := flag.Float64("latitude", 37.755244, "")
	center_lon := flag.Float64("longitude", -122.447777, "")

	flag.Parse()

	if *center != "" {

		latlon := strings.Split(*center, ",")

		lat, err := strconv.ParseFloat(latlon[0], 64)

		if err != nil {
			log.Fatal(err)
		}

		lon, err := strconv.ParseFloat(latlon[1], 64)

		if err != nil {
			log.Fatal(err)
		}

		*center_lat = lat
		*center_lon = lon
	}

	green := color.NRGBA{0x00, 0x64, 0x3c, 192}
	g := globe.New()
	g.DrawGraticule(10.0)

	t1 := time.Now()

	if *mode == "meta" {

		max_fh := 10
		throttle := make(chan bool, max_fh)

		for i := 0; i < max_fh; i++ {
			throttle <- true
		}

		for _, path := range flag.Args() {

			reader, err := csv.NewDictReaderFromPath(path)

			if err != nil {
				log.Fatal(err)
			}

			for {
				row, err := reader.Read()

				if err == io.EOF {
					break
				}

				if err != nil {
					log.Println(err, path)
					break
				}

				if *feature {
					DrawRow(path, row, g, throttle)
					continue
				}

				str_lat, ok := row["geom_latitude"]

				if !ok {
					continue
				}

				str_lon, ok := row["geom_longitude"]

				if !ok {
					continue
				}

				lat, err := strconv.ParseFloat(str_lat, 64)

				if err != nil {
					log.Println(err, str_lat)
					continue
				}

				lon, err := strconv.ParseFloat(str_lon, 64)

				if err != nil {
					log.Println(err, str_lon)
					continue
				}

				g.DrawDot(lat, lon, 0.01, globe.Color(green))
			}
		}

	} else if *mode == "repo" {

		for _, path := range flag.Args() {

			var cb = func(path string, info os.FileInfo) error {

				if info.IsDir() {
					return nil
				}

				is_wof, err := uri.IsWOFFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is a WOF file, because %s\n", path, err)
					return err
				}

				if !is_wof {
					return nil
				}

				is_alt, err := uri.IsAltFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is an alt (WOF) file, because %s\n", path, err)
					return err
				}

				if is_alt {
					return nil
				}

				fh, err := os.Open(path)

				if err != nil {
					log.Printf("failed to open %s, because %s\n", path, err)
					return err
				}

				defer fh.Close()

				feature, err := ioutil.ReadAll(fh)

				if err != nil {
					log.Printf("failed to read %s, because %s\n", path, err)
					return err
				}

				return DrawFeature(feature, g)
			}

			cr := crawl.NewCrawler(path)
			cr.Crawl(cb)
		}

	} else {

		log.Fatal("Invalid mode")
	}

	t2 := time.Since(t1)

	log.Printf("time to read all the things %v\n", t2)

	t3 := time.Now()

	g.CenterOn(*center_lat, *center_lon)
	err := g.SavePNG(*outfile, *size)

	t4 := time.Since(t3)

	log.Printf("time to draw all the things %v\n", t4)

	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>first pass at MultiPolgons<commit_after>package main

import (
	"errors"
	"flag"
	"github.com/tidwall/gjson"
	"github.com/whosonfirst/globe" // for to make DrawPreparedPaths public
	"github.com/whosonfirst/go-whosonfirst-crawl"
	"github.com/whosonfirst/go-whosonfirst-csv"
	"github.com/whosonfirst/go-whosonfirst-uri"
	"image/color"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func DrawFeature(feature []byte, gl *globe.Globe) error {

	geom_type := gjson.GetBytes(feature, "geometry.type")

	if !geom_type.Exists() {
		return errors.New("Geometry is missing a type property")
	}

	coords := gjson.GetBytes(feature, "geometry.coordinates")

	if !coords.Exists() {
		return errors.New("Geometry is missing a coordinates property")
	}

	switch geom_type.String() {

	case "Point":

	/*
		lonlat := coords.Array()
		lat := lonlat[1].Float()
		lon := lonlat[0].Float()

		gl.DrawDot(lat, lon, 0.01, globe.Color(green))
	*/

	// http://geojson.org/geojson-spec.html#id4

	case "Polygon":

		paths := make([][]*globe.Point, 0)

		for _, ring := range coords.Array() {

			path := make([]*globe.Point, 0)

			for _, r := range ring.Array() {

				lonlat := r.Array()
				lat := lonlat[1].Float()
				lon := lonlat[0].Float()

				pt := globe.NewPoint(lat, lon)
				path = append(path, &pt)
			}

			paths = append(paths, path)
		}

		gl.DrawPaths(paths)

	// http://geojson.org/geojson-spec.html#id7

	case "MultiPolygon":

		for _, polys := range coords.Array() {

			paths := make([][]*globe.Point, 0)

			for _, ring := range polys.Array() {

				path := make([]*globe.Point, 0)

				for _, r := range ring.Array() {

					lonlat := r.Array()
					lat := lonlat[1].Float()
					lon := lonlat[0].Float()

					pt := globe.NewPoint(lat, lon)
					path = append(path, &pt)
				}

				paths = append(paths, path)
			}

			gl.DrawPaths(paths)
		}

	default:
		return errors.New("Unsupported geometry type")
	}

	return nil
}

func DrawRow(path string, row map[string]string, g *globe.Globe, throttle chan bool) error {

	<-throttle

	defer func() {
		throttle <- true
	}()

	rel_path, ok := row["path"]

	if !ok {
		log.Println("Missing path")
		return nil
	}

	meta := filepath.Dir(path)
	root := filepath.Dir(meta)
	data := filepath.Join(root, "data")

	abs_path := filepath.Join(data, rel_path)

	fh, err := os.Open(abs_path)

	if err != nil {
		log.Fatal("failed to open %s, because %s\n", abs_path, err)
	}

	defer fh.Close()

	feature, err := ioutil.ReadAll(fh)

	if err != nil {
		log.Fatal("failed to read %s, because %s\n", abs_path, err)
	}

	return DrawFeature(feature, g)
}

func main() {

	outfile := flag.String("out", "", "Where to write globe")
	size := flag.Int("size", 1600, "The size of the globe (in pixels)")
	mode := flag.String("mode", "meta", "... (default is 'meta' for one or more meta files)")

	feature := flag.Bool("feature", false, "...")

	center := flag.String("center", "", "")
	center_lat := flag.Float64("latitude", 37.755244, "")
	center_lon := flag.Float64("longitude", -122.447777, "")

	flag.Parse()

	if *center != "" {

		latlon := strings.Split(*center, ",")

		lat, err := strconv.ParseFloat(latlon[0], 64)

		if err != nil {
			log.Fatal(err)
		}

		lon, err := strconv.ParseFloat(latlon[1], 64)

		if err != nil {
			log.Fatal(err)
		}

		*center_lat = lat
		*center_lon = lon
	}

	green := color.NRGBA{0x00, 0x64, 0x3c, 192}
	g := globe.New()
	g.DrawGraticule(10.0)

	t1 := time.Now()

	if *mode == "meta" {

		max_fh := 10
		throttle := make(chan bool, max_fh)

		for i := 0; i < max_fh; i++ {
			throttle <- true
		}

		for _, path := range flag.Args() {

			reader, err := csv.NewDictReaderFromPath(path)

			if err != nil {
				log.Fatal(err)
			}

			for {
				row, err := reader.Read()

				if err == io.EOF {
					break
				}

				if err != nil {
					log.Println(err, path)
					break
				}

				if *feature {
					DrawRow(path, row, g, throttle)
					continue
				}

				str_lat, ok := row["geom_latitude"]

				if !ok {
					continue
				}

				str_lon, ok := row["geom_longitude"]

				if !ok {
					continue
				}

				lat, err := strconv.ParseFloat(str_lat, 64)

				if err != nil {
					log.Println(err, str_lat)
					continue
				}

				lon, err := strconv.ParseFloat(str_lon, 64)

				if err != nil {
					log.Println(err, str_lon)
					continue
				}

				g.DrawDot(lat, lon, 0.01, globe.Color(green))
			}
		}

	} else if *mode == "repo" {

		for _, path := range flag.Args() {

			var cb = func(path string, info os.FileInfo) error {

				if info.IsDir() {
					return nil
				}

				is_wof, err := uri.IsWOFFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is a WOF file, because %s\n", path, err)
					return err
				}

				if !is_wof {
					return nil
				}

				is_alt, err := uri.IsAltFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is an alt (WOF) file, because %s\n", path, err)
					return err
				}

				if is_alt {
					return nil
				}

				fh, err := os.Open(path)

				if err != nil {
					log.Printf("failed to open %s, because %s\n", path, err)
					return err
				}

				defer fh.Close()

				feature, err := ioutil.ReadAll(fh)

				if err != nil {
					log.Printf("failed to read %s, because %s\n", path, err)
					return err
				}

				return DrawFeature(feature, g)
			}

			cr := crawl.NewCrawler(path)
			cr.Crawl(cb)
		}

	} else {

		log.Fatal("Invalid mode")
	}

	t2 := time.Since(t1)

	log.Printf("time to read all the things %v\n", t2)

	t3 := time.Now()

	g.CenterOn(*center_lat, *center_lon)
	err := g.SavePNG(*outfile, *size)

	t4 := time.Since(t3)

	log.Printf("time to draw all the things %v\n", t4)

	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/cozy/afero"
	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 20 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(manFilename string, log *logrus.Entry) *gitFetcher {
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

// ManifestClient is the client used to HTTP resources from the git fetcher. It
// is exported for tests purposes only.
var ManifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if isGitSSHScheme(src.Scheme) {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("Fetching manifest on %s", u)
	res, err := ManifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

// Use the git archive method to download a manifest from the git repository.
func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	g.log.Infof("Fetching manifest %s", strings.Join(cmd.Args, " "))
	stdout, err := cmd.Output()
	if err != nil {
		if err == exec.ErrNotFound {
			return nil, ErrNotSupportedSource
		}
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
		return ioutil.NopCloser(buf), nil
	}
	return nil, ErrManifestNotReachable
}

func (g *gitFetcher) Fetch(src *url.URL, fs Copier, man Manifest) (err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("Error while fetching or copying repository %s: %s",
				src.String(), err.Error())
		}
	}()

	osFs := afero.NewOsFs()
	gitDir, err := afero.TempDir(osFs, "", "cozy-app-"+man.Slug())
	if err != nil {
		return err
	}
	defer osFs.RemoveAll(gitDir)

	gitFs := afero.NewBasePathFs(osFs, gitDir)
	// XXX Gitlab doesn't support the git protocol
	if src.Scheme == "git" && isGitlab(src) {
		src.Scheme = "https"
	}

	// If the scheme uses ssh, we have to use the git command.
	if isGitSSHScheme(src.Scheme) {
		err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
		if err == exec.ErrNotFound {
			return ErrNotSupportedSource
		}
		return err
	}

	err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
	if err != exec.ErrNotFound {
		return err
	}

	return g.fetchWithGoGit(gitDir, src, fs, man)
}

func (g *gitFetcher) fetchWithGit(gitFs afero.Fs, gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)
	srcStr := src.String()

	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()

	// The first command we execute is a ls-remote to check the last commit from
	// the remote branch and see if we already have a checked-out version of this
	// tree.
	cmd := exec.CommandContext(ctx, "git",
		"ls-remote", "--quiet",
		srcStr, fmt.Sprintf("refs/heads/%s", branch)) // #nosec
	lsRemote, err := cmd.Output()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("ls-remote error of %s: %s",
				strings.Join(cmd.Args, " "), err.Error())
		}
		return err
	}

	lsRemoteFields := bytes.Fields(lsRemote)
	if len(lsRemoteFields) == 0 {
		return fmt.Errorf("git: unexpected ls-remote output")
	}

	slug := man.Slug()
	version := man.Version() + "-" + string(lsRemoteFields[0])

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil || exists {
		return err
	}
	defer func() {
		if err != nil {
			fs.Abort()
		} else {
			err = fs.Commit()
		}
	}()

	cmd = exec.CommandContext(ctx, "git",
		"clone",
		"--quiet",
		"--depth", "1",
		"--single-branch",
		"--branch", branch,
		"--", srcStr, gitDir) // #nosec

	g.log.Infof("Clone with git: %s", strings.Join(cmd.Args, " "))
	stdoutStderr, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("Clone error of %s %s: %s", srcStr, stdoutStderr,
				err.Error())
		}
		return err
	}

	return afero.Walk(gitFs, "/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if info.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		src, err := gitFs.Open(path)
		if err != nil {
			return err
		}
		return fs.Copy(&fileInfo{
			name: path,
			size: info.Size(),
			mode: info.Mode(),
		}, src)
	})
}

func (g *gitFetcher) fetchWithGoGit(gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)

	storage, err := gitStorage.NewStorage(gitOsFS.New(gitDir))
	if err != nil {
		return err
	}

	errch := make(chan error)
	repch := make(chan *git.Repository)

	srcStr := src.String()
	g.log.Infof("Clone with go-git %s %s in %s", srcStr, branch, gitDir)
	go func() {
		repc, errc := git.Clone(storage, nil, &git.CloneOptions{
			URL:           srcStr,
			Depth:         1,
			SingleBranch:  true,
			ReferenceName: gitPlumbing.ReferenceName(branch),
		})
		if errc != nil {
			errch <- errc
		} else {
			repch <- repc
		}
	}()

	var rep *git.Repository
	select {
	case rep = <-repch:
	case err = <-errch:
		g.log.Errorf("Clone error of %s: %s", srcStr, err.Error())
		return err
	case <-time.After(cloneTimeout):
		g.log.Errorf("Clone timeout of %s", srcStr)
		return errCloneTimeout
	}

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	slug := man.Slug()
	version := man.Version() + "-" + ref.Hash().String()

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil || exists {
		return err
	}
	defer func() {
		if err != nil {
			fs.Abort()
		} else {
			err = fs.Commit()
		}
	}()

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		var r io.ReadCloser
		r, err = f.Reader()
		if err != nil {
			return err
		}
		defer r.Close()
		return fs.Copy(&fileInfo{
			name: f.Name,
			size: f.Size,
			mode: os.FileMode(f.Mode),
		}, r)
	})
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

func getRemoteURL(src *url.URL) (*url.URL, string) {
	branch := src.Fragment
	if branch == "" {
		branch = "master"
	}
	clonedSrc := *src
	clonedSrc.Fragment = ""
	return &clonedSrc, branch
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

func isGitSSHScheme(scheme string) bool {
	return scheme == "git+ssh" || scheme == "ssh+git"
}

var (
	_ Fetcher = &gitFetcher{}
)
<commit_msg>Increase git clone timeout<commit_after>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/cozy/afero"
	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 30 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(manFilename string, log *logrus.Entry) *gitFetcher {
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

// ManifestClient is the client used to HTTP resources from the git fetcher. It
// is exported for tests purposes only.
var ManifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if isGitSSHScheme(src.Scheme) {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("Fetching manifest on %s", u)
	res, err := ManifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

// Use the git archive method to download a manifest from the git repository.
func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	g.log.Infof("Fetching manifest %s", strings.Join(cmd.Args, " "))
	stdout, err := cmd.Output()
	if err != nil {
		if err == exec.ErrNotFound {
			return nil, ErrNotSupportedSource
		}
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
		return ioutil.NopCloser(buf), nil
	}
	return nil, ErrManifestNotReachable
}

func (g *gitFetcher) Fetch(src *url.URL, fs Copier, man Manifest) (err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("Error while fetching or copying repository %s: %s",
				src.String(), err.Error())
		}
	}()

	osFs := afero.NewOsFs()
	gitDir, err := afero.TempDir(osFs, "", "cozy-app-"+man.Slug())
	if err != nil {
		return err
	}
	defer osFs.RemoveAll(gitDir)

	gitFs := afero.NewBasePathFs(osFs, gitDir)
	// XXX Gitlab doesn't support the git protocol
	if src.Scheme == "git" && isGitlab(src) {
		src.Scheme = "https"
	}

	// If the scheme uses ssh, we have to use the git command.
	if isGitSSHScheme(src.Scheme) {
		err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
		if err == exec.ErrNotFound {
			return ErrNotSupportedSource
		}
		return err
	}

	err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
	if err != exec.ErrNotFound {
		return err
	}

	return g.fetchWithGoGit(gitDir, src, fs, man)
}

func (g *gitFetcher) fetchWithGit(gitFs afero.Fs, gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)
	srcStr := src.String()

	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()

	// The first command we execute is a ls-remote to check the last commit from
	// the remote branch and see if we already have a checked-out version of this
	// tree.
	cmd := exec.CommandContext(ctx, "git",
		"ls-remote", "--quiet",
		srcStr, fmt.Sprintf("refs/heads/%s", branch)) // #nosec
	lsRemote, err := cmd.Output()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("ls-remote error of %s: %s",
				strings.Join(cmd.Args, " "), err.Error())
		}
		return err
	}

	lsRemoteFields := bytes.Fields(lsRemote)
	if len(lsRemoteFields) == 0 {
		return fmt.Errorf("git: unexpected ls-remote output")
	}

	slug := man.Slug()
	version := man.Version() + "-" + string(lsRemoteFields[0])

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil || exists {
		return err
	}
	defer func() {
		if err != nil {
			fs.Abort()
		} else {
			err = fs.Commit()
		}
	}()

	cmd = exec.CommandContext(ctx, "git",
		"clone",
		"--quiet",
		"--depth", "1",
		"--single-branch",
		"--branch", branch,
		"--", srcStr, gitDir) // #nosec

	g.log.Infof("Clone with git: %s", strings.Join(cmd.Args, " "))
	stdoutStderr, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("Clone error of %s %s: %s", srcStr, stdoutStderr,
				err.Error())
		}
		return err
	}

	return afero.Walk(gitFs, "/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if info.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		src, err := gitFs.Open(path)
		if err != nil {
			return err
		}
		return fs.Copy(&fileInfo{
			name: path,
			size: info.Size(),
			mode: info.Mode(),
		}, src)
	})
}

func (g *gitFetcher) fetchWithGoGit(gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)

	storage, err := gitStorage.NewStorage(gitOsFS.New(gitDir))
	if err != nil {
		return err
	}

	errch := make(chan error)
	repch := make(chan *git.Repository)

	srcStr := src.String()
	g.log.Infof("Clone with go-git %s %s in %s", srcStr, branch, gitDir)
	go func() {
		repc, errc := git.Clone(storage, nil, &git.CloneOptions{
			URL:           srcStr,
			Depth:         1,
			SingleBranch:  true,
			ReferenceName: gitPlumbing.ReferenceName(branch),
		})
		if errc != nil {
			errch <- errc
		} else {
			repch <- repc
		}
	}()

	var rep *git.Repository
	select {
	case rep = <-repch:
	case err = <-errch:
		g.log.Errorf("Clone error of %s: %s", srcStr, err.Error())
		return err
	case <-time.After(cloneTimeout):
		g.log.Errorf("Clone timeout of %s", srcStr)
		return errCloneTimeout
	}

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	slug := man.Slug()
	version := man.Version() + "-" + ref.Hash().String()

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil || exists {
		return err
	}
	defer func() {
		if err != nil {
			fs.Abort()
		} else {
			err = fs.Commit()
		}
	}()

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		var r io.ReadCloser
		r, err = f.Reader()
		if err != nil {
			return err
		}
		defer r.Close()
		return fs.Copy(&fileInfo{
			name: f.Name,
			size: f.Size,
			mode: os.FileMode(f.Mode),
		}, r)
	})
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

func getRemoteURL(src *url.URL) (*url.URL, string) {
	branch := src.Fragment
	if branch == "" {
		branch = "master"
	}
	clonedSrc := *src
	clonedSrc.Fragment = ""
	return &clonedSrc, branch
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

func isGitSSHScheme(scheme string) bool {
	return scheme == "git+ssh" || scheme == "ssh+git"
}

var (
	_ Fetcher = &gitFetcher{}
)
<|endoftext|>


<commit_before>package backends

import (
	"database/sql"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/openshift/osin"

	"github.com/liut/staffio/pkg/models/oauth"
	"github.com/liut/staffio/pkg/settings"
)

var (
	clients_sortable_fields           = []string{"id", "created"}
	_                       OSINStore = (*DbStorage)(nil)
)

type OSINStore interface {
	osin.Storage
	LoadClients(limit, offset int, sort map[string]int) ([]*oauth.Client, error)
	CountClients() uint
	GetClientWithCode(code string) (*oauth.Client, error)
	SaveClient(client *oauth.Client) error
	LoadScopes() (scopes []*oauth.Scope, err error)
	IsAuthorized(client_id, username string) bool
	SaveAuthorized(client_id, username string) error
}

type DbStorage struct {
	refresh map[string]string
	isDebug bool
}

func NewStorage() *DbStorage {

	s := &DbStorage{
		refresh: make(map[string]string),
		isDebug: settings.Debug,
	}

	return s
}

func (s *DbStorage) Clone() osin.Storage {
	return s
}

func (s *DbStorage) Close() {
}

func (s *DbStorage) logf(format string, args ...interface{}) {
	if s.isDebug {
		log.Printf(format, args...)
	}
}

func (s *DbStorage) GetClient(id string) (osin.Client, error) {
	s.logf("GetClient: '%s'", id)
	c, err := s.GetClientWithCode(id)
	if err == nil {
		return c, nil
	}
	return nil, fmt.Errorf("Client %q not found", id)
}

func (s *DbStorage) SaveAuthorize(data *osin.AuthorizeData) error {
	s.logf("SaveAuthorize: '%s'\n", data.Code)
	qs := func(tx dbTxer) error {
		sql := `INSERT INTO
		 oauth_authorization_code(code, client_id, username, redirect_uri, expires_in, scopes, created)
		 VALUES($1, $2, $3, $4, $5, $6, $7);`
		r, err := tx.Exec(sql, data.Code, data.Client.GetId(), data.UserData.(string),
			data.RedirectUri, data.ExpiresIn, data.Scope, data.CreatedAt)
		if err != nil {
			return err
		}

		s.logf("save authorizeData code %s OK %v", data.Code, r)

		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadAuthorize(code string) (*osin.AuthorizeData, error) {
	s.logf("LoadAuthorize: '%s'\n", code)
	var (
		client_id string
		username  string
		err       error
	)
	a := &osin.AuthorizeData{Code: code}
	qs := func(db dber) error {
		return db.QueryRow(`SELECT client_id, username, redirect_uri, expires_in, scopes, created
		 FROM oauth_authorization_code WHERE code = $1`,
			code).Scan(&client_id, &username, &a.RedirectUri, &a.ExpiresIn, &a.Scope, &a.CreatedAt)
	}
	err = withDbQuery(qs)
	if err == nil {
		a.UserData = username
		a.Client, err = s.GetClientWithCode(client_id)
		if err != nil {
			return nil, err
		}
		s.logf("loaded authorization ok, createdAt %s", a.CreatedAt)
		return a, nil
	}

	s.logf("load authorize error: %s", err)
	return nil, fmt.Errorf("Authorize %q not found", code)
}

func (s *DbStorage) RemoveAuthorize(code string) error {
	s.logf("RemoveAuthorize: '%s'\n", code)
	if code == "" {
		log.Print("authorize code is empty")
		return nil
	}
	qs := func(tx dbTxer) error {
		sql := `DELETE FROM oauth_authorization_code WHERE code = $1;`
		r, err := tx.Exec(sql, code)
		if err != nil {
			return err
		}

		s.logf("delete authorizeData code %s OK %v", code, r)

		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) SaveAccess(data *osin.AccessData) error {
	s.logf("SaveAccess: '%s'\n", data.AccessToken)
	qs := func(tx dbTxer) error {
		str := `INSERT INTO
		 oauth_access_token(client_id, username, access_token, refresh_token, expires_in, scopes, created)
		 VALUES($1, $2, $3, $4, $5, $6, $7);`
		r, err := tx.Exec(str, data.Client.GetId(), data.UserData.(string),
			data.AccessToken, data.RefreshToken, data.ExpiresIn, data.Scope, data.CreatedAt)
		if err != nil {
			return err
		}

		s.logf("save AccessData token %s OK %v", data.AccessToken, r)

		if data.RefreshToken != "" {
			s.refresh[data.RefreshToken] = data.AccessToken
		}
		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadAccess(code string) (*osin.AccessData, error) {
	s.logf("LoadAccess: '%s'", code)
	var (
		client_id string
		username  string
		err       error
		is_frozen bool
		id        int
	)
	a := &osin.AccessData{AccessToken: code}
	qs := func(db dber) error {
		return db.QueryRow(`SELECT id, client_id, username, refresh_token, expires_in, scopes, is_frozen, created
		 FROM oauth_access_token WHERE access_token = $1`,
			code).Scan(&id, &client_id, &username, &a.RefreshToken, &a.ExpiresIn, &a.Scope, &is_frozen, &a.CreatedAt)
	}
	err = withDbQuery(qs)
	if err == nil {
		a.UserData = username
		a.Client, err = s.GetClientWithCode(client_id)
		if err != nil {
			return nil, err
		}
		s.logf("access token '%d' expires: \n\t%s created \n\t%s expire_at \n\t%s now \n\tis_expired %v", id, a.CreatedAt, a.ExpireAt(), time.Now(), a.IsExpired())
		return a, nil
	}

	log.Printf("load access error: %s", err)
	return nil, fmt.Errorf("AccessToken %q not found", code)
}

func (s *DbStorage) RemoveAccess(code string) error {
	s.logf("RemoveAccess: %s\n", code)
	qs := func(tx dbTxer) error {
		str := `DELETE FROM oauth_access_token WHERE access_token = $1;`
		r, err := tx.Exec(str, code)
		if err != nil {
			return err
		}

		s.logf("delete accessToken %s OK %v", code, r)

		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadRefresh(code string) (*osin.AccessData, error) {
	s.logf("LoadRefresh: %s\n", code)
	if d, ok := s.refresh[code]; ok {
		return s.LoadAccess(d)
	}
	return nil, fmt.Errorf("RefreshToken %q not found", code)
}

func (s *DbStorage) RemoveRefresh(code string) error {
	log.Printf("RemoveRefresh: %s\n", code)
	delete(s.refresh, code)
	return nil
}

func (s *DbStorage) GetClientWithCode(code string) (*oauth.Client, error) {
	c := new(oauth.Client)
	qs := func(db dber) error {
		return db.QueryRow("SELECT id, name, code, secret, redirect_uri, created FROM oauth_client WHERE code = $1",
			code).Scan(&c.Id, &c.Name, &c.Code, &c.Secret, &c.RedirectUri, &c.CreatedAt)
	}
	if err := withDbQuery(qs); err != nil {
		log.Printf("GetClientWithCode ERROR: %s", err)
		return nil, err
	}
	return c, nil
}

func (s *DbStorage) LoadClients(limit, offset int, sort map[string]int) (clients []*oauth.Client, err error) {
	if limit < 1 {
		limit = 1
	}
	if offset < 0 {
		offset = 0
	}

	var orders []string
	for k, v := range sort {
		if inArray(k, clients_sortable_fields) {
			var o string
			if v == ASCENDING {
				o = "ASC"
			} else {
				o = "DESC"
			}
			orders = append(orders, k+" "+o)
		}
	}

	str := `SELECT id, name, code, secret, redirect_uri, created
	  , allowed_grant_types, allowed_response_types, allowed_scopes
	   FROM oauth_client `

	if len(orders) > 0 {
		str = str + " ORDER BY " + strings.Join(orders, ",")
	}

	str = fmt.Sprintf("%s LIMIT %d OFFSET %d", str, limit, offset)

	clients = make([]*oauth.Client, 0)
	qs := func(db dber) error {
		rows, err := db.Query(str)
		if err != nil {
			log.Printf("db query error: %s for sql %s", err, str)
			return err
		}
		defer rows.Close()
		for rows.Next() {
			c := new(oauth.Client)
			var (
				grandTypes, responseTypes, scopes string
			)
			err = rows.Scan(&c.Id, &c.Name, &c.Code, &c.Secret, &c.RedirectUri, &c.CreatedAt,
				&grandTypes, &responseTypes, &scopes)
			if err != nil {
				log.Printf("rows scan error: %s", err)
				continue
			}
			c.AllowedGrantTypes = strings.Split(grandTypes, ",")
			c.AllowedResponseTypes = strings.Split(responseTypes, ",")
			c.AllowedScopes = strings.Split(scopes, ",")
			clients = append(clients, c)
		}
		return rows.Err()
	}

	if err := withDbQuery(qs); err != nil {
		return nil, err
	}

	return clients, nil
}

func (s *DbStorage) CountClients() (total uint) {
	qs := func(db dber) error {
		return db.QueryRow("SELECT COUND(id) FROM oauth_client").Scan(&total)
	}
	withDbQuery(qs)
	return
}

func (s *DbStorage) SaveClient(client *oauth.Client) error {
	log.Printf("SaveClient: id %d code %s", client.Id, client.Code)
	if client.Name == "" || client.Code == "" || client.Secret == "" || client.RedirectUri == "" {
		return valueError
	}
	qs := func(tx dbTxer) error {
		var err error
		if client.Id > 0 {
			str := `UPDATE oauth_client SET name = $1, code = $2, secret = $3, redirect_uri = $4
			 WHERE id = $5`
			var r sql.Result
			r, err = tx.Exec(str, client.Name, client.Code, client.Secret, client.RedirectUri, client.Id)
			log.Printf("UPDATE client result: %v", r)
		} else {
			str := `INSERT INTO
		 oauth_client(name, code, secret, redirect_uri, allowed_grant_types, allowed_scopes, created)
		 VALUES($1, $2, $3, $4, $5, $6, $7) RETURNING id;`
			err = tx.QueryRow(str,
				client.Name,
				client.Code,
				client.Secret,
				client.RedirectUri,
				strings.Join(client.AllowedGrantTypes, ","),
				strings.Join(client.AllowedScopes, ","),
				client.CreatedAt).Scan(&client.Id)
		}
		return err
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadScopes() (scopes []*oauth.Scope, err error) {
	scopes = make([]*oauth.Scope, 0)
	qs := func(db dber) error {
		rows, err := db.Query("SELECT name, label, description, is_default FROM oauth_scope")
		if err != nil {
			log.Printf("load scopes error: %s", err)
			return err
		}
		defer rows.Close()
		for rows.Next() {
			s := new(oauth.Scope)
			err = rows.Scan(&s.Name, &s.Label, &s.Description, &s.IsDefault)
			if err != nil {
				log.Printf("rows scan error: %s", err)
			}
			scopes = append(scopes, s)
		}
		return rows.Err()
	}

	if err := withDbQuery(qs); err != nil {
		return nil, err
	}

	return scopes, nil
}

func (s *DbStorage) IsAuthorized(client_id, username string) bool {
	var (
		created time.Time
	)
	if err := withDbQuery(func(db dber) error {
		return db.QueryRow("SELECT created FROM oauth_client_user_authorized WHERE client_id = $1 AND username = $2",
			client_id, username).Scan(&created)
	}); err != nil {
		log.Printf("load IsAuthorized(%s, %s) ERROR: %s", client_id, username, err)
		return false
	}
	return true
}

func (s *DbStorage) SaveAuthorized(client_id, username string) error {
	return withDbQuery(func(db dber) error {
		_, err := db.Exec("INSERT INTO oauth_client_user_authorized(client_id, username) VALUES($1, $2) ",
			client_id, username)
		return err
	})
}
<commit_msg>Use sync.Map instead of map<commit_after>package backends

import (
	"database/sql"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/openshift/osin"

	"github.com/liut/staffio/pkg/models/oauth"
	"github.com/liut/staffio/pkg/settings"
)

var (
	clientsSortableFields           = []string{"id", "created"}
	_                     OSINStore = (*DbStorage)(nil)
)

type OSINStore interface {
	osin.Storage
	LoadClients(limit, offset int, sort map[string]int) ([]*oauth.Client, error)
	CountClients() uint
	GetClientWithCode(code string) (*oauth.Client, error)
	SaveClient(client *oauth.Client) error
	LoadScopes() (scopes []*oauth.Scope, err error)
	IsAuthorized(client_id, username string) bool
	SaveAuthorized(client_id, username string) error
}

type DbStorage struct {
	refresh *sync.Map
	isDebug bool
}

func NewStorage() *DbStorage {

	s := &DbStorage{
		refresh: new(sync.Map),
		isDebug: settings.Debug,
	}

	return s
}

func (s *DbStorage) Clone() osin.Storage {
	return s
}

func (s *DbStorage) Close() {
}

func (s *DbStorage) logf(format string, args ...interface{}) {
	if s.isDebug {
		log.Printf(format, args...)
	}
}

func (s *DbStorage) GetClient(id string) (osin.Client, error) {
	s.logf("GetClient: '%s'", id)
	c, err := s.GetClientWithCode(id)
	if err == nil {
		return c, nil
	}
	return nil, fmt.Errorf("Client %q not found", id)
}

func (s *DbStorage) SaveAuthorize(data *osin.AuthorizeData) error {
	s.logf("SaveAuthorize: '%s'\n", data.Code)
	qs := func(tx dbTxer) error {
		sql := `INSERT INTO
		 oauth_authorization_code(code, client_id, username, redirect_uri, expires_in, scopes, created)
		 VALUES($1, $2, $3, $4, $5, $6, $7);`
		r, err := tx.Exec(sql, data.Code, data.Client.GetId(), data.UserData.(string),
			data.RedirectUri, data.ExpiresIn, data.Scope, data.CreatedAt)
		if err != nil {
			return err
		}

		s.logf("save authorizeData code %s OK %v", data.Code, r)

		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadAuthorize(code string) (*osin.AuthorizeData, error) {
	s.logf("LoadAuthorize: '%s'\n", code)
	var (
		client_id string
		username  string
		err       error
	)
	a := &osin.AuthorizeData{Code: code}
	qs := func(db dber) error {
		return db.QueryRow(`SELECT client_id, username, redirect_uri, expires_in, scopes, created
		 FROM oauth_authorization_code WHERE code = $1`,
			code).Scan(&client_id, &username, &a.RedirectUri, &a.ExpiresIn, &a.Scope, &a.CreatedAt)
	}
	err = withDbQuery(qs)
	if err == nil {
		a.UserData = username
		a.Client, err = s.GetClientWithCode(client_id)
		if err != nil {
			return nil, err
		}
		s.logf("loaded authorization ok, createdAt %s", a.CreatedAt)
		return a, nil
	}

	s.logf("load authorize error: %s", err)
	return nil, fmt.Errorf("Authorize %q not found", code)
}

func (s *DbStorage) RemoveAuthorize(code string) error {
	s.logf("RemoveAuthorize: '%s'\n", code)
	if code == "" {
		log.Print("authorize code is empty")
		return nil
	}
	qs := func(tx dbTxer) error {
		sql := `DELETE FROM oauth_authorization_code WHERE code = $1;`
		r, err := tx.Exec(sql, code)
		if err != nil {
			return err
		}

		s.logf("delete authorizeData code %s OK %v", code, r)

		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) SaveAccess(data *osin.AccessData) error {
	s.logf("SaveAccess: '%s'\n", data.AccessToken)
	qs := func(tx dbTxer) error {
		str := `INSERT INTO
		 oauth_access_token(client_id, username, access_token, refresh_token, expires_in, scopes, created)
		 VALUES($1, $2, $3, $4, $5, $6, $7);`
		r, err := tx.Exec(str, data.Client.GetId(), data.UserData.(string),
			data.AccessToken, data.RefreshToken, data.ExpiresIn, data.Scope, data.CreatedAt)
		if err != nil {
			return err
		}

		s.logf("save AccessData token %s OK %v", data.AccessToken, r)

		if data.RefreshToken != "" {
			s.refresh.Store(data.RefreshToken, data.AccessToken)
		}
		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadAccess(code string) (*osin.AccessData, error) {
	s.logf("LoadAccess: '%s'", code)
	var (
		client_id string
		username  string
		err       error
		is_frozen bool
		id        int
	)
	a := &osin.AccessData{AccessToken: code}
	qs := func(db dber) error {
		return db.QueryRow(`SELECT id, client_id, username, refresh_token, expires_in, scopes, is_frozen, created
		 FROM oauth_access_token WHERE access_token = $1`,
			code).Scan(&id, &client_id, &username, &a.RefreshToken, &a.ExpiresIn, &a.Scope, &is_frozen, &a.CreatedAt)
	}
	err = withDbQuery(qs)
	if err == nil {
		a.UserData = username
		a.Client, err = s.GetClientWithCode(client_id)
		if err != nil {
			return nil, err
		}
		s.logf("access token '%d' expires: \n\t%s created \n\t%s expire_at \n\t%s now \n\tis_expired %v", id, a.CreatedAt, a.ExpireAt(), time.Now(), a.IsExpired())
		return a, nil
	}

	log.Printf("load access error: %s", err)
	return nil, fmt.Errorf("AccessToken %q not found", code)
}

func (s *DbStorage) RemoveAccess(code string) error {
	s.logf("RemoveAccess: %s\n", code)
	qs := func(tx dbTxer) error {
		str := `DELETE FROM oauth_access_token WHERE access_token = $1;`
		r, err := tx.Exec(str, code)
		if err != nil {
			return err
		}

		s.logf("delete accessToken %s OK %v", code, r)

		return nil
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadRefresh(code string) (*osin.AccessData, error) {
	s.logf("LoadRefresh: %s\n", code)
	if v, ok := s.refresh.Load(code); ok {
		return s.LoadAccess(v.(string))
	}
	return nil, fmt.Errorf("RefreshToken %q not found", code)
}

func (s *DbStorage) RemoveRefresh(code string) error {
	log.Printf("RemoveRefresh: %s\n", code)
	s.refresh.Delete(code)
	return nil
}

func (s *DbStorage) GetClientWithCode(code string) (*oauth.Client, error) {
	c := new(oauth.Client)
	qs := func(db dber) error {
		return db.QueryRow("SELECT id, name, code, secret, redirect_uri, created FROM oauth_client WHERE code = $1",
			code).Scan(&c.Id, &c.Name, &c.Code, &c.Secret, &c.RedirectUri, &c.CreatedAt)
	}
	if err := withDbQuery(qs); err != nil {
		log.Printf("GetClientWithCode ERROR: %s", err)
		return nil, err
	}
	return c, nil
}

func (s *DbStorage) LoadClients(limit, offset int, sort map[string]int) (clients []*oauth.Client, err error) {
	if limit < 1 {
		limit = 1
	}
	if offset < 0 {
		offset = 0
	}

	var orders []string
	for k, v := range sort {
		if inArray(k, clientsSortableFields) {
			var o string
			if v == ASCENDING {
				o = "ASC"
			} else {
				o = "DESC"
			}
			orders = append(orders, k+" "+o)
		}
	}

	str := `SELECT id, name, code, secret, redirect_uri, created
	  , allowed_grant_types, allowed_response_types, allowed_scopes
	   FROM oauth_client `

	if len(orders) > 0 {
		str = str + " ORDER BY " + strings.Join(orders, ",")
	}

	str = fmt.Sprintf("%s LIMIT %d OFFSET %d", str, limit, offset)

	clients = make([]*oauth.Client, 0)
	qs := func(db dber) error {
		rows, err := db.Query(str)
		if err != nil {
			log.Printf("db query error: %s for sql %s", err, str)
			return err
		}
		defer rows.Close()
		for rows.Next() {
			c := new(oauth.Client)
			var (
				grandTypes, responseTypes, scopes string
			)
			err = rows.Scan(&c.Id, &c.Name, &c.Code, &c.Secret, &c.RedirectUri, &c.CreatedAt,
				&grandTypes, &responseTypes, &scopes)
			if err != nil {
				log.Printf("rows scan error: %s", err)
				continue
			}
			c.AllowedGrantTypes = strings.Split(grandTypes, ",")
			c.AllowedResponseTypes = strings.Split(responseTypes, ",")
			c.AllowedScopes = strings.Split(scopes, ",")
			clients = append(clients, c)
		}
		return rows.Err()
	}

	if err := withDbQuery(qs); err != nil {
		return nil, err
	}

	return clients, nil
}

func (s *DbStorage) CountClients() (total uint) {
	qs := func(db dber) error {
		return db.QueryRow("SELECT COUND(id) FROM oauth_client").Scan(&total)
	}
	withDbQuery(qs)
	return
}

func (s *DbStorage) SaveClient(client *oauth.Client) error {
	log.Printf("SaveClient: id %d code %s", client.Id, client.Code)
	if client.Name == "" || client.Code == "" || client.Secret == "" || client.RedirectUri == "" {
		return valueError
	}
	qs := func(tx dbTxer) error {
		var err error
		if client.Id > 0 {
			str := `UPDATE oauth_client SET name = $1, code = $2, secret = $3, redirect_uri = $4
			 WHERE id = $5`
			var r sql.Result
			r, err = tx.Exec(str, client.Name, client.Code, client.Secret, client.RedirectUri, client.Id)
			log.Printf("UPDATE client result: %v", r)
		} else {
			str := `INSERT INTO
		 oauth_client(name, code, secret, redirect_uri, allowed_grant_types, allowed_scopes, created)
		 VALUES($1, $2, $3, $4, $5, $6, $7) RETURNING id;`
			err = tx.QueryRow(str,
				client.Name,
				client.Code,
				client.Secret,
				client.RedirectUri,
				strings.Join(client.AllowedGrantTypes, ","),
				strings.Join(client.AllowedScopes, ","),
				client.CreatedAt).Scan(&client.Id)
		}
		return err
	}
	return withTxQuery(qs)
}

func (s *DbStorage) LoadScopes() (scopes []*oauth.Scope, err error) {
	scopes = make([]*oauth.Scope, 0)
	qs := func(db dber) error {
		rows, err := db.Query("SELECT name, label, description, is_default FROM oauth_scope")
		if err != nil {
			log.Printf("load scopes error: %s", err)
			return err
		}
		defer rows.Close()
		for rows.Next() {
			s := new(oauth.Scope)
			err = rows.Scan(&s.Name, &s.Label, &s.Description, &s.IsDefault)
			if err != nil {
				log.Printf("rows scan error: %s", err)
			}
			scopes = append(scopes, s)
		}
		return rows.Err()
	}

	if err := withDbQuery(qs); err != nil {
		return nil, err
	}

	return scopes, nil
}

func (s *DbStorage) IsAuthorized(client_id, username string) bool {
	var (
		created time.Time
	)
	if err := withDbQuery(func(db dber) error {
		return db.QueryRow("SELECT created FROM oauth_client_user_authorized WHERE client_id = $1 AND username = $2",
			client_id, username).Scan(&created)
	}); err != nil {
		log.Printf("load IsAuthorized(%s, %s) ERROR: %s", client_id, username, err)
		return false
	}
	return true
}

func (s *DbStorage) SaveAuthorized(client_id, username string) error {
	return withDbQuery(func(db dber) error {
		_, err := db.Exec("INSERT INTO oauth_client_user_authorized(client_id, username) VALUES($1, $2) ",
			client_id, username)
		return err
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"io"
	"net"

	"github.com/spf13/cobra"

	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apiserver/pkg/admission"
	genericapiserver "k8s.io/apiserver/pkg/server"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/sample-apiserver/pkg/admission/plugin/banflunder"
	"k8s.io/sample-apiserver/pkg/admission/wardleinitializer"
	"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
	"k8s.io/sample-apiserver/pkg/apiserver"
	clientset "k8s.io/sample-apiserver/pkg/generated/clientset/versioned"
	informers "k8s.io/sample-apiserver/pkg/generated/informers/externalversions"
)

const defaultEtcdPathPrefix = "/registry/wardle.kubernetes.io"

type WardleServerOptions struct {
	RecommendedOptions *genericoptions.RecommendedOptions

	SharedInformerFactory informers.SharedInformerFactory
	StdOut                io.Writer
	StdErr                io.Writer
}

func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
	o := &WardleServerOptions{
		RecommendedOptions: genericoptions.NewRecommendedOptions(
			defaultEtcdPathPrefix,
			apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion),
			genericoptions.NewProcessInfo("wardle-apiserver", "wardle"),
		),

		StdOut: out,
		StdErr: errOut,
	}

	return o
}

// NewCommandStartWardleServer provides a CLI handler for 'start master' command
// with a default WardleServerOptions.
func NewCommandStartWardleServer(defaults *WardleServerOptions, stopCh <-chan struct{}) *cobra.Command {
	o := *defaults
	cmd := &cobra.Command{
		Short: "Launch a wardle API server",
		Long:  "Launch a wardle API server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Complete(); err != nil {
				return err
			}
			if err := o.Validate(args); err != nil {
				return err
			}
			if err := o.RunWardleServer(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	flags := cmd.Flags()
	o.RecommendedOptions.AddFlags(flags)

	return cmd
}

func (o WardleServerOptions) Validate(args []string) error {
	errors := []error{}
	errors = append(errors, o.RecommendedOptions.Validate()...)
	return utilerrors.NewAggregate(errors)
}

func (o *WardleServerOptions) Complete() error {
	// register admission plugins
	banflunder.Register(o.RecommendedOptions.Admission.Plugins)

	// add admisison plugins to the RecommendedPluginOrder
	o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder")

	return nil
}

func (o *WardleServerOptions) Config() (*apiserver.Config, error) {
	// TODO have a "real" external address
	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	o.RecommendedOptions.ExtraAdmissionInitializers = func(c *genericapiserver.RecommendedConfig) ([]admission.PluginInitializer, error) {
		client, err := clientset.NewForConfig(c.LoopbackClientConfig)
		if err != nil {
			return nil, err
		}
		informerFactory := informers.NewSharedInformerFactory(client, c.LoopbackClientConfig.Timeout)
		o.SharedInformerFactory = informerFactory
		return []admission.PluginInitializer{wardleinitializer.New(informerFactory)}, nil
	}

	serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs)
	if err := o.RecommendedOptions.ApplyTo(serverConfig, apiserver.Scheme); err != nil {
		return nil, err
	}

	config := &apiserver.Config{
		GenericConfig: serverConfig,
		ExtraConfig:   apiserver.ExtraConfig{},
	}
	return config, nil
}

func (o WardleServerOptions) RunWardleServer(stopCh <-chan struct{}) error {
	config, err := o.Config()
	if err != nil {
		return err
	}

	server, err := config.Complete().New()
	if err != nil {
		return err
	}

	server.GenericAPIServer.AddPostStartHook("start-sample-server-informers", func(context genericapiserver.PostStartHookContext) error {
		config.GenericConfig.SharedInformerFactory.Start(context.StopCh)
		o.SharedInformerFactory.Start(context.StopCh)
		return nil
	})

	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<commit_msg>prevent unhandled errors on colliding poststarthook registration<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"io"
	"net"

	"github.com/spf13/cobra"

	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apiserver/pkg/admission"
	genericapiserver "k8s.io/apiserver/pkg/server"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/sample-apiserver/pkg/admission/plugin/banflunder"
	"k8s.io/sample-apiserver/pkg/admission/wardleinitializer"
	"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
	"k8s.io/sample-apiserver/pkg/apiserver"
	clientset "k8s.io/sample-apiserver/pkg/generated/clientset/versioned"
	informers "k8s.io/sample-apiserver/pkg/generated/informers/externalversions"
)

const defaultEtcdPathPrefix = "/registry/wardle.kubernetes.io"

type WardleServerOptions struct {
	RecommendedOptions *genericoptions.RecommendedOptions

	SharedInformerFactory informers.SharedInformerFactory
	StdOut                io.Writer
	StdErr                io.Writer
}

func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
	o := &WardleServerOptions{
		RecommendedOptions: genericoptions.NewRecommendedOptions(
			defaultEtcdPathPrefix,
			apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion),
			genericoptions.NewProcessInfo("wardle-apiserver", "wardle"),
		),

		StdOut: out,
		StdErr: errOut,
	}

	return o
}

// NewCommandStartWardleServer provides a CLI handler for 'start master' command
// with a default WardleServerOptions.
func NewCommandStartWardleServer(defaults *WardleServerOptions, stopCh <-chan struct{}) *cobra.Command {
	o := *defaults
	cmd := &cobra.Command{
		Short: "Launch a wardle API server",
		Long:  "Launch a wardle API server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Complete(); err != nil {
				return err
			}
			if err := o.Validate(args); err != nil {
				return err
			}
			if err := o.RunWardleServer(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	flags := cmd.Flags()
	o.RecommendedOptions.AddFlags(flags)

	return cmd
}

func (o WardleServerOptions) Validate(args []string) error {
	errors := []error{}
	errors = append(errors, o.RecommendedOptions.Validate()...)
	return utilerrors.NewAggregate(errors)
}

func (o *WardleServerOptions) Complete() error {
	// register admission plugins
	banflunder.Register(o.RecommendedOptions.Admission.Plugins)

	// add admisison plugins to the RecommendedPluginOrder
	o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder")

	return nil
}

func (o *WardleServerOptions) Config() (*apiserver.Config, error) {
	// TODO have a "real" external address
	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	o.RecommendedOptions.ExtraAdmissionInitializers = func(c *genericapiserver.RecommendedConfig) ([]admission.PluginInitializer, error) {
		client, err := clientset.NewForConfig(c.LoopbackClientConfig)
		if err != nil {
			return nil, err
		}
		informerFactory := informers.NewSharedInformerFactory(client, c.LoopbackClientConfig.Timeout)
		o.SharedInformerFactory = informerFactory
		return []admission.PluginInitializer{wardleinitializer.New(informerFactory)}, nil
	}

	serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs)
	if err := o.RecommendedOptions.ApplyTo(serverConfig, apiserver.Scheme); err != nil {
		return nil, err
	}

	config := &apiserver.Config{
		GenericConfig: serverConfig,
		ExtraConfig:   apiserver.ExtraConfig{},
	}
	return config, nil
}

func (o WardleServerOptions) RunWardleServer(stopCh <-chan struct{}) error {
	config, err := o.Config()
	if err != nil {
		return err
	}

	server, err := config.Complete().New()
	if err != nil {
		return err
	}

	server.GenericAPIServer.AddPostStartHookOrDie("start-sample-server-informers", func(context genericapiserver.PostStartHookContext) error {
		config.GenericConfig.SharedInformerFactory.Start(context.StopCh)
		o.SharedInformerFactory.Start(context.StopCh)
		return nil
	})

	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"os"
)

// GetConf returns *Root for unittest
// e.g. `conf, err = config.GetConf("settings.toml")`
func GetConf(fileName string) (*Root, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	basePath := fmt.Sprintf("%s/../../configs", pwd)

	return newConf(fmt.Sprintf("%s/%s", basePath, fileName))
}

// GetEnvConf returns *Root from environment variable `$GOING_CONF` for unittest
func GetEnvConf() (*Root, error) {
	return newConf(os.Getenv("GO_GIN_CONF"))
}

func newConf(filePath string) (*Root, error) {
	conf, err := NewConfig(filePath, false)
	if err != nil {
		return nil, err
	}
	return conf, nil
}
<commit_msg>refactoring<commit_after>package config

import (
	"fmt"
	"os"
)

// GetConf returns *Root for unittest
// e.g. `conf, err = config.GetConf("settings.toml")`
func GetConf(fileName string) (*Root, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	basePath := fmt.Sprintf("%s/../../configs", pwd)

	return newConf(fmt.Sprintf("%s/%s", basePath, fileName))
}

// GetEnvConf returns *Root from environment variable `$GO-GIN_CONF` for unittest
func GetEnvConf() (*Root, error) {
	return newConf(os.Getenv("GO_GIN_CONF"))
}

func newConf(filePath string) (*Root, error) {
	conf, err := NewConfig(filePath, false)
	if err != nil {
		return nil, err
	}
	return conf, nil
}
<|endoftext|>


<commit_before>package helm

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	helm_v3 "helm.sh/helm/v3/cmd/helm"
	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/cli"
	helm_kube "helm.sh/helm/v3/pkg/kube"
	kubefake "helm.sh/helm/v3/pkg/kube/fake"
	"helm.sh/helm/v3/pkg/release"
	"helm.sh/helm/v3/pkg/storage/driver"

	"github.com/werf/kubedog/pkg/kube"
	"github.com/werf/logboek"
)

type InitActionConfigOptions struct {
	StatusProgressPeriod      time.Duration
	HooksStatusProgressPeriod time.Duration
	KubeConfigOptions         kube.KubeConfigOptions
	ReleasesHistoryMax        int
}

func InitActionConfig(ctx context.Context, kubeInitializer KubeInitializer, namespace string, envSettings *cli.EnvSettings, registryClientHandle *helm_v3.RegistryClientHandle, actionConfig *action.Configuration, opts InitActionConfigOptions) error {
	configGetter, err := kube.NewKubeConfigGetter(kube.KubeConfigGetterOptions{
		KubeConfigOptions: opts.KubeConfigOptions,
		Namespace:         namespace,
	})
	if err != nil {
		return fmt.Errorf("error creating kube config getter: %s", err)
	}

	*envSettings.GetConfigP() = configGetter
	*envSettings.GetNamespaceP() = namespace
	if opts.KubeConfigOptions.Context != "" {
		envSettings.KubeContext = opts.KubeConfigOptions.Context
	}
	if opts.KubeConfigOptions.ConfigPath != "" {
		envSettings.KubeConfig = opts.KubeConfigOptions.ConfigPath
	}
	if opts.ReleasesHistoryMax != 0 {
		envSettings.MaxHistory = opts.ReleasesHistoryMax
	}

	helmDriver := os.Getenv("HELM_DRIVER")
	if err := actionConfig.Init(envSettings.RESTClientGetter(), envSettings.Namespace(), helmDriver, logboek.Context(ctx).Debug().LogF); err != nil {
		return fmt.Errorf("action config init failed: %s", err)
	}
	if helmDriver == "memory" {
		loadReleasesInMemory(envSettings, actionConfig)
	}

	kubeClient := actionConfig.KubeClient.(*helm_kube.Client)
	kubeClient.ResourcesWaiter = NewResourcesWaiter(kubeInitializer, kubeClient, time.Now(), opts.StatusProgressPeriod, opts.HooksStatusProgressPeriod)
	kubeClient.Extender = NewHelmKubeClientExtender()

	actionConfig.RegistryClient = registryClientHandle.RegistryClient

	return nil
}

// This function loads releases into the memory storage if the
// environment variable is properly set.
func loadReleasesInMemory(envSettings *cli.EnvSettings, actionConfig *action.Configuration) {
	filePaths := strings.Split(os.Getenv("HELM_MEMORY_DRIVER_DATA"), ":")
	if len(filePaths) == 0 {
		return
	}

	store := actionConfig.Releases
	mem, ok := store.Driver.(*driver.Memory)
	if !ok {
		// For an unexpected reason we are not dealing with the memory storage driver.
		return
	}

	actionConfig.KubeClient = &kubefake.PrintingKubeClient{Out: ioutil.Discard}

	for _, path := range filePaths {
		b, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatal("Unable to read memory driver data", err)
		}

		releases := []*release.Release{}
		if err := yaml.Unmarshal(b, &releases); err != nil {
			log.Fatal("Unable to unmarshal memory driver data: ", err)
		}

		for _, rel := range releases {
			if err := store.Create(rel); err != nil {
				log.Fatal(err)
			}
		}
	}
	// Must reset namespace to the proper one
	mem.SetNamespace(envSettings.Namespace())
}
<commit_msg>[helm] Force helm kube client namespace initialization<commit_after>package helm

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	helm_v3 "helm.sh/helm/v3/cmd/helm"
	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/cli"
	helm_kube "helm.sh/helm/v3/pkg/kube"
	kubefake "helm.sh/helm/v3/pkg/kube/fake"
	"helm.sh/helm/v3/pkg/release"
	"helm.sh/helm/v3/pkg/storage/driver"

	"github.com/werf/kubedog/pkg/kube"
	"github.com/werf/logboek"
)

type InitActionConfigOptions struct {
	StatusProgressPeriod      time.Duration
	HooksStatusProgressPeriod time.Duration
	KubeConfigOptions         kube.KubeConfigOptions
	ReleasesHistoryMax        int
}

func InitActionConfig(ctx context.Context, kubeInitializer KubeInitializer, namespace string, envSettings *cli.EnvSettings, registryClientHandle *helm_v3.RegistryClientHandle, actionConfig *action.Configuration, opts InitActionConfigOptions) error {
	configGetter, err := kube.NewKubeConfigGetter(kube.KubeConfigGetterOptions{
		KubeConfigOptions: opts.KubeConfigOptions,
		Namespace:         namespace,
	})
	if err != nil {
		return fmt.Errorf("error creating kube config getter: %s", err)
	}

	*envSettings.GetConfigP() = configGetter
	*envSettings.GetNamespaceP() = namespace
	if opts.KubeConfigOptions.Context != "" {
		envSettings.KubeContext = opts.KubeConfigOptions.Context
	}
	if opts.KubeConfigOptions.ConfigPath != "" {
		envSettings.KubeConfig = opts.KubeConfigOptions.ConfigPath
	}
	if opts.ReleasesHistoryMax != 0 {
		envSettings.MaxHistory = opts.ReleasesHistoryMax
	}

	helmDriver := os.Getenv("HELM_DRIVER")
	if err := actionConfig.Init(envSettings.RESTClientGetter(), envSettings.Namespace(), helmDriver, logboek.Context(ctx).Debug().LogF); err != nil {
		return fmt.Errorf("action config init failed: %s", err)
	}
	if helmDriver == "memory" {
		loadReleasesInMemory(envSettings, actionConfig)
	}

	kubeClient := actionConfig.KubeClient.(*helm_kube.Client)
	kubeClient.Namespace = namespace
	kubeClient.ResourcesWaiter = NewResourcesWaiter(kubeInitializer, kubeClient, time.Now(), opts.StatusProgressPeriod, opts.HooksStatusProgressPeriod)
	kubeClient.Extender = NewHelmKubeClientExtender()

	actionConfig.RegistryClient = registryClientHandle.RegistryClient

	return nil
}

// This function loads releases into the memory storage if the
// environment variable is properly set.
func loadReleasesInMemory(envSettings *cli.EnvSettings, actionConfig *action.Configuration) {
	filePaths := strings.Split(os.Getenv("HELM_MEMORY_DRIVER_DATA"), ":")
	if len(filePaths) == 0 {
		return
	}

	store := actionConfig.Releases
	mem, ok := store.Driver.(*driver.Memory)
	if !ok {
		// For an unexpected reason we are not dealing with the memory storage driver.
		return
	}

	actionConfig.KubeClient = &kubefake.PrintingKubeClient{Out: ioutil.Discard}

	for _, path := range filePaths {
		b, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatal("Unable to read memory driver data", err)
		}

		releases := []*release.Release{}
		if err := yaml.Unmarshal(b, &releases); err != nil {
			log.Fatal("Unable to unmarshal memory driver data: ", err)
		}

		for _, rel := range releases {
			if err := store.Create(rel); err != nil {
				log.Fatal(err)
			}
		}
	}
	// Must reset namespace to the proper one
	mem.SetNamespace(envSettings.Namespace())
}
<|endoftext|>


<commit_before>package helm

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/flant/helm/pkg/chartutil"
	"github.com/flant/helm/pkg/lint"
	"github.com/flant/helm/pkg/lint/support"
)

type LintOptions struct {
	Strict bool
}

func Lint(out io.Writer, chartPath, namespace string, values, set, setString []string, opts LintOptions) error {
	var lowestTolerance int
	if opts.Strict {
		lowestTolerance = support.WarningSev
	} else {
		lowestTolerance = support.ErrorSev
	}

	rvals, err := vals(values, set, setString, []string{}, "", "", "")
	if err != nil {
		return err
	}

	var total int
	var failures int
	if linter, err := lintChart(chartPath, rvals, namespace, opts.Strict); err != nil {
		fmt.Fprintln(out, "==> Skipping", chartPath)
		fmt.Fprintln(out, err)
		if err == errors.New("no chart found for linting (missing Chart.yaml)") {
			failures = failures + 1
		}
	} else {
		fmt.Fprintln(out, "==> Linting", chartPath)

		if len(linter.Messages) == 0 {
			fmt.Fprintln(out, "Lint OK")
		}

		for _, msg := range linter.Messages {
			fmt.Fprintln(out, msg)
		}

		total = total + 1
		if linter.HighestSeverity >= lowestTolerance {
			failures = failures + 1
		}
	}
	fmt.Fprintln(out)

	msg := fmt.Sprintf("%d chart(s) linted", total)
	if failures > 0 {
		return fmt.Errorf("%s, %d chart(s) failed", msg, failures)
	}

	fmt.Fprintf(out, "%s, no failures\n", msg)

	return nil
}

func lintChart(path string, vals []byte, namespace string, strict bool) (support.Linter, error) {
	var chartPath string
	linter := support.Linter{}

	if strings.HasSuffix(path, ".tgz") {
		tempDir, err := ioutil.TempDir("", "helm-lint")
		if err != nil {
			return linter, err
		}
		defer os.RemoveAll(tempDir)

		file, err := os.Open(path)
		if err != nil {
			return linter, err
		}
		defer file.Close()

		if err = chartutil.Expand(tempDir, file); err != nil {
			return linter, err
		}

		lastHyphenIndex := strings.LastIndex(filepath.Base(path), "-")
		if lastHyphenIndex <= 0 {
			return linter, fmt.Errorf("unable to parse chart archive %q, missing '-'", filepath.Base(path))
		}
		base := filepath.Base(path)[:lastHyphenIndex]
		chartPath = filepath.Join(tempDir, base)
	} else {
		chartPath = path
	}

	// Guard: Error out of this is not a chart.
	if _, err := os.Stat(filepath.Join(chartPath, "Chart.yaml")); err != nil {
		return linter, errors.New("no chart found for linting (missing Chart.yaml)")
	}

	return lint.All(chartPath, vals, namespace, strict), nil
}
<commit_msg>[lint] Ignore Chart.yaml icon validation<commit_after>package helm

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/Masterminds/semver"
	"github.com/asaskevich/govalidator"

	"github.com/flant/helm/pkg/chartutil"
	"github.com/flant/helm/pkg/lint/rules"
	"github.com/flant/helm/pkg/lint/support"
	"github.com/flant/helm/pkg/proto/hapi/chart"
)

type LintOptions struct {
	Strict bool
}

func Lint(out io.Writer, chartPath, namespace string, values, set, setString []string, opts LintOptions) error {
	var lowestTolerance int
	if opts.Strict {
		lowestTolerance = support.WarningSev
	} else {
		lowestTolerance = support.ErrorSev
	}

	rvals, err := vals(values, set, setString, []string{}, "", "", "")
	if err != nil {
		return err
	}

	var total int
	var failures int
	if linter, err := lintChart(chartPath, rvals, namespace, opts.Strict); err != nil {
		fmt.Fprintln(out, "==> Skipping", chartPath)
		fmt.Fprintln(out, err)
		if err == errors.New("no chart found for linting (missing Chart.yaml)") {
			failures = failures + 1
		}
	} else {
		fmt.Fprintln(out, "==> Linting", chartPath)

		if len(linter.Messages) == 0 {
			fmt.Fprintln(out, "Lint OK")
		}

		for _, msg := range linter.Messages {
			fmt.Fprintln(out, msg)
		}

		total = total + 1
		if linter.HighestSeverity >= lowestTolerance {
			failures = failures + 1
		}
	}
	fmt.Fprintln(out)

	msg := fmt.Sprintf("%d chart(s) linted", total)
	if failures > 0 {
		return fmt.Errorf("%s, %d chart(s) failed", msg, failures)
	}

	fmt.Fprintf(out, "%s, no failures\n", msg)

	return nil
}

func lintChart(path string, vals []byte, namespace string, strict bool) (support.Linter, error) {
	var chartPath string
	linter := support.Linter{}

	if strings.HasSuffix(path, ".tgz") {
		tempDir, err := ioutil.TempDir("", "helm-lint")
		if err != nil {
			return linter, err
		}
		defer os.RemoveAll(tempDir)

		file, err := os.Open(path)
		if err != nil {
			return linter, err
		}
		defer file.Close()

		if err = chartutil.Expand(tempDir, file); err != nil {
			return linter, err
		}

		lastHyphenIndex := strings.LastIndex(filepath.Base(path), "-")
		if lastHyphenIndex <= 0 {
			return linter, fmt.Errorf("unable to parse chart archive %q, missing '-'", filepath.Base(path))
		}
		base := filepath.Base(path)[:lastHyphenIndex]
		chartPath = filepath.Join(tempDir, base)
	} else {
		chartPath = path
	}

	// Guard: Error out of this is not a chart.
	if _, err := os.Stat(filepath.Join(chartPath, "Chart.yaml")); err != nil {
		return linter, errors.New("no chart found for linting (missing Chart.yaml)")
	}

	return lintByRules(chartPath, vals, namespace, strict), nil
}

func lintByRules(basedir string, values []byte, namespace string, strict bool) support.Linter {
	// Using abs path to get directory context
	chartDir, _ := filepath.Abs(basedir)

	linter := support.Linter{ChartDir: chartDir}
	LintChartfileRules(&linter)
	rules.Values(&linter)
	rules.Templates(&linter, values, namespace, strict)
	return linter
}

func LintChartfileRules(linter *support.Linter) {
	chartFileName := "Chart.yaml"
	chartPath := filepath.Join(linter.ChartDir, chartFileName)

	linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartYamlNotDirectory(chartPath))

	chartFile, err := chartutil.LoadChartfile(chartPath)
	validChartFile := linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartYamlFormat(err))

	// Guard clause. Following linter rules require a parseable ChartFile
	if !validChartFile {
		return
	}

	linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartNamePresence(chartFile))
	linter.RunLinterRule(support.WarningSev, chartFileName, validateChartNameFormat(chartFile))
	linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartNameDirMatch(linter.ChartDir, chartFile))

	// Chart metadata
	linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartVersion(chartFile))
	linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartEngine(chartFile))
	linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartMaintainer(chartFile))
	linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartSources(chartFile))
	//linter.RunLinterRule(support.InfoSev, chartFileName, validateChartIconPresence(chartFile))
	//linter.RunLinterRule(support.ErrorSev, chartFileName, validateChartIconURL(chartFile))
}

func validateChartYamlNotDirectory(chartPath string) error {
	fi, err := os.Stat(chartPath)

	if err == nil && fi.IsDir() {
		return errors.New("should be a file, not a directory")
	}
	return nil
}

func validateChartYamlFormat(chartFileError error) error {
	if chartFileError != nil {
		return fmt.Errorf("unable to parse YAML\n\t%s", chartFileError.Error())
	}
	return nil
}

func validateChartNamePresence(cf *chart.Metadata) error {
	if cf.Name == "" {
		return errors.New("name is required")
	}
	return nil
}

func validateChartNameFormat(cf *chart.Metadata) error {
	if strings.Contains(cf.Name, ".") {
		return errors.New("name should be lower case letters and numbers. Words may be separated with dashes")
	}
	return nil
}

func validateChartNameDirMatch(chartDir string, cf *chart.Metadata) error {
	if cf.Name != filepath.Base(chartDir) {
		return fmt.Errorf("directory name (%s) and chart name (%s) must be the same", filepath.Base(chartDir), cf.Name)
	}
	return nil
}

func validateChartVersion(cf *chart.Metadata) error {
	if cf.Version == "" {
		return errors.New("version is required")
	}

	version, err := semver.NewVersion(cf.Version)

	if err != nil {
		return fmt.Errorf("version '%s' is not a valid SemVer", cf.Version)
	}

	c, err := semver.NewConstraint("> 0")
	if err != nil {
		return err
	}
	valid, msg := c.Validate(version)

	if !valid && len(msg) > 0 {
		return fmt.Errorf("version %v", msg[0])
	}

	return nil
}

func validateChartEngine(cf *chart.Metadata) error {
	if cf.Engine == "" {
		return nil
	}

	keys := make([]string, 0, len(chart.Metadata_Engine_value))
	for engine := range chart.Metadata_Engine_value {
		str := strings.ToLower(engine)

		if str == "unknown" {
			continue
		}

		if str == cf.Engine {
			return nil
		}

		keys = append(keys, str)
	}

	return fmt.Errorf("engine '%v' not valid. Valid options are %v", cf.Engine, keys)
}

func validateChartMaintainer(cf *chart.Metadata) error {
	for _, maintainer := range cf.Maintainers {
		if maintainer.Name == "" {
			return errors.New("each maintainer requires a name")
		} else if maintainer.Email != "" && !govalidator.IsEmail(maintainer.Email) {
			return fmt.Errorf("invalid email '%s' for maintainer '%s'", maintainer.Email, maintainer.Name)
		} else if maintainer.Url != "" && !govalidator.IsURL(maintainer.Url) {
			return fmt.Errorf("invalid url '%s' for maintainer '%s'", maintainer.Url, maintainer.Name)
		}
	}
	return nil
}

func validateChartSources(cf *chart.Metadata) error {
	for _, source := range cf.Sources {
		if source == "" || !govalidator.IsRequestURL(source) {
			return fmt.Errorf("invalid source URL '%s'", source)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdutil

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdhttp"
	"github.com/coreos/etcd/pkg/types"
)

const (
	tickDuration = 10 * time.Millisecond
	clusterName  = "etcd"
)

func newLocalListener(t *testing.T) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	return l
}

type member struct {
	etcdserver.ServerConfig
	PeerListeners, ClientListeners []net.Listener

	s   *etcdserver.EtcdServer
	hss []*httptest.Server
}

func StartNewEtcdServer(t *testing.T, name string) *member {
	m := MustNewMember(t, name)
	m.Launch()
	return m
}

func (m *member) URL() string {
	return fmt.Sprintf("http://%s", m.ClientListeners[0].Addr().String())
}

func MustNewMember(t *testing.T, name string) *member {
	var err error
	m := &member{}

	pln := newLocalListener(t)
	m.PeerListeners = []net.Listener{pln}
	m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	cln := newLocalListener(t)
	m.ClientListeners = []net.Listener{cln}
	m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	m.Name = name

	m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd")
	if err != nil {
		t.Fatal(err)
	}
	clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
	m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
	if err != nil {
		t.Fatal(err)
	}
	m.NewCluster = true
	m.Transport = newTransport()
	return m
}

// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch() error {
	var err error
	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
		return fmt.Errorf("failed to initialize the etcd server: %v", err)
	}
	m.s.Ticker = time.Tick(tickDuration)
	m.s.SyncTicker = time.Tick(500 * time.Millisecond)
	m.s.Start()

	for _, ln := range m.PeerListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: etcdhttp.NewPeerHandler(m.s)},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	for _, ln := range m.ClientListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: etcdhttp.NewClientHandler(m.s)},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	return nil
}

// Terminate stops the member and removes the data dir.
func (m *member) Terminate(t *testing.T) {
	m.s.Stop()
	for _, hs := range m.hss {
		hs.CloseClientConnections()
		hs.Close()
	}
	if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
		t.Fatal(err)
	}
}

func newTransport() *http.Transport {
	tr := &http.Transport{}
	// TODO: need the support of graceful stop in Sender to remove this
	tr.DisableKeepAlives = true
	tr.Dial = (&net.Dialer{Timeout: 100 * time.Millisecond}).Dial
	return tr
}
<commit_msg>update etcd cluster test helper functions<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdutil

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdhttp"
	"github.com/coreos/etcd/pkg/testutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/rafthttp"
)

const (
	tickDuration   = 10 * time.Millisecond
	clusterName    = "etcd"
	requestTimeout = 2 * time.Second
)

var (
	electionTicks = 10
)

func newLocalListener(t *testing.T) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	return l
}

func StartNewEtcdServer(t *testing.T, name string) *member {
	m := MustNewMember(t, name)
	m.Launch()
	return m
}

type member struct {
	etcdserver.ServerConfig
	PeerListeners, ClientListeners []net.Listener

	raftHandler *testutil.PauseableHandler
	s           *etcdserver.EtcdServer
	hss         []*httptest.Server
}

func MustNewMember(t *testing.T, name string) *member {
	var err error
	m := &member{}

	pln := newLocalListener(t)
	m.PeerListeners = []net.Listener{pln}
	m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	cln := newLocalListener(t)
	m.ClientListeners = []net.Listener{cln}
	m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	m.Name = name

	m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd")
	if err != nil {
		t.Fatal(err)
	}
	clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
	m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
	if err != nil {
		t.Fatal(err)
	}
	m.NewCluster = true
	m.Transport = mustNewTransport(t)
	m.ElectionTimeoutTicks = electionTicks
	return m
}

// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch() error {
	var err error
	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
		return fmt.Errorf("failed to initialize the etcd server: %v", err)
	}
	m.s.Ticker = time.Tick(tickDuration)
	m.s.SyncTicker = time.Tick(500 * time.Millisecond)
	m.s.Start()

	m.raftHandler = &testutil.PauseableHandler{Next: etcdhttp.NewPeerHandler(m.s.Cluster, m.s.RaftHandler())}

	for _, ln := range m.PeerListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: m.raftHandler},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	for _, ln := range m.ClientListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: etcdhttp.NewClientHandler(m.s)},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	return nil
}

func (m *member) URL() string { return m.ClientURLs[0].String() }

// Terminate stops the member and removes the data dir.
func (m *member) Terminate(t *testing.T) {
	m.s.Stop()
	for _, hs := range m.hss {
		hs.CloseClientConnections()
		hs.Close()
	}
	if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
		t.Fatal(err)
	}
}
func mustNewTransport(t *testing.T) *http.Transport {
	tr, err := transport.NewTimeoutTransport(transport.TLSInfo{}, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
	if err != nil {
		t.Fatal(err)
	}
	return tr
}
<|endoftext|>