<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"

	"gopkg.in/lucsky/go-exml.v3"
)

type E4fObject interface {
	Id()   int
	Type() string
}

type E4fDb struct {
	Version      string
	Cameras      []*Camera
	Makes        []*Make
	GpsLocations []*GpsLocation
	ExposedRolls []*ExposedRoll
	Exposures    []*Exposure
	Films        []*Film
	Lenses       []*Lens
	Artists      []*Artist

	GpsMap   map[int]*GpsLocation
	LensMap  map[int]*Lens
	ObjectsMap   map[int] E4fObject
}

// Build the id -> data maps for the various elements
func (db *E4fDb) buildMaps() {
	db.ObjectsMap = make(map[int] E4fObject)

	for _, cam := range db.Cameras {
		o, present := db.ObjectsMap[cam.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				cam.id, o.Type())
		}
		db.ObjectsMap[cam.id] = cam
	}

	for _, mk := range db.Makes {
		o, present := db.ObjectsMap[mk.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				mk.id, o.Type())
		}
		db.ObjectsMap[mk.id] = mk
	}

	db.GpsMap = make(map[int]*GpsLocation)
	for _, gps := range db.GpsLocations {
		o, present := db.GpsMap[gps.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				gps.id, o.Type())
		}
		db.GpsMap[gps.id] = gps
	}

	for _, roll := range db.ExposedRolls {
		o, present := db.ObjectsMap[roll.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				roll.id, o.Type())
		}
		db.ObjectsMap[roll.id] = roll
	}

	for _, exp := range db.Exposures {
		o, present := db.ObjectsMap[exp.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				exp.id, o.Type())
		}
		db.ObjectsMap[exp.id] = exp
	}

	for _, film := range db.Films {
		o, present := db.ObjectsMap[film.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				film.id, o.Type())
		}
		db.ObjectsMap[film.id] = film
	}

	db.LensMap = make(map[int]*Lens)
	for _, lens := range db.Lenses {
		o, present := db.LensMap[lens.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				lens.id, o.Type())
		}
		db.LensMap[lens.id] = lens
	}
}
func (db *E4fDb) exposuresForRoll(id int) (exposures []*Exposure) {
	obj := db.ObjectsMap[id]
	if obj.Type() != "ExposedRoll" {
		fmt.Printf("Found type %s\n", obj.Type())
		return nil
	}

	for _, exp := range db.Exposures {
		if exp.RollId == id {
			exposures = append(exposures, exp)
		}
	}
	return
}

type Camera struct {
	id                int
	DefaultFrameCount int
	MakeId            int
	SerialNumber      string
	DefaultFilmType   string
	Title             string
}

func (o *Camera) Id() int {
	return o.id
}
func (o *Camera) Type() string {
	return "Camera"
}

type Make struct {
	id   int
	Name string
}

func (o *Make) Id() int {
	return o.id
}
func (o *Make) Type() string {
	return "Make"
}

type GpsLocation struct {
	id int
	Long, Lat, Alt float64
}
func (o *GpsLocation) Id() int {
	return o.id
}
func (o *GpsLocation) Type() string {
	return "GpsLocation"
}

type ExposedRoll struct {
	id           int
	FilmType     string
	CameraId     int
	Iso          int
        FrameCount   int
	TimeUnloaded string
	TimeLoaded   string
	FilmId       int
}
func (o *ExposedRoll) Id() int {
	return o.id
}
func (o *ExposedRoll) Type() string {
	return "ExposedRoll"
}

type Exposure struct {
	id           int
	FlashOn      bool
	Desc         string
	Number       int
	GpsLocId     int
	ExpComp      int
	RollId       int
	FocalLength  int
	LightSource  string
	TimeTaken    string
	ShutterSpeed string
	LensId       int
	Aperture     string
	MeteringMode string
}
func (o *Exposure) Id() int {
	return o.id
}
func (o *Exposure) Type() string {
	return "Exposure"
}

type Film struct {
	id             int
	Process        string
	Title          string
	ColorType      string
	Iso            int
	MakeId         int
}
func (o *Film) Id() int {
	return o.id
}
func (o *Film) Type() string {
	return "Film"
}

type Lens struct {
	id             int
	Title          string
	SerialNumber   string
	MakeId         int
	ApertureMin    string
	ApertureMax    string
	FocalLengthMin int
	FocalLengthMax int
}
func (o *Lens) Id() int {
	return o.id
}
func (o *Lens) Type() string {
	return "Lens"
}

type Artist struct {
	Name string
}

func toInt(dst *int) exml.TextCallback {
	return func(c exml.CharData) {
		n, err := strconv.ParseInt(string(c), 0, 32)
		if err == nil {
			*dst = int(n)
		}
	}
}

func toFloat(dst *float64) exml.TextCallback {
	return func(c exml.CharData) {
		f, err := strconv.ParseFloat(string(c), 64)
		if err == nil {
			*dst = float64(f)
		}
	}
}

func toBool(dst *bool) exml.TextCallback {
	return func(c exml.CharData) {
		if string(c) == "true" {
			*dst = true
		} else {
			*dst = false
		}
	}
}

func parse(file string) (*E4fDb) {

	reader, _ := os.Open(file)
	defer reader.Close()

	e4fDb := &E4fDb{}
	decoder := exml.NewDecoder(reader)

	decoder.On("Exif4Film", func(attrs exml.Attrs) {

		e4fDb.Version, _= attrs.Get("version")

		decoder.On("Camera/dk.codeunited.exif4film.model.Camera",
			func(attrs exml.Attrs) {
				camera := &Camera{}
				e4fDb.Cameras = append(e4fDb.Cameras, camera)
				decoder.OnTextOf("camera_default_frame_count",
					toInt(&camera.DefaultFrameCount))
				decoder.OnTextOf("id", toInt(&camera.id))
				decoder.OnTextOf("camera_make_id",
					toInt(&camera.MakeId))
				decoder.OnTextOf("camera_serial_number",
					exml.Assign(&camera.SerialNumber))
				decoder.OnTextOf("camera_default_film_type",
					exml.Assign(&camera.DefaultFilmType))
				decoder.OnTextOf("camera_title",
					exml.Assign(&camera.Title))
			})
		decoder.On("Make/dk.codeunited.exif4film.model.Make",
			func(attrs exml.Attrs) {
				m := &Make{}
				e4fDb.Makes = append(e4fDb.Makes, m)
				decoder.OnTextOf("id", toInt(&m.id))
				decoder.OnTextOf("make_name",
					exml.Assign(&m.Name))
			})
		decoder.On(
			"GpsLocation/dk.codeunited.exif4film.model.GpsLocation",
			func(attrs exml.Attrs) {
				gps := &GpsLocation{}
				e4fDb.GpsLocations = append(e4fDb.GpsLocations,
					gps)
				decoder.OnTextOf("id", toInt(&gps.id))
				decoder.OnTextOf("gps_latitude",
					toFloat(&gps.Lat))
				decoder.OnTextOf("gps_longitude",
					toFloat(&gps.Long))
				decoder.OnTextOf("gps_altitude",
					toFloat(&gps.Alt))
			})
		decoder.On(
			"ExposedRoll/dk.codeunited.exif4film.model.ExposedRoll",
			func(attrs exml.Attrs) {
				roll := &ExposedRoll{}
				e4fDb.ExposedRolls = append(e4fDb.ExposedRolls,
					roll)
				decoder.OnTextOf("id", toInt(&roll.id))
				decoder.OnTextOf("exposedroll_film_type",
					exml.Assign(&roll.FilmType))
				decoder.OnTextOf("exposedroll_camera_id",
					toInt(&roll.CameraId))
				decoder.OnTextOf("exposedroll_film_id",
					toInt(&roll.FilmId))
				decoder.OnTextOf("exposedroll_iso",
					toInt(&roll.Iso))
				decoder.OnTextOf("exposedroll_frame_count",
					toInt(&roll.FrameCount))
				decoder.OnTextOf("exposedroll_time_unloaded",
					exml.Assign(&roll.TimeUnloaded))
				decoder.OnTextOf("exposedroll_time_loaded",
					exml.Assign(&roll.TimeLoaded))
			})
		decoder.On("Exposure/dk.codeunited.exif4film.model.Exposure",
			func(attrs exml.Attrs) {
				exp := &Exposure{}
				e4fDb.Exposures = append(e4fDb.Exposures, exp)
				decoder.OnTextOf("id", toInt(&exp.id))
				decoder.OnTextOf("exposure_flash_on",
					toBool(&exp.FlashOn))
				decoder.OnTextOf("exposure_description",
					exml.Assign(&exp.Desc))
				decoder.OnTextOf("exposure_number",
					toInt(&exp.Number))
				decoder.OnTextOf("exposure_gps_location",
					toInt(&exp.GpsLocId))
				decoder.OnTextOf("exposure_compensation",
					toInt(&exp.ExpComp))
				decoder.OnTextOf("exposure_roll_id",
					toInt(&exp.RollId))
				decoder.OnTextOf("exposure_focal_length",
					toInt(&exp.FocalLength))
				decoder.OnTextOf("exposure_light_source",
					exml.Assign(&exp.LightSource))
				decoder.OnTextOf("exposure_time_taken",
					exml.Assign(&exp.TimeTaken))
				decoder.OnTextOf("exposure_shutter_speed",
					exml.Assign(&exp.ShutterSpeed))
				decoder.OnTextOf("exposure_lens_id",
					toInt(&exp.LensId))
				decoder.OnTextOf("exposure_aperture",
					exml.Assign(&exp.Aperture))
				decoder.OnTextOf("exposure_metering_mode",
					exml.Assign(&exp.MeteringMode))
			})
		decoder.On("Film/dk.codeunited.exif4film.model.Film",
			func(attrs exml.Attrs) {
				film := &Film{}
				e4fDb.Films = append(e4fDb.Films, film)
				decoder.OnTextOf("id", toInt(&film.id))
				decoder.OnTextOf("film_title",
					exml.Assign(&film.Title))
				decoder.OnTextOf("film_make_process",
					exml.Assign(&film.Process))
				decoder.OnTextOf("film_color_type",
					exml.Assign(&film.ColorType))
				decoder.OnTextOf("film_iso", toInt(&film.Iso))
				decoder.OnTextOf("film_make_id",
					toInt(&film.MakeId))
			})
		decoder.On("Lens/dk.codeunited.exif4film.model.Lens",
			func(attrs exml.Attrs) {
				lens := &Lens{}
				e4fDb.Lenses = append(e4fDb.Lenses, lens)
				decoder.OnTextOf("id", toInt(&lens.id))
				decoder.OnTextOf("lens_title",
					exml.Assign(&lens.Title))
				decoder.OnTextOf("lens_serial_number",
					exml.Assign(&lens.SerialNumber))
				decoder.OnTextOf("lens_make_id",
					toInt(&lens.MakeId))
				decoder.OnTextOf("lens_aperture_min",
					exml.Assign(&lens.ApertureMin))
				decoder.OnTextOf("lens_aperture_max",
					exml.Assign(&lens.ApertureMax))
				decoder.OnTextOf("lens_focal_length_min",
					toInt(&lens.FocalLengthMin));
				decoder.OnTextOf("lens_focal_length_max",
					toInt(&lens.FocalLengthMax));
			})
		decoder.On("Artist/dk.codeunited.exif4film.model.Artist",
			func(attrs exml.Attrs) {
				artist := &Artist{}
				e4fDb.Artists = append(e4fDb.Artists, artist)
				decoder.OnTextOf("artist_name",
					exml.Assign(&artist.Name))
			})
	})
	decoder.Run()

	return e4fDb
}

func main() {
	e4fDb := parse("samples/export-Roll-20130630_203650.xml");

	e4fDb.buildMaps()

	for _, roll := range e4fDb.ExposedRolls {
		id := roll.Id()
		fmt.Println("Roll:")
		fmt.Println(roll)
		exps := e4fDb.exposuresForRoll(id)
		for _, exp := range exps {
			fmt.Printf("Exposure %d: ", exp.Number)
			fmt.Println(exp)
		}
	}
}
<commit_msg>Ran gofmt<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"

	"gopkg.in/lucsky/go-exml.v3"
)

type E4fObject interface {
	Id() int
	Type() string
}

type E4fDb struct {
	Version      string
	Cameras      []*Camera
	Makes        []*Make
	GpsLocations []*GpsLocation
	ExposedRolls []*ExposedRoll
	Exposures    []*Exposure
	Films        []*Film
	Lenses       []*Lens
	Artists      []*Artist

	GpsMap     map[int]*GpsLocation
	LensMap    map[int]*Lens
	ObjectsMap map[int]E4fObject
}

// Build the id -> data maps for the various elements
func (db *E4fDb) buildMaps() {
	db.ObjectsMap = make(map[int]E4fObject)

	for _, cam := range db.Cameras {
		o, present := db.ObjectsMap[cam.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				cam.id, o.Type())
		}
		db.ObjectsMap[cam.id] = cam
	}

	for _, mk := range db.Makes {
		o, present := db.ObjectsMap[mk.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				mk.id, o.Type())
		}
		db.ObjectsMap[mk.id] = mk
	}

	db.GpsMap = make(map[int]*GpsLocation)
	for _, gps := range db.GpsLocations {
		o, present := db.GpsMap[gps.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				gps.id, o.Type())
		}
		db.GpsMap[gps.id] = gps
	}

	for _, roll := range db.ExposedRolls {
		o, present := db.ObjectsMap[roll.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				roll.id, o.Type())
		}
		db.ObjectsMap[roll.id] = roll
	}

	for _, exp := range db.Exposures {
		o, present := db.ObjectsMap[exp.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				exp.id, o.Type())
		}
		db.ObjectsMap[exp.id] = exp
	}

	for _, film := range db.Films {
		o, present := db.ObjectsMap[film.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				film.id, o.Type())
		}
		db.ObjectsMap[film.id] = film
	}

	db.LensMap = make(map[int]*Lens)
	for _, lens := range db.Lenses {
		o, present := db.LensMap[lens.id]
		if present {
			fmt.Printf("Object %d present already of type %s\n",
				lens.id, o.Type())
		}
		db.LensMap[lens.id] = lens
	}
}
func (db *E4fDb) exposuresForRoll(id int) (exposures []*Exposure) {
	obj := db.ObjectsMap[id]
	if obj.Type() != "ExposedRoll" {
		fmt.Printf("Found type %s\n", obj.Type())
		return nil
	}

	for _, exp := range db.Exposures {
		if exp.RollId == id {
			exposures = append(exposures, exp)
		}
	}
	return
}

type Camera struct {
	id                int
	DefaultFrameCount int
	MakeId            int
	SerialNumber      string
	DefaultFilmType   string
	Title             string
}

func (o *Camera) Id() int {
	return o.id
}
func (o *Camera) Type() string {
	return "Camera"
}

type Make struct {
	id   int
	Name string
}

func (o *Make) Id() int {
	return o.id
}
func (o *Make) Type() string {
	return "Make"
}

type GpsLocation struct {
	id             int
	Long, Lat, Alt float64
}

func (o *GpsLocation) Id() int {
	return o.id
}
func (o *GpsLocation) Type() string {
	return "GpsLocation"
}

type ExposedRoll struct {
	id           int
	FilmType     string
	CameraId     int
	Iso          int
	FrameCount   int
	TimeUnloaded string
	TimeLoaded   string
	FilmId       int
}

func (o *ExposedRoll) Id() int {
	return o.id
}
func (o *ExposedRoll) Type() string {
	return "ExposedRoll"
}

type Exposure struct {
	id           int
	FlashOn      bool
	Desc         string
	Number       int
	GpsLocId     int
	ExpComp      int
	RollId       int
	FocalLength  int
	LightSource  string
	TimeTaken    string
	ShutterSpeed string
	LensId       int
	Aperture     string
	MeteringMode string
}

func (o *Exposure) Id() int {
	return o.id
}
func (o *Exposure) Type() string {
	return "Exposure"
}

type Film struct {
	id        int
	Process   string
	Title     string
	ColorType string
	Iso       int
	MakeId    int
}

func (o *Film) Id() int {
	return o.id
}
func (o *Film) Type() string {
	return "Film"
}

type Lens struct {
	id             int
	Title          string
	SerialNumber   string
	MakeId         int
	ApertureMin    string
	ApertureMax    string
	FocalLengthMin int
	FocalLengthMax int
}

func (o *Lens) Id() int {
	return o.id
}
func (o *Lens) Type() string {
	return "Lens"
}

type Artist struct {
	Name string
}

func toInt(dst *int) exml.TextCallback {
	return func(c exml.CharData) {
		n, err := strconv.ParseInt(string(c), 0, 32)
		if err == nil {
			*dst = int(n)
		}
	}
}

func toFloat(dst *float64) exml.TextCallback {
	return func(c exml.CharData) {
		f, err := strconv.ParseFloat(string(c), 64)
		if err == nil {
			*dst = float64(f)
		}
	}
}

func toBool(dst *bool) exml.TextCallback {
	return func(c exml.CharData) {
		if string(c) == "true" {
			*dst = true
		} else {
			*dst = false
		}
	}
}

func parse(file string) *E4fDb {

	reader, _ := os.Open(file)
	defer reader.Close()

	e4fDb := &E4fDb{}
	decoder := exml.NewDecoder(reader)

	decoder.On("Exif4Film", func(attrs exml.Attrs) {

		e4fDb.Version, _ = attrs.Get("version")

		decoder.On("Camera/dk.codeunited.exif4film.model.Camera",
			func(attrs exml.Attrs) {
				camera := &Camera{}
				e4fDb.Cameras = append(e4fDb.Cameras, camera)
				decoder.OnTextOf("camera_default_frame_count",
					toInt(&camera.DefaultFrameCount))
				decoder.OnTextOf("id", toInt(&camera.id))
				decoder.OnTextOf("camera_make_id",
					toInt(&camera.MakeId))
				decoder.OnTextOf("camera_serial_number",
					exml.Assign(&camera.SerialNumber))
				decoder.OnTextOf("camera_default_film_type",
					exml.Assign(&camera.DefaultFilmType))
				decoder.OnTextOf("camera_title",
					exml.Assign(&camera.Title))
			})
		decoder.On("Make/dk.codeunited.exif4film.model.Make",
			func(attrs exml.Attrs) {
				m := &Make{}
				e4fDb.Makes = append(e4fDb.Makes, m)
				decoder.OnTextOf("id", toInt(&m.id))
				decoder.OnTextOf("make_name",
					exml.Assign(&m.Name))
			})
		decoder.On(
			"GpsLocation/dk.codeunited.exif4film.model.GpsLocation",
			func(attrs exml.Attrs) {
				gps := &GpsLocation{}
				e4fDb.GpsLocations = append(e4fDb.GpsLocations,
					gps)
				decoder.OnTextOf("id", toInt(&gps.id))
				decoder.OnTextOf("gps_latitude",
					toFloat(&gps.Lat))
				decoder.OnTextOf("gps_longitude",
					toFloat(&gps.Long))
				decoder.OnTextOf("gps_altitude",
					toFloat(&gps.Alt))
			})
		decoder.On(
			"ExposedRoll/dk.codeunited.exif4film.model.ExposedRoll",
			func(attrs exml.Attrs) {
				roll := &ExposedRoll{}
				e4fDb.ExposedRolls = append(e4fDb.ExposedRolls,
					roll)
				decoder.OnTextOf("id", toInt(&roll.id))
				decoder.OnTextOf("exposedroll_film_type",
					exml.Assign(&roll.FilmType))
				decoder.OnTextOf("exposedroll_camera_id",
					toInt(&roll.CameraId))
				decoder.OnTextOf("exposedroll_film_id",
					toInt(&roll.FilmId))
				decoder.OnTextOf("exposedroll_iso",
					toInt(&roll.Iso))
				decoder.OnTextOf("exposedroll_frame_count",
					toInt(&roll.FrameCount))
				decoder.OnTextOf("exposedroll_time_unloaded",
					exml.Assign(&roll.TimeUnloaded))
				decoder.OnTextOf("exposedroll_time_loaded",
					exml.Assign(&roll.TimeLoaded))
			})
		decoder.On("Exposure/dk.codeunited.exif4film.model.Exposure",
			func(attrs exml.Attrs) {
				exp := &Exposure{}
				e4fDb.Exposures = append(e4fDb.Exposures, exp)
				decoder.OnTextOf("id", toInt(&exp.id))
				decoder.OnTextOf("exposure_flash_on",
					toBool(&exp.FlashOn))
				decoder.OnTextOf("exposure_description",
					exml.Assign(&exp.Desc))
				decoder.OnTextOf("exposure_number",
					toInt(&exp.Number))
				decoder.OnTextOf("exposure_gps_location",
					toInt(&exp.GpsLocId))
				decoder.OnTextOf("exposure_compensation",
					toInt(&exp.ExpComp))
				decoder.OnTextOf("exposure_roll_id",
					toInt(&exp.RollId))
				decoder.OnTextOf("exposure_focal_length",
					toInt(&exp.FocalLength))
				decoder.OnTextOf("exposure_light_source",
					exml.Assign(&exp.LightSource))
				decoder.OnTextOf("exposure_time_taken",
					exml.Assign(&exp.TimeTaken))
				decoder.OnTextOf("exposure_shutter_speed",
					exml.Assign(&exp.ShutterSpeed))
				decoder.OnTextOf("exposure_lens_id",
					toInt(&exp.LensId))
				decoder.OnTextOf("exposure_aperture",
					exml.Assign(&exp.Aperture))
				decoder.OnTextOf("exposure_metering_mode",
					exml.Assign(&exp.MeteringMode))
			})
		decoder.On("Film/dk.codeunited.exif4film.model.Film",
			func(attrs exml.Attrs) {
				film := &Film{}
				e4fDb.Films = append(e4fDb.Films, film)
				decoder.OnTextOf("id", toInt(&film.id))
				decoder.OnTextOf("film_title",
					exml.Assign(&film.Title))
				decoder.OnTextOf("film_make_process",
					exml.Assign(&film.Process))
				decoder.OnTextOf("film_color_type",
					exml.Assign(&film.ColorType))
				decoder.OnTextOf("film_iso", toInt(&film.Iso))
				decoder.OnTextOf("film_make_id",
					toInt(&film.MakeId))
			})
		decoder.On("Lens/dk.codeunited.exif4film.model.Lens",
			func(attrs exml.Attrs) {
				lens := &Lens{}
				e4fDb.Lenses = append(e4fDb.Lenses, lens)
				decoder.OnTextOf("id", toInt(&lens.id))
				decoder.OnTextOf("lens_title",
					exml.Assign(&lens.Title))
				decoder.OnTextOf("lens_serial_number",
					exml.Assign(&lens.SerialNumber))
				decoder.OnTextOf("lens_make_id",
					toInt(&lens.MakeId))
				decoder.OnTextOf("lens_aperture_min",
					exml.Assign(&lens.ApertureMin))
				decoder.OnTextOf("lens_aperture_max",
					exml.Assign(&lens.ApertureMax))
				decoder.OnTextOf("lens_focal_length_min",
					toInt(&lens.FocalLengthMin))
				decoder.OnTextOf("lens_focal_length_max",
					toInt(&lens.FocalLengthMax))
			})
		decoder.On("Artist/dk.codeunited.exif4film.model.Artist",
			func(attrs exml.Attrs) {
				artist := &Artist{}
				e4fDb.Artists = append(e4fDb.Artists, artist)
				decoder.OnTextOf("artist_name",
					exml.Assign(&artist.Name))
			})
	})
	decoder.Run()

	return e4fDb
}

func main() {
	e4fDb := parse("samples/export-Roll-20130630_203650.xml")

	e4fDb.buildMaps()

	for _, roll := range e4fDb.ExposedRolls {
		id := roll.Id()
		fmt.Println("Roll:")
		fmt.Println(roll)
		exps := e4fDb.exposuresForRoll(id)
		for _, exp := range exps {
			fmt.Printf("Exposure %d: ", exp.Number)
			fmt.Println(exp)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"os"
	"sort"
	"strings"
	"text/template"

	"github.com/miquella/vaulted/lib"
)

type Env struct {
	VaultName string
	Role      string

	DetectedShell string
	Format        string
	Command       string
}

type templateVals struct {
	vaulted.Variables
	Command string
}

var (
	sessionFormatters = map[string]string{
		"fish": `# To load these variables into your shell, execute:
#   eval ({{ .Command }})
{{ range $var := .Unset}}set -e {{ $var }};
{{ end -}}
{{ range $var, $value := .Set }}set -x {{ $var }} "{{ replace $value "\"" "\\\"" }}";
{{ end }}`,
		"sh": `# To load these variables into your shell, execute:
#   eval $({{ .Command }})
{{ range $var := .Unset}}unset {{ $var }}
{{ end -}}
{{ range $var, $value := .Set }}export {{ $var }}="{{ replace $value "\"" "\\\"" }}"
{{ end }}`,
		"json": "{{ json .Set }}\n",
	}
)

var templateFuncMap = template.FuncMap{
	"replace": func(val string, toReplace string, replacement string) string {
		return strings.Replace(val, toReplace, replacement, -1)
	},
	"json": func(val interface{}) (string, error) {
		json, err := json.MarshalIndent(val, "", "  ")
		if err != nil {
			return "", err
		} else {
			return string(json), nil
		}
	},
}

func (e *Env) Run(steward Steward) error {
	session, err := e.getSession(steward)
	if err != nil {
		return err
	}

	var templateStr string
	format := e.Format

	if format == "shell" {
		format = e.DetectedShell
	}

	if foundTemplate, ok := sessionFormatters[format]; ok {
		templateStr = foundTemplate
	} else {
		templateStr = format
	}
	tmpl, err := template.New("sessionTmpl").Funcs(templateFuncMap).Parse(templateStr)

	vals := templateVals{}
	variables := session.Variables()

	vals.Set = variables.Set

	sort.Strings(variables.Unset)
	vals.Unset = variables.Unset

	vals.Command = e.Command

	if err != nil {
		return ErrorWithExitCode{err, 64}
	}
	return tmpl.Execute(os.Stdout, vals)
}

func (e *Env) getSession(steward Steward) (*vaulted.Session, error) {
	var err error

	// default session
	session := DefaultSession()

	if e.VaultName != "" {
		// get specific session
		_, session, err = steward.GetSession(e.VaultName, nil)
		if err != nil {
			return nil, err
		}
	}

	if e.Role != "" {
		return session.Assume(e.Role)
	}

	return session, nil
}
<commit_msg>Default shell to 'sh' for env subcommand<commit_after>package main

import (
	"encoding/json"
	"os"
	"sort"
	"strings"
	"text/template"

	"github.com/miquella/vaulted/lib"
)

type Env struct {
	VaultName string
	Role      string

	DetectedShell string
	Format        string
	Command       string
}

type templateVals struct {
	vaulted.Variables
	Command string
}

var (
	sessionFormatters = map[string]string{
		"fish": `# To load these variables into your shell, execute:
#   eval ({{ .Command }})
{{ range $var := .Unset}}set -e {{ $var }};
{{ end -}}
{{ range $var, $value := .Set }}set -x {{ $var }} "{{ replace $value "\"" "\\\"" }}";
{{ end }}`,
		"sh": `# To load these variables into your shell, execute:
#   eval $({{ .Command }})
{{ range $var := .Unset}}unset {{ $var }}
{{ end -}}
{{ range $var, $value := .Set }}export {{ $var }}="{{ replace $value "\"" "\\\"" }}"
{{ end }}`,
		"json": "{{ json .Set }}\n",
	}
)

var templateFuncMap = template.FuncMap{
	"replace": func(val string, toReplace string, replacement string) string {
		return strings.Replace(val, toReplace, replacement, -1)
	},
	"json": func(val interface{}) (string, error) {
		json, err := json.MarshalIndent(val, "", "  ")
		if err != nil {
			return "", err
		} else {
			return string(json), nil
		}
	},
}

func (e *Env) Run(steward Steward) error {
	session, err := e.getSession(steward)
	if err != nil {
		return err
	}

	var templateStr string
	format := e.Format

	if format == "shell" {
		format = e.DetectedShell
		if _, ok := sessionFormatters[format]; !ok {
			format = "sh"
		}
	}

	if foundTemplate, ok := sessionFormatters[format]; ok {
		templateStr = foundTemplate
	} else {
		templateStr = format
	}
	tmpl, err := template.New("sessionTmpl").Funcs(templateFuncMap).Parse(templateStr)

	vals := templateVals{}
	variables := session.Variables()

	vals.Set = variables.Set

	sort.Strings(variables.Unset)
	vals.Unset = variables.Unset

	vals.Command = e.Command

	if err != nil {
		return ErrorWithExitCode{err, 64}
	}
	return tmpl.Execute(os.Stdout, vals)
}

func (e *Env) getSession(steward Steward) (*vaulted.Session, error) {
	var err error

	// default session
	session := DefaultSession()

	if e.VaultName != "" {
		// get specific session
		_, session, err = steward.GetSession(e.VaultName, nil)
		if err != nil {
			return nil, err
		}
	}

	if e.Role != "" {
		return session.Assume(e.Role)
	}

	return session, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/yi-jiayu/datamall"
	"golang.org/x/net/context"
)

var collapseWhitespaceRegex = regexp.MustCompile(`\s+`)
var illegalCharsRegex = regexp.MustCompile(`[^A-Z0-9 ]`)

var (
	errBusStopIDTooLong = errors.New("bus stop id too long")
	errBusStopIDInvalid = errors.New("bus stop id invalid")
)

// BusEtas represents the calculated time before buses arrive at a bus stop
type BusEtas struct {
	BusStopID   string
	UpdatedTime time.Time
	Services    [][4]string
}

// EtaCallbackData represents the data to be included with the Refresh inline keyboard button in eta messages.
type EtaCallbackData struct {
	Type       string   `json:"t"`
	BusStopID  string   `json:"b"`
	ServiceNos []string `json:"s,omitempty"`
	Argstr     string   `json:"a,omitempty"`
}

// InferEtaQuery extracts a bus stop ID and service numbers from a text message.
func InferEtaQuery(text string) (string, []string, error) {
	if len(text) > 30 {
		text = text[:30]
	}

	text = strings.ToUpper(text)
	tokens := collapseWhitespaceRegex.Split(text, -1)
	busStopID, serviceNos := tokens[0], tokens[1:]

	if len(busStopID) > 5 {
		return busStopID, serviceNos, errBusStopIDTooLong
	}

	if illegalCharsRegex.MatchString(busStopID) {
		return busStopID, serviceNos, errBusStopIDInvalid
	}

	return busStopID, serviceNos, nil
}

// CalculateEtas calculates the time before buses arrive from the LTA DataMall bus arrival response
func CalculateEtas(t time.Time, busArrival datamall.BusArrival) (BusEtas, error) {
	services := make([][4]string, 0)
	for _, service := range busArrival.Services {
		etas := [4]string{service.ServiceNo}

		var placeholder string
		if service.Status == "Not In Operation" {
			placeholder = "-"
		} else {
			placeholder = "?"
		}

		if next := service.NextBus.EstimatedArrival; next != "" {
			eta, err := time.Parse(time.RFC3339, next)
			if err != nil {
				return BusEtas{}, err
			}

			diff := (eta.Unix() - t.Unix()) / 60
			etas[1] = fmt.Sprintf("%d", diff)
		} else {
			etas[1] = placeholder
		}

		if next2 := service.SubsequentBus.EstimatedArrival; next2 != "" {
			eta, err := time.Parse(time.RFC3339, next2)
			if err != nil {
				return BusEtas{}, err
			}

			diff := (eta.Unix() - t.Unix()) / 60
			etas[2] = fmt.Sprintf("%d", diff)
		} else {
			etas[2] = placeholder
		}

		if next3 := service.SubsequentBus3.EstimatedArrival; next3 != "" {
			eta, err := time.Parse(time.RFC3339, next3)
			if err != nil {
				return BusEtas{}, err
			}

			diff := (eta.Unix() - t.Unix()) / 60
			etas[3] = fmt.Sprintf("%d", diff)
		} else {
			etas[3] = placeholder
		}

		services = append(services, etas)
	}

	return BusEtas{
		BusStopID:   busArrival.BusStopID,
		UpdatedTime: t,
		Services:    services,
	}, nil
}

func contains(serviceNos []string, serviceNo string) bool {
	for _, s := range serviceNos {
		if s == serviceNo {
			return true
		}
	}

	return false
}

type byServiceNo [][4]string

func (s byServiceNo) Len() int           { return len(s) }
func (s byServiceNo) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s byServiceNo) Less(i, j int) bool { return s[i][0] < s[j][0] }

// FormatEtas formats bus etas nicely to be sent in a message
func FormatEtas(busEtas BusEtas, busStop *BusStop, serviceNos []string) string {
	showing := 0
	services := make([][4]string, 0)
	for _, etas := range busEtas.Services {
		serviceNo := etas[0]
		if serviceNos != nil && len(serviceNos) != 0 && !contains(serviceNos, serviceNo) {
			continue
		}

		services = append(services, etas)
		showing++
	}

	sort.Sort(byServiceNo(services))

	table := EtaTable(services)

	var header string
	if busStop.Description != "" {
		header = fmt.Sprintf("*%s (%s)*\n", busStop.Description, busStop.BusStopID)
	} else {
		header = fmt.Sprintf("*%s*\n", busStop.BusStopID)
	}

	if busStop.Road != "" {
		header += fmt.Sprintf("%s\n", busStop.Road)
	}

	shown := fmt.Sprintf("Showing %d out of %d services for this bus stop.", showing, len(busEtas.Services))
	updated := fmt.Sprintf("Last updated at %s", busEtas.UpdatedTime.Format(time.RFC822))
	formatted := fmt.Sprintf("%s```\n%s```\n%s\n\n_%s_", header, table, shown, updated)

	return formatted
}

func padLeft(s string, l int, c string) string {
	for {
		if len(s) >= l {
			return s
		}

		s = c + s
	}
}

func padRight(s string, l int, c string) string {
	for {
		if len(s) >= l {
			return s
		}

		s += c
	}
}

// EtaTable generates the table of bus etas used in a eta message
func EtaTable(etas [][4]string) string {
	table := [][4]string{
		{"Svc", "Next", " 2nd", " 3rd"},
	}

	for _, eta := range etas {
		table = append(table, eta)
	}

	maxWidths := make([]int, len(table[0]))
	for _, row := range table {
		for pos, col := range row {
			if len(col) > maxWidths[pos] {
				maxWidths[pos] = len(col)
			}
		}
	}

	output := ""
	for i, row := range table {
		if i == 1 {
			for pos := range row {
				switch pos {
				case 0:
					output += "|-" + padRight("", maxWidths[pos], "-")
				case 3:
					output += "-|-" + padRight("", maxWidths[pos], "-") + "-|"
				default:
					output += "-|-" + padRight("", maxWidths[pos], "-")
				}
			}

			output += "\n"
		}

		for pos, col := range row {
			switch pos {
			case 0:
				output += "| " + padRight(col, maxWidths[pos], " ")
			case 3:
				output += " | " + padLeft(col, maxWidths[pos], " ") + " |"
			default:
				output += " | " + padLeft(col, maxWidths[pos], " ")
			}
		}

		output += "\n"
	}

	// remove final newline
	return output[:len(output)-1]
}

// EtaMessage generates and returns the text for an eta message
func EtaMessage(ctx context.Context, bot *BusEtaBot, busStopID string, serviceNos []string) (string, error) {
	busArrival, err := bot.Datamall.GetBusArrival(busStopID, nil)
	if err != nil {
		return "", err
	}

	etas, err := CalculateEtas(bot.NowFunc(), busArrival)
	if err != nil {
		return "", err
	}

	busStop, err := GetBusStop(ctx, busStopID)
	if err != nil {
		if err == errNotFound {
			if len(etas.Services) == 0 {
				return fmt.Sprintf("Oh no! I couldn't find any information about bus stop %s.", busStopID), err
			}

			busStop.BusStopID = busStopID
		} else {
			return "", err
		}
	}

	msg := FormatEtas(etas, &busStop, serviceNos)
	return msg, nil
}
<commit_msg>Add context to errors returned from datamall<commit_after>package main

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/yi-jiayu/datamall"
	"golang.org/x/net/context"
)

var collapseWhitespaceRegex = regexp.MustCompile(`\s+`)
var illegalCharsRegex = regexp.MustCompile(`[^A-Z0-9 ]`)

var (
	errBusStopIDTooLong = errors.New("bus stop id too long")
	errBusStopIDInvalid = errors.New("bus stop id invalid")
)

// BusEtas represents the calculated time before buses arrive at a bus stop
type BusEtas struct {
	BusStopID   string
	UpdatedTime time.Time
	Services    [][4]string
}

// EtaCallbackData represents the data to be included with the Refresh inline keyboard button in eta messages.
type EtaCallbackData struct {
	Type       string   `json:"t"`
	BusStopID  string   `json:"b"`
	ServiceNos []string `json:"s,omitempty"`
	Argstr     string   `json:"a,omitempty"`
}

// InferEtaQuery extracts a bus stop ID and service numbers from a text message.
func InferEtaQuery(text string) (string, []string, error) {
	if len(text) > 30 {
		text = text[:30]
	}

	text = strings.ToUpper(text)
	tokens := collapseWhitespaceRegex.Split(text, -1)
	busStopID, serviceNos := tokens[0], tokens[1:]

	if len(busStopID) > 5 {
		return busStopID, serviceNos, errBusStopIDTooLong
	}

	if illegalCharsRegex.MatchString(busStopID) {
		return busStopID, serviceNos, errBusStopIDInvalid
	}

	return busStopID, serviceNos, nil
}

// CalculateEtas calculates the time before buses arrive from the LTA DataMall bus arrival response
func CalculateEtas(t time.Time, busArrival datamall.BusArrival) (BusEtas, error) {
	services := make([][4]string, 0)
	for _, service := range busArrival.Services {
		etas := [4]string{service.ServiceNo}

		var placeholder string
		if service.Status == "Not In Operation" {
			placeholder = "-"
		} else {
			placeholder = "?"
		}

		if next := service.NextBus.EstimatedArrival; next != "" {
			eta, err := time.Parse(time.RFC3339, next)
			if err != nil {
				return BusEtas{}, err
			}

			diff := (eta.Unix() - t.Unix()) / 60
			etas[1] = fmt.Sprintf("%d", diff)
		} else {
			etas[1] = placeholder
		}

		if next2 := service.SubsequentBus.EstimatedArrival; next2 != "" {
			eta, err := time.Parse(time.RFC3339, next2)
			if err != nil {
				return BusEtas{}, err
			}

			diff := (eta.Unix() - t.Unix()) / 60
			etas[2] = fmt.Sprintf("%d", diff)
		} else {
			etas[2] = placeholder
		}

		if next3 := service.SubsequentBus3.EstimatedArrival; next3 != "" {
			eta, err := time.Parse(time.RFC3339, next3)
			if err != nil {
				return BusEtas{}, err
			}

			diff := (eta.Unix() - t.Unix()) / 60
			etas[3] = fmt.Sprintf("%d", diff)
		} else {
			etas[3] = placeholder
		}

		services = append(services, etas)
	}

	return BusEtas{
		BusStopID:   busArrival.BusStopID,
		UpdatedTime: t,
		Services:    services,
	}, nil
}

func contains(serviceNos []string, serviceNo string) bool {
	for _, s := range serviceNos {
		if s == serviceNo {
			return true
		}
	}

	return false
}

type byServiceNo [][4]string

func (s byServiceNo) Len() int           { return len(s) }
func (s byServiceNo) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s byServiceNo) Less(i, j int) bool { return s[i][0] < s[j][0] }

// FormatEtas formats bus etas nicely to be sent in a message
func FormatEtas(busEtas BusEtas, busStop *BusStop, serviceNos []string) string {
	showing := 0
	services := make([][4]string, 0)
	for _, etas := range busEtas.Services {
		serviceNo := etas[0]
		if serviceNos != nil && len(serviceNos) != 0 && !contains(serviceNos, serviceNo) {
			continue
		}

		services = append(services, etas)
		showing++
	}

	sort.Sort(byServiceNo(services))

	table := EtaTable(services)

	var header string
	if busStop.Description != "" {
		header = fmt.Sprintf("*%s (%s)*\n", busStop.Description, busStop.BusStopID)
	} else {
		header = fmt.Sprintf("*%s*\n", busStop.BusStopID)
	}

	if busStop.Road != "" {
		header += fmt.Sprintf("%s\n", busStop.Road)
	}

	shown := fmt.Sprintf("Showing %d out of %d services for this bus stop.", showing, len(busEtas.Services))
	updated := fmt.Sprintf("Last updated at %s", busEtas.UpdatedTime.Format(time.RFC822))
	formatted := fmt.Sprintf("%s```\n%s```\n%s\n\n_%s_", header, table, shown, updated)

	return formatted
}

func padLeft(s string, l int, c string) string {
	for {
		if len(s) >= l {
			return s
		}

		s = c + s
	}
}

func padRight(s string, l int, c string) string {
	for {
		if len(s) >= l {
			return s
		}

		s += c
	}
}

// EtaTable generates the table of bus etas used in a eta message
func EtaTable(etas [][4]string) string {
	table := [][4]string{
		{"Svc", "Next", " 2nd", " 3rd"},
	}

	for _, eta := range etas {
		table = append(table, eta)
	}

	maxWidths := make([]int, len(table[0]))
	for _, row := range table {
		for pos, col := range row {
			if len(col) > maxWidths[pos] {
				maxWidths[pos] = len(col)
			}
		}
	}

	output := ""
	for i, row := range table {
		if i == 1 {
			for pos := range row {
				switch pos {
				case 0:
					output += "|-" + padRight("", maxWidths[pos], "-")
				case 3:
					output += "-|-" + padRight("", maxWidths[pos], "-") + "-|"
				default:
					output += "-|-" + padRight("", maxWidths[pos], "-")
				}
			}

			output += "\n"
		}

		for pos, col := range row {
			switch pos {
			case 0:
				output += "| " + padRight(col, maxWidths[pos], " ")
			case 3:
				output += " | " + padLeft(col, maxWidths[pos], " ") + " |"
			default:
				output += " | " + padLeft(col, maxWidths[pos], " ")
			}
		}

		output += "\n"
	}

	// remove final newline
	return output[:len(output)-1]
}

// EtaMessage generates and returns the text for an eta message
func EtaMessage(ctx context.Context, bot *BusEtaBot, busStopID string, serviceNos []string) (string, error) {
	busArrival, err := bot.Datamall.GetBusArrival(busStopID, nil)
	if err != nil {
		return "", errors.Wrap(err, "error getting etas from datamall")
	}

	etas, err := CalculateEtas(bot.NowFunc(), busArrival)
	if err != nil {
		return "", err
	}

	busStop, err := GetBusStop(ctx, busStopID)
	if err != nil {
		if err == errNotFound {
			if len(etas.Services) == 0 {
				return fmt.Sprintf("Oh no! I couldn't find any information about bus stop %s.", busStopID), err
			}

			busStop.BusStopID = busStopID
		} else {
			return "", err
		}
	}

	msg := FormatEtas(etas, &busStop, serviceNos)
	return msg, nil
}
<|endoftext|>


<commit_before>package financial

import (
	"fmt"
	"math"
)

const irrMaxInterations = 20
const irrAccuracy = 1E-7
const irrInitialGuess = 0

// NPV returns the NPV (Net Present Value) of a cash flow series.
func NPV(rate float64, values []float64) (float64, error) {
	if len(values) == 0 {
		return -1, fmt.Errorf("values must include the initial investment (usually negative number) and period cash flows")
	}
	npv := values[0]
	values = values[1:]
	for i, v := range values {
		npv += (v / math.Pow(1+rate, float64(i+1)))
	}
	return npv, nil
}

// IRR return the Internal Rate of Return (IRR).
func IRR(values []float64) (float64, error) {
	if len(values) == 0 {
		return -1, fmt.Errorf("values must include the initial investment (usually negative number) and period cash flows")
	}
	x0 := float64(irrInitialGuess)
	var x1 float64
	for i := 0; i < irrMaxInterations; i++ {
		fValue := float64(0)
		fDerivative := float64(0)
		for k := 0; k < len(values); k++ {
			fk := float64(k)
			fValue += values[k] / math.Pow(1.0+x0, fk)
			fDerivative += -fk * values[k] / math.Pow(1.0+x0, fk+1.0)
		}
		x1 = x0 - fValue/fDerivative

		if math.Abs(x1-x0) <= irrAccuracy {
			return x1, nil
		}
		x0 = x1
	}
	return -1, fmt.Errorf("could not find irr for the provided values")
}
<commit_msg>fix err in NPV<commit_after>package financial

import (
	"errors"
	"fmt"
	"math"
)

const irrMaxInterations = 20
const irrAccuracy = 1E-7
const irrInitialGuess = 0

// NPV returns the NPV (Net Present Value) of a cash flow series.
func NPV(rate float64, values []float64) (float64, error) {
	if len(values) == 0 {
		return -1, errors.New("values must include the initial investment (usually negative number) and period cash flows")
	}
	npv := values[0]
	values = values[1:]
	for i, v := range values {
		npv += (v / math.Pow(1+rate, float64(i+1)))
	}
	return npv, nil
}

// IRR return the Internal Rate of Return (IRR).
func IRR(values []float64) (float64, error) {
	if len(values) == 0 {
		return -1, fmt.Errorf("values must include the initial investment (usually negative number) and period cash flows")
	}
	x0 := float64(irrInitialGuess)
	var x1 float64
	for i := 0; i < irrMaxInterations; i++ {
		fValue := float64(0)
		fDerivative := float64(0)
		for k := 0; k < len(values); k++ {
			fk := float64(k)
			fValue += values[k] / math.Pow(1.0+x0, fk)
			fDerivative += -fk * values[k] / math.Pow(1.0+x0, fk+1.0)
		}
		x1 = x0 - fValue/fDerivative

		if math.Abs(x1-x0) <= irrAccuracy {
			return x1, nil
		}
		x0 = x1
	}
	return -1, fmt.Errorf("could not find irr for the provided values")
}
<|endoftext|>


<commit_before>// (c) Copyright IBM Corp. 2021
// (c) Copyright Instana Inc. 2016

package instana

import (
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	f "github.com/looplab/fsm"
)

const (
	eInit     = "init"
	eLookup   = "lookup"
	eAnnounce = "announce"
	eTest     = "test"

	retryPeriod                = 30 * 1000 * time.Millisecond
	exponentialRetryPeriodBase = 10 * 1000 * time.Millisecond
	maximumRetries             = 3
)

type fsmS struct {
	agent       *agentS
	fsm         *f.FSM
	timer       *time.Timer
	retriesLeft int
}

func newFSM(agent *agentS) *fsmS {
	agent.logger.Warn("Stan is on the scene. Starting Instana instrumentation.")
	agent.logger.Debug("initializing fsm")

	ret := &fsmS{
		agent:       agent,
		retriesLeft: maximumRetries,
	}

	ret.fsm = f.NewFSM(
		"none",
		f.Events{
			{Name: eInit, Src: []string{"none", "unannounced", "announced", "ready"}, Dst: "init"},
			{Name: eLookup, Src: []string{"init"}, Dst: "unannounced"},
			{Name: eAnnounce, Src: []string{"unannounced"}, Dst: "announced"},
			{Name: eTest, Src: []string{"announced"}, Dst: "ready"}},
		f.Callbacks{
			"init":              ret.lookupAgentHost,
			"enter_unannounced": ret.announceSensor,
			"enter_announced":   ret.testAgent,
		})
	ret.fsm.Event(eInit)

	return ret
}

func (r *fsmS) scheduleRetry(e *f.Event, cb func(e *f.Event)) {
	r.timer = time.NewTimer(retryPeriod)
	go func() {
		<-r.timer.C
		cb(e)
	}()
}

func (r *fsmS) scheduleRetryWithExponentialDelay(e *f.Event, cb func(e *f.Event), retryNumber int) {
	r.timer = time.NewTimer(getRetryPeriodForIteration(retryNumber))
	go func() {
		<-r.timer.C
		cb(e)
	}()
}

func (r *fsmS) lookupAgentHost(e *f.Event) {
	cb := func(found bool, host string) {
		if found {
			r.lookupSuccess(host)
			return
		}

		gateway, err := getDefaultGateway("/proc/net/route")
		if err != nil {
			r.agent.logger.Error("failed to fetch the default gateway, scheduling retry: ", err)
			r.scheduleRetry(e, r.lookupAgentHost)

			return
		}

		if gateway == "" {
			r.agent.logger.Error("default gateway not available, scheduling retry")
			r.scheduleRetry(e, r.lookupAgentHost)

			return
		}

		go r.checkHost(gateway, func(found bool, host string) {
			if found {
				r.lookupSuccess(host)
				return
			}

			r.agent.logger.Error("cannot connect to the agent through localhost or default gateway, scheduling retry")
			r.scheduleRetry(e, r.lookupAgentHost)
		})

	}

	go r.checkHost(r.agent.host, cb)
}

func (r *fsmS) checkHost(host string, cb func(found bool, host string)) {
	r.agent.logger.Debug("checking host ", host)

	header, err := r.agent.requestHeader(r.agent.makeHostURL(host, "/"), "GET", "Server")

	cb(err == nil && header == agentHeader, host)
}

func (r *fsmS) lookupSuccess(host string) {
	r.agent.logger.Debug("agent lookup success ", host)

	r.agent.setHost(host)
	r.retriesLeft = maximumRetries
	r.fsm.Event(eLookup)
}

func (r *fsmS) announceSensor(e *f.Event) {
	cb := func(success bool, resp agentResponse) {
		if !success {
			r.agent.logger.Error("Cannot announce sensor. Scheduling retry.")
			r.retriesLeft--
			if r.retriesLeft == 0 {
				r.fsm.Event(eInit)
				return
			}

			retryNumber := maximumRetries - r.retriesLeft + 1
			r.scheduleRetryWithExponentialDelay(e, r.announceSensor, retryNumber)

			return
		}

		r.agent.logger.Info("Host agent available. We're in business. Announced pid:", resp.Pid)
		r.agent.applyHostAgentSettings(resp)

		r.retriesLeft = maximumRetries
		r.fsm.Event(eAnnounce)
	}

	r.agent.logger.Debug("announcing sensor to the agent")

	go func(cb func(success bool, resp agentResponse)) {
		defer func() {
			if err := recover(); err != nil {
				r.agent.logger.Debug("Announce recovered:", err)
			}
		}()

		pid := os.Getpid()
		cpuSetFileContent := ""

		if runtime.GOOS == "linux" {
			cpuSetFileContent = r.cpuSetFileContent(pid)
		}

		d := &discoveryS{
			PID:               pid,
			CPUSetFileContent: cpuSetFileContent,
			Name:              os.Args[0],
			Args:              os.Args[1:],
		}
		if name, args, ok := getProcCommandLine(); ok {
			r.agent.logger.Debug("got cmdline from /proc: ", name, args)
			d.Name, d.Args = name, args
		} else {
			r.agent.logger.Debug("no /proc, using OS reported cmdline")
		}

		if _, err := os.Stat("/proc"); err == nil {
			if addr, err := net.ResolveTCPAddr("tcp", r.agent.host+":42699"); err == nil {
				if tcpConn, err := net.DialTCP("tcp", nil, addr); err == nil {
					defer tcpConn.Close()

					file, err := tcpConn.File()

					if err != nil {
						r.agent.logger.Error(err)
					} else {
						d.Fd = fmt.Sprintf("%v", file.Fd())

						link := fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), file.Fd())
						if _, err := os.Stat(link); err == nil {
							d.Inode, _ = os.Readlink(link)
						}
					}
				}
			}
		}

		var resp agentResponse
		_, err := r.agent.announceRequest(r.agent.makeURL(agentDiscoveryURL), "PUT", d, &resp)
		cb(err == nil, resp)
	}(cb)
}

func (r *fsmS) testAgent(e *f.Event) {
	cb := func(b bool) {
		if b {
			r.retriesLeft = maximumRetries
			r.fsm.Event(eTest)
		} else {
			r.agent.logger.Debug("Agent is not yet ready. Scheduling retry.")
			r.retriesLeft--
			if r.retriesLeft > 0 {
				retryNumber := maximumRetries - r.retriesLeft + 1
				r.scheduleRetryWithExponentialDelay(e, r.testAgent, retryNumber)
			} else {
				r.fsm.Event(eInit)
			}
		}
	}

	r.agent.logger.Debug("testing communication with the agent")

	go func(cb func(b bool)) {
		_, err := r.agent.head(r.agent.makeURL(agentDataURL))
		cb(err == nil)
	}(cb)
}

func (r *fsmS) reset() {
	r.retriesLeft = maximumRetries
	r.fsm.Event(eInit)
}

func (r *fsmS) cpuSetFileContent(pid int) string {
	path := filepath.Join("proc", strconv.Itoa(pid), "cpuset")
	data, err := ioutil.ReadFile(path)
	if err != nil {
		r.agent.logger.Info("error while reading ", path, ":", err.Error())
		return ""
	}

	return string(data)
}

func getRetryPeriodForIteration(retryNumber int) time.Duration {
	return time.Duration(math.Pow(2, float64(retryNumber-1))) * exponentialRetryPeriodBase
}
<commit_msg>change scheduleRetryWithExponentialDelay to be sync<commit_after>// (c) Copyright IBM Corp. 2021
// (c) Copyright Instana Inc. 2016

package instana

import (
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	f "github.com/looplab/fsm"
)

const (
	eInit     = "init"
	eLookup   = "lookup"
	eAnnounce = "announce"
	eTest     = "test"

	retryPeriod                = 30 * 1000 * time.Millisecond
	exponentialRetryPeriodBase = 10 * 1000 * time.Millisecond
	maximumRetries             = 3
)

type fsmS struct {
	agent       *agentS
	fsm         *f.FSM
	timer       *time.Timer
	retriesLeft int
}

func newFSM(agent *agentS) *fsmS {
	agent.logger.Warn("Stan is on the scene. Starting Instana instrumentation.")
	agent.logger.Debug("initializing fsm")

	ret := &fsmS{
		agent:       agent,
		retriesLeft: maximumRetries,
	}

	ret.fsm = f.NewFSM(
		"none",
		f.Events{
			{Name: eInit, Src: []string{"none", "unannounced", "announced", "ready"}, Dst: "init"},
			{Name: eLookup, Src: []string{"init"}, Dst: "unannounced"},
			{Name: eAnnounce, Src: []string{"unannounced"}, Dst: "announced"},
			{Name: eTest, Src: []string{"announced"}, Dst: "ready"}},
		f.Callbacks{
			"init":              ret.lookupAgentHost,
			"enter_unannounced": ret.announceSensor,
			"enter_announced":   ret.testAgent,
		})
	ret.fsm.Event(eInit)

	return ret
}

func (r *fsmS) scheduleRetry(e *f.Event, cb func(e *f.Event)) {
	r.timer = time.NewTimer(retryPeriod)
	go func() {
		<-r.timer.C
		cb(e)
	}()
}

func (r *fsmS) scheduleRetryWithExponentialDelay(e *f.Event, cb func(e *f.Event), retryNumber int) {
	time.Sleep(getRetryPeriodForIteration(retryNumber))
	cb(e)
}

func (r *fsmS) lookupAgentHost(e *f.Event) {
	cb := func(found bool, host string) {
		if found {
			r.lookupSuccess(host)
			return
		}

		gateway, err := getDefaultGateway("/proc/net/route")
		if err != nil {
			r.agent.logger.Error("failed to fetch the default gateway, scheduling retry: ", err)
			r.scheduleRetry(e, r.lookupAgentHost)

			return
		}

		if gateway == "" {
			r.agent.logger.Error("default gateway not available, scheduling retry")
			r.scheduleRetry(e, r.lookupAgentHost)

			return
		}

		go r.checkHost(gateway, func(found bool, host string) {
			if found {
				r.lookupSuccess(host)
				return
			}

			r.agent.logger.Error("cannot connect to the agent through localhost or default gateway, scheduling retry")
			r.scheduleRetry(e, r.lookupAgentHost)
		})

	}

	go r.checkHost(r.agent.host, cb)
}

func (r *fsmS) checkHost(host string, cb func(found bool, host string)) {
	r.agent.logger.Debug("checking host ", host)

	header, err := r.agent.requestHeader(r.agent.makeHostURL(host, "/"), "GET", "Server")

	cb(err == nil && header == agentHeader, host)
}

func (r *fsmS) lookupSuccess(host string) {
	r.agent.logger.Debug("agent lookup success ", host)

	r.agent.setHost(host)
	r.retriesLeft = maximumRetries
	r.fsm.Event(eLookup)
}

func (r *fsmS) announceSensor(e *f.Event) {
	cb := func(success bool, resp agentResponse) {
		if !success {
			r.agent.logger.Error("Cannot announce sensor. Scheduling retry.")
			r.retriesLeft--
			if r.retriesLeft == 0 {
				r.fsm.Event(eInit)
				return
			}

			retryNumber := maximumRetries - r.retriesLeft + 1
			go r.scheduleRetryWithExponentialDelay(e, r.announceSensor, retryNumber)

			return
		}

		r.agent.logger.Info("Host agent available. We're in business. Announced pid:", resp.Pid)
		r.agent.applyHostAgentSettings(resp)

		r.retriesLeft = maximumRetries
		r.fsm.Event(eAnnounce)
	}

	r.agent.logger.Debug("announcing sensor to the agent")

	go func(cb func(success bool, resp agentResponse)) {
		defer func() {
			if err := recover(); err != nil {
				r.agent.logger.Debug("Announce recovered:", err)
			}
		}()

		pid := os.Getpid()
		cpuSetFileContent := ""

		if runtime.GOOS == "linux" {
			cpuSetFileContent = r.cpuSetFileContent(pid)
		}

		d := &discoveryS{
			PID:               pid,
			CPUSetFileContent: cpuSetFileContent,
			Name:              os.Args[0],
			Args:              os.Args[1:],
		}
		if name, args, ok := getProcCommandLine(); ok {
			r.agent.logger.Debug("got cmdline from /proc: ", name, args)
			d.Name, d.Args = name, args
		} else {
			r.agent.logger.Debug("no /proc, using OS reported cmdline")
		}

		if _, err := os.Stat("/proc"); err == nil {
			if addr, err := net.ResolveTCPAddr("tcp", r.agent.host+":42699"); err == nil {
				if tcpConn, err := net.DialTCP("tcp", nil, addr); err == nil {
					defer tcpConn.Close()

					file, err := tcpConn.File()

					if err != nil {
						r.agent.logger.Error(err)
					} else {
						d.Fd = fmt.Sprintf("%v", file.Fd())

						link := fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), file.Fd())
						if _, err := os.Stat(link); err == nil {
							d.Inode, _ = os.Readlink(link)
						}
					}
				}
			}
		}

		var resp agentResponse
		_, err := r.agent.announceRequest(r.agent.makeURL(agentDiscoveryURL), "PUT", d, &resp)
		cb(err == nil, resp)
	}(cb)
}

func (r *fsmS) testAgent(e *f.Event) {
	cb := func(b bool) {
		if b {
			r.retriesLeft = maximumRetries
			r.fsm.Event(eTest)
		} else {
			r.agent.logger.Debug("Agent is not yet ready. Scheduling retry.")
			r.retriesLeft--
			if r.retriesLeft > 0 {
				retryNumber := maximumRetries - r.retriesLeft + 1
				go r.scheduleRetryWithExponentialDelay(e, r.testAgent, retryNumber)
			} else {
				r.fsm.Event(eInit)
			}
		}
	}

	r.agent.logger.Debug("testing communication with the agent")

	go func(cb func(b bool)) {
		_, err := r.agent.head(r.agent.makeURL(agentDataURL))
		cb(err == nil)
	}(cb)
}

func (r *fsmS) reset() {
	r.retriesLeft = maximumRetries
	r.fsm.Event(eInit)
}

func (r *fsmS) cpuSetFileContent(pid int) string {
	path := filepath.Join("proc", strconv.Itoa(pid), "cpuset")
	data, err := ioutil.ReadFile(path)
	if err != nil {
		r.agent.logger.Info("error while reading ", path, ":", err.Error())
		return ""
	}

	return string(data)
}

func getRetryPeriodForIteration(retryNumber int) time.Duration {
	return time.Duration(math.Pow(2, float64(retryNumber-1))) * exponentialRetryPeriodBase
}
<|endoftext|>


<commit_before>//    Copyright 2017 drillbits
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package srgen

import (
	"bufio"
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"os"
	"sort"
	"strings"
	"text/template"
)

var servicesTmpl = `package {{.Package}}

import (
	"fmt"
	"strings"
)

var reg = &ServiceRegistry{}

// ServiceRegistry is an registry for services.
type ServiceRegistry struct {
	valid bool

{{- range .Services}}
	{{.Name}} {{.Name}}
{{- end}}
}

// Validate reports whether all services are fulfilled.
func (reg *ServiceRegistry) Validate() error {
	if reg.valid {
		return nil
	}

	var errs []string

{{- range .Services}}
	if reg.{{.Name}} == nil {
		errs = append(errs, "{{.Name}}")
	}
{{- end}}

	if len(errs) > 0 {
		return fmt.Errorf("nil service(s): %s", strings.Join(errs, ", "))
	}

	return nil
}

// InitServiceRegistry initializes the ServiceRegistry.
func InitServiceRegistry(
{{- range .Services}}
	{{.Name}} {{.Name}},
{{- end}}
) {
{{- range .Services}}
	reg.{{.Name}} = {{.Name}}
{{- end}}
}

// Services returns the ServiceRegistry.
func Services() *ServiceRegistry {
	return reg
}

// MustServices is like Services but panics if all services are not fulfilled.
func MustServices() *ServiceRegistry {
	if err := reg.Validate(); err != nil {
		panic(err)
	}
	return reg
}

{{- range .Services}}
// {{.Name}}Mock implements {{.Name}} for mocking.
type {{.Name}}Mock struct {
	{{- range $_, $m := .Methods}}
	{{- range $i, $v := .Results}}
	{{$m.Name}}Ret{{$i}} {{$v}}
	{{- end}}
	{{- end}}
}

{{end}}
`

type Service struct {
	Name    string
	Methods []*ServiceMethod
}

type ServiceMethod struct {
	Name    string
	Params  []string
	Results []string
}

func Generate(files []string, outfile string) error {
	var pkg string
	var services []*Service
	for _, file := range files {
		fset := token.NewFileSet()
		f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
		if err != nil {
			return err
		}

		if pkg == "" {
			pkg = f.Name.Name
		} else if pkg != f.Name.Name {
			return fmt.Errorf("multiple packages: %s, %s", pkg, f.Name.Name)
		}

		ast.Inspect(f, func(node ast.Node) bool {
			// type or not
			d, ok := node.(*ast.GenDecl)
			if !ok || d.Tok != token.TYPE {
				return true
			}
			// tag
			if !findTag(d, "+srgen") {
				return true
			}

			for _, spec := range d.Specs {
				if !isInterface(spec) {
					continue
				}
				t := spec.(*ast.TypeSpec)
				svc := &Service{Name: t.Name.Name}

				i := t.Type.(*ast.InterfaceType)
				for _, f := range i.Methods.List {
					var fnName string
					for _, n := range f.Names {
						fnName = n.Name
					}
					if fnName == "" {
						continue
					}
					m := &ServiceMethod{Name: fnName}
					svc.Methods = append(svc.Methods, m)

					fn, ok := f.Type.(*ast.FuncType)
					if !ok {
						continue
					}

					for _, f := range fn.Params.List {
						s := fieldString(f.Type)
						if s != "" {
							m.Params = append(m.Params, s)
						}
					}

					for _, f := range fn.Results.List {
						s := fieldString(f.Type)
						if s != "" {
							m.Results = append(m.Results, s)
						}
					}
				}
				services = append(services, svc)
			}

			return true
		})
	}
	sort.SliceStable(services, func(i, j int) bool {
		return services[i].Name < services[j].Name
	})

	if outfile == "" {
		outfile = "services.go"
	}

	var buf bytes.Buffer
	w := bufio.NewWriter(&buf)
	t := template.Must(template.New("services").Parse(servicesTmpl))
	t.Execute(w, struct {
		Package  string
		Services []*Service
	}{
		Package:  pkg,
		Services: services,
	})
	w.Flush()

	b, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	f, err := os.Create(outfile)
	if err != nil {
		return err
	}
	defer f.Close()

	w = bufio.NewWriter(f)
	defer w.Flush()
	w.Write(b)

	return nil
}

func findTag(d *ast.GenDecl, tag string) bool {
	if d.Doc == nil {
		return false
	}
	for _, c := range d.Doc.List {
		comment := strings.TrimSpace(strings.TrimLeft(c.Text, "//"))
		if strings.HasPrefix(comment, tag) {
			return true
		}
	}
	return false
}

func isInterface(spec ast.Spec) bool {
	t, ok := spec.(*ast.TypeSpec)
	if !ok {
		return false
	}
	if _, ok := t.Type.(*ast.InterfaceType); !ok {
		return false
	}
	return true
}

func fieldString(expr ast.Expr) string {
	switch v := expr.(type) {
	case *ast.ArrayType:
		s := fieldString(v.Elt)
		if s != "" {
			return "[]" + s
		}
	case *ast.StarExpr:
		s := fieldString(v.X)
		if s != "" {
			return "*" + s
		}
	case *ast.Ident:
		return v.Name
	case *ast.SelectorExpr:
		if x, ok := v.X.(*ast.Ident); ok {
			// TODO: import unknown selector
			return fmt.Sprintf("%s.%s", x.Name, v.Sel.Name)
		}
	}
	return ""
}
<commit_msg>Generate functions of mocks<commit_after>//    Copyright 2017 drillbits
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package srgen

import (
	"bufio"
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"os"
	"sort"
	"strings"
	"text/template"
)

var servicesTmpl = `package {{.Package}}

import (
	"fmt"
	"strings"
)

var reg = &ServiceRegistry{}

// ServiceRegistry is an registry for services.
type ServiceRegistry struct {
	valid bool

{{- range .Services}}
	{{.Name}} {{.Name}}
{{- end}}
}

// Validate reports whether all services are fulfilled.
func (reg *ServiceRegistry) Validate() error {
	if reg.valid {
		return nil
	}

	var errs []string

{{- range .Services}}
	if reg.{{.Name}} == nil {
		errs = append(errs, "{{.Name}}")
	}
{{- end}}

	if len(errs) > 0 {
		return fmt.Errorf("nil service(s): %s", strings.Join(errs, ", "))
	}

	return nil
}

// InitServiceRegistry initializes the ServiceRegistry.
func InitServiceRegistry(
{{- range .Services}}
	{{.Name}} {{.Name}},
{{- end}}
) {
{{- range .Services}}
	reg.{{.Name}} = {{.Name}}
{{- end}}
}

// Services returns the ServiceRegistry.
func Services() *ServiceRegistry {
	return reg
}

// MustServices is like Services but panics if all services are not fulfilled.
func MustServices() *ServiceRegistry {
	if err := reg.Validate(); err != nil {
		panic(err)
	}
	return reg
}

{{- range $_, $s := .Services}}
// {{$s.Name}}Mock implements {{$s.Name}} for mocking.
type {{$s.Name}}Mock struct {
	{{- range $_, $m := $s.Methods}}
	{{- range $i, $r := $m.Results}}
	{{$m.Name}}Ret{{$i}} {{$r}}
	{{- end}}
	{{- end}}
}

{{- range $_, $m := $s.Methods}}
func (s *{{$s.Name}}Mock) {{$m.Name}}(
	{{- range $i, $p := $m.Params}}{{if $i}}, {{end}}{{$p}}{{end -}}
) (
	{{- range $i, $r := $m.Results}}{{if $i}}, {{end}}{{$r}}{{end -}}
) {
	return {{range $i, $r := $m.Results -}}
	{{- if $i}}, {{end -}}
	s.{{$m.Name}}Ret{{$i}}
	{{- end}}
}
{{- end}}

{{end}}
`

type Service struct {
	Name    string
	Methods []*ServiceMethod
}

type ServiceMethod struct {
	Name    string
	Params  []string
	Results []string
}

func Generate(files []string, outfile string) error {
	var pkg string
	var services []*Service
	for _, file := range files {
		fset := token.NewFileSet()
		f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
		if err != nil {
			return err
		}

		if pkg == "" {
			pkg = f.Name.Name
		} else if pkg != f.Name.Name {
			return fmt.Errorf("multiple packages: %s, %s", pkg, f.Name.Name)
		}

		ast.Inspect(f, func(node ast.Node) bool {
			// type or not
			d, ok := node.(*ast.GenDecl)
			if !ok || d.Tok != token.TYPE {
				return true
			}
			// tag
			if !findTag(d, "+srgen") {
				return true
			}

			for _, spec := range d.Specs {
				if !isInterface(spec) {
					continue
				}
				t := spec.(*ast.TypeSpec)
				svc := &Service{Name: t.Name.Name}

				i := t.Type.(*ast.InterfaceType)
				for _, f := range i.Methods.List {
					var fnName string
					for _, n := range f.Names {
						fnName = n.Name
					}
					if fnName == "" {
						continue
					}
					m := &ServiceMethod{Name: fnName}
					svc.Methods = append(svc.Methods, m)

					fn, ok := f.Type.(*ast.FuncType)
					if !ok {
						continue
					}

					for _, f := range fn.Params.List {
						s := fieldString(f.Type)
						if s != "" {
							m.Params = append(m.Params, s)
						}
					}

					for _, f := range fn.Results.List {
						s := fieldString(f.Type)
						if s != "" {
							m.Results = append(m.Results, s)
						}
					}
				}
				services = append(services, svc)
			}

			return true
		})
	}
	sort.SliceStable(services, func(i, j int) bool {
		return services[i].Name < services[j].Name
	})

	if outfile == "" {
		outfile = "services.go"
	}

	var buf bytes.Buffer
	w := bufio.NewWriter(&buf)
	t := template.Must(template.New("services").Parse(servicesTmpl))
	t.Execute(w, struct {
		Package  string
		Services []*Service
	}{
		Package:  pkg,
		Services: services,
	})
	w.Flush()

	b, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	f, err := os.Create(outfile)
	if err != nil {
		return err
	}
	defer f.Close()

	w = bufio.NewWriter(f)
	defer w.Flush()
	w.Write(b)

	return nil
}

func findTag(d *ast.GenDecl, tag string) bool {
	if d.Doc == nil {
		return false
	}
	for _, c := range d.Doc.List {
		comment := strings.TrimSpace(strings.TrimLeft(c.Text, "//"))
		if strings.HasPrefix(comment, tag) {
			return true
		}
	}
	return false
}

func isInterface(spec ast.Spec) bool {
	t, ok := spec.(*ast.TypeSpec)
	if !ok {
		return false
	}
	if _, ok := t.Type.(*ast.InterfaceType); !ok {
		return false
	}
	return true
}

func fieldString(expr ast.Expr) string {
	switch v := expr.(type) {
	case *ast.ArrayType:
		s := fieldString(v.Elt)
		if s != "" {
			return "[]" + s
		}
	case *ast.StarExpr:
		s := fieldString(v.X)
		if s != "" {
			return "*" + s
		}
	case *ast.Ident:
		return v.Name
	case *ast.SelectorExpr:
		if x, ok := v.X.(*ast.Ident); ok {
			// TODO: import unknown selector
			return fmt.Sprintf("%s.%s", x.Name, v.Sel.Name)
		}
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore
// +build ignore

// gen runs go generate on Unicode- and CLDR-related package in the text
// repositories, taking into account dependencies and versions.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/format"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"unicode"

	"golang.org/x/text/collate"
	"golang.org/x/text/internal/gen"
	"golang.org/x/text/language"
)

var (
	verbose     = flag.Bool("v", false, "verbose output")
	force       = flag.Bool("force", false, "ignore failing dependencies")
	doCore      = flag.Bool("core", false, "force an update to core")
	skipTest    = flag.Bool("skiptest", false, "skip tests")
	excludeList = flag.String("exclude", "",
		"comma-separated list of packages to exclude")

	// The user can specify a selection of packages to build on the command line.
	args []string
)

func exclude(pkg string) bool {
	if len(args) > 0 {
		return !contains(args, pkg)
	}
	return contains(strings.Split(*excludeList, ","), pkg)
}

// TODO:
// - Better version handling.
// - Generate tables for the core unicode package?
// - Add generation for encodings. This requires some retooling here and there.
// - Running repo-wide "long" tests.

var vprintf = fmt.Printf

func main() {
	gen.Init()
	args = flag.Args()
	if !*verbose {
		// Set vprintf to a no-op.
		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
	}

	// TODO: create temporary cache directory to load files and create and set
	// a "cache" option if the user did not specify the UNICODE_DIR environment
	// variable. This will prevent duplicate downloads and also will enable long
	// tests, which really need to be run after each generated package.

	updateCore := *doCore
	if gen.UnicodeVersion() != unicode.Version {
		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
			gen.UnicodeVersion(),
			unicode.Version)
		c := collate.New(language.Und, collate.Numeric)
		if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force {
			os.Exit(2)
		}
		updateCore = true
		goroot := os.Getenv("GOROOT")
		appendToFile(
			filepath.Join(goroot, "api", "except.txt"),
			fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version),
		)
		const lines = `pkg unicode, const Version = %q
// TODO: add a new line of the following form for each new script and property.
pkg unicode, var <new script or property> *RangeTable
`
		appendToFile(
			filepath.Join(goroot, "api", "next.txt"),
			fmt.Sprintf(lines, gen.UnicodeVersion()),
		)
	}

	var unicode = &dependency{}
	if updateCore {
		fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion())
		unicodeInternal := generate("./internal/export/unicode")
		unicode = generate("unicode", unicodeInternal)

		// Test some users of the unicode packages, especially the ones that
		// keep a mirrored table. These may need to be corrected by hand.
		generate("regexp", unicode)
		generate("strconv", unicode) // mimics Unicode table
		generate("strings", unicode)
		generate("testing", unicode) // mimics Unicode table
	}

	var (
		cldr       = generate("./unicode/cldr", unicode)
		intlang    = generate("./internal/language", cldr)
		compact    = generate("./internal/language/compact", intlang, cldr)
		language   = generate("./language", cldr, compact)
		internal   = generate("./internal", unicode, language)
		norm       = generate("./unicode/norm", unicode)
		rangetable = generate("./unicode/rangetable", unicode)
		cases      = generate("./cases", unicode, norm, language, rangetable)
		width      = generate("./width", unicode)
		bidi       = generate("./unicode/bidi", unicode, norm, rangetable)
		mib        = generate("./encoding/internal/identifier", unicode)
		number     = generate("./internal/number", unicode, cldr, language, internal)
		cldrtree   = generate("./internal/cldrtree", language, internal)
		_          = generate("./unicode/runenames", unicode)
		_          = generate("./encoding/htmlindex", unicode, language, mib)
		_          = generate("./encoding/ianaindex", unicode, language, mib)
		_          = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi)
		_          = generate("./currency", unicode, cldr, language, internal, number)
		_          = generate("./feature/plural", unicode, cldr, language, internal, number)
		_          = generate("./internal/export/idna", unicode, bidi, norm)
		_          = generate("./language/display", unicode, cldr, language, internal, number)
		_          = generate("./collate", unicode, norm, cldr, language, rangetable)
		_          = generate("./search", unicode, norm, cldr, language, rangetable)
		_          = generate("./date", cldr, language, cldrtree)
	)
	all.Wait()

	// Copy exported packages to the destination golang.org repo.
	copyExported("golang.org/x/net/idna")

	if hasErrors {
		fmt.Println("FAIL")
		os.Exit(1)
	}
	vprintf("SUCCESS\n")
}

func appendToFile(file, text string) {
	fmt.Println("Augmenting", file)
	w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		fmt.Println("Failed to open file:", err)
		os.Exit(1)
	}
	defer w.Close()
	if _, err := w.WriteString(text); err != nil {
		fmt.Println("Failed to write to file:", err)
		os.Exit(1)
	}
}

var (
	all       sync.WaitGroup
	hasErrors bool
)

type dependency struct {
	sync.WaitGroup
	hasErrors bool
}

func generate(pkg string, deps ...*dependency) *dependency {
	var wg dependency
	if exclude(pkg) {
		return &wg
	}
	wg.Add(1)
	all.Add(1)
	go func() {
		defer wg.Done()
		defer all.Done()
		// Wait for dependencies to finish.
		for _, d := range deps {
			d.Wait()
			if d.hasErrors && !*force {
				fmt.Printf("--- ABORT: %s\n", pkg)
				wg.hasErrors = true
				return
			}
		}
		vprintf("=== GENERATE %s\n", pkg)
		args := []string{"generate"}
		if *verbose {
			args = append(args, "-v")
		}
		args = append(args, pkg)
		cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
		w := &bytes.Buffer{}
		cmd.Stderr = w
		cmd.Stdout = w
		if err := cmd.Run(); err != nil {
			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err)
			hasErrors = true
			wg.hasErrors = true
			return
		}

		if *skipTest {
			return
		}

		vprintf("=== TEST %s\n", pkg)
		args[0] = "test"
		cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
		wt := &bytes.Buffer{}
		cmd.Stderr = wt
		cmd.Stdout = wt
		if err := cmd.Run(); err != nil {
			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err)
			hasErrors = true
			wg.hasErrors = true
			return
		}
		vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w))
		fmt.Print(wt.String())
	}()
	return &wg
}

// copyExported copies a package in x/text/internal/export to the
// destination repository.
func copyExported(p string) {
	copyPackage(
		filepath.Join("internal", "export", path.Base(p)),
		filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])),
		"golang.org/x/text/internal/export/"+path.Base(p),
		p)
}

// goGenRE is used to remove go:generate lines.
var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n")

// copyPackage copies relevant files from a directory in x/text to the
// destination package directory. The destination package is assumed to have
// the same name. For each copied file go:generate lines are removed and
// package comments are rewritten to the new path.
func copyPackage(dirSrc, dirDst, search, replace string) {
	err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error {
		base := filepath.Base(file)
		if err != nil || info.IsDir() ||
			!strings.HasSuffix(base, ".go") ||
			strings.HasSuffix(base, "_test.go") ||
			// Don't process subdirectories.
			filepath.Dir(file) != dirSrc {
			return nil
		}
		if strings.HasPrefix(base, "tables") {
			if !strings.HasSuffix(base, gen.UnicodeVersion()+".go") {
				return nil
			}
			base = "tables.go"
		}
		b, err := ioutil.ReadFile(file)
		if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) {
			return err
		}
		// Fix paths.
		b = bytes.Replace(b, []byte(search), []byte(replace), -1)
		b = bytes.Replace(b, []byte("internal/export"), []byte(""), -1)
		// Remove go:generate lines.
		b = goGenRE.ReplaceAllLiteral(b, nil)
		comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n"
		if !bytes.HasPrefix(b, []byte(comment)) {
			b = append([]byte(comment), b...)
		}
		if b, err = format.Source(b); err != nil {
			fmt.Println("Failed to format file:", err)
			os.Exit(1)
		}
		file = filepath.Join(dirDst, base)
		vprintf("=== COPY %s\n", file)
		return ioutil.WriteFile(file, b, 0666)
	})
	if err != nil {
		fmt.Println("Copying exported files failed:", err)
		os.Exit(1)
	}
}

func contains(a []string, s string) bool {
	for _, e := range a {
		if s == e {
			return true
		}
	}
	return false
}

func indent(b *bytes.Buffer) string {
	return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1)
}
<commit_msg>gen.go: copy all tablesXX.X.X.go versions to golang.org/x/net<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore
// +build ignore

// gen runs go generate on Unicode- and CLDR-related package in the text
// repositories, taking into account dependencies and versions.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/format"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"unicode"

	"golang.org/x/text/collate"
	"golang.org/x/text/internal/gen"
	"golang.org/x/text/language"
)

var (
	verbose     = flag.Bool("v", false, "verbose output")
	force       = flag.Bool("force", false, "ignore failing dependencies")
	doCore      = flag.Bool("core", false, "force an update to core")
	skipTest    = flag.Bool("skiptest", false, "skip tests")
	excludeList = flag.String("exclude", "",
		"comma-separated list of packages to exclude")

	// The user can specify a selection of packages to build on the command line.
	args []string
)

func exclude(pkg string) bool {
	if len(args) > 0 {
		return !contains(args, pkg)
	}
	return contains(strings.Split(*excludeList, ","), pkg)
}

// TODO:
// - Better version handling.
// - Generate tables for the core unicode package?
// - Add generation for encodings. This requires some retooling here and there.
// - Running repo-wide "long" tests.

var vprintf = fmt.Printf

func main() {
	gen.Init()
	args = flag.Args()
	if !*verbose {
		// Set vprintf to a no-op.
		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
	}

	// TODO: create temporary cache directory to load files and create and set
	// a "cache" option if the user did not specify the UNICODE_DIR environment
	// variable. This will prevent duplicate downloads and also will enable long
	// tests, which really need to be run after each generated package.

	updateCore := *doCore
	if gen.UnicodeVersion() != unicode.Version {
		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
			gen.UnicodeVersion(),
			unicode.Version)
		c := collate.New(language.Und, collate.Numeric)
		if c.CompareString(gen.UnicodeVersion(), unicode.Version) < 0 && !*force {
			os.Exit(2)
		}
		updateCore = true
		goroot := os.Getenv("GOROOT")
		appendToFile(
			filepath.Join(goroot, "api", "except.txt"),
			fmt.Sprintf("pkg unicode, const Version = %q\n", unicode.Version),
		)
		const lines = `pkg unicode, const Version = %q
// TODO: add a new line of the following form for each new script and property.
pkg unicode, var <new script or property> *RangeTable
`
		appendToFile(
			filepath.Join(goroot, "api", "next.txt"),
			fmt.Sprintf(lines, gen.UnicodeVersion()),
		)
	}

	var unicode = &dependency{}
	if updateCore {
		fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion())
		unicodeInternal := generate("./internal/export/unicode")
		unicode = generate("unicode", unicodeInternal)

		// Test some users of the unicode packages, especially the ones that
		// keep a mirrored table. These may need to be corrected by hand.
		generate("regexp", unicode)
		generate("strconv", unicode) // mimics Unicode table
		generate("strings", unicode)
		generate("testing", unicode) // mimics Unicode table
	}

	var (
		cldr       = generate("./unicode/cldr", unicode)
		intlang    = generate("./internal/language", cldr)
		compact    = generate("./internal/language/compact", intlang, cldr)
		language   = generate("./language", cldr, compact)
		internal   = generate("./internal", unicode, language)
		norm       = generate("./unicode/norm", unicode)
		rangetable = generate("./unicode/rangetable", unicode)
		cases      = generate("./cases", unicode, norm, language, rangetable)
		width      = generate("./width", unicode)
		bidi       = generate("./unicode/bidi", unicode, norm, rangetable)
		mib        = generate("./encoding/internal/identifier", unicode)
		number     = generate("./internal/number", unicode, cldr, language, internal)
		cldrtree   = generate("./internal/cldrtree", language, internal)
		_          = generate("./unicode/runenames", unicode)
		_          = generate("./encoding/htmlindex", unicode, language, mib)
		_          = generate("./encoding/ianaindex", unicode, language, mib)
		_          = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi)
		_          = generate("./currency", unicode, cldr, language, internal, number)
		_          = generate("./feature/plural", unicode, cldr, language, internal, number)
		_          = generate("./internal/export/idna", unicode, bidi, norm)
		_          = generate("./language/display", unicode, cldr, language, internal, number)
		_          = generate("./collate", unicode, norm, cldr, language, rangetable)
		_          = generate("./search", unicode, norm, cldr, language, rangetable)
		_          = generate("./date", cldr, language, cldrtree)
	)
	all.Wait()

	// Copy exported packages to the destination golang.org repo.
	copyExported("golang.org/x/net/idna")

	if hasErrors {
		fmt.Println("FAIL")
		os.Exit(1)
	}
	vprintf("SUCCESS\n")
}

func appendToFile(file, text string) {
	fmt.Println("Augmenting", file)
	w, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		fmt.Println("Failed to open file:", err)
		os.Exit(1)
	}
	defer w.Close()
	if _, err := w.WriteString(text); err != nil {
		fmt.Println("Failed to write to file:", err)
		os.Exit(1)
	}
}

var (
	all       sync.WaitGroup
	hasErrors bool
)

type dependency struct {
	sync.WaitGroup
	hasErrors bool
}

func generate(pkg string, deps ...*dependency) *dependency {
	var wg dependency
	if exclude(pkg) {
		return &wg
	}
	wg.Add(1)
	all.Add(1)
	go func() {
		defer wg.Done()
		defer all.Done()
		// Wait for dependencies to finish.
		for _, d := range deps {
			d.Wait()
			if d.hasErrors && !*force {
				fmt.Printf("--- ABORT: %s\n", pkg)
				wg.hasErrors = true
				return
			}
		}
		vprintf("=== GENERATE %s\n", pkg)
		args := []string{"generate"}
		if *verbose {
			args = append(args, "-v")
		}
		args = append(args, pkg)
		cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
		w := &bytes.Buffer{}
		cmd.Stderr = w
		cmd.Stdout = w
		if err := cmd.Run(); err != nil {
			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(w), err)
			hasErrors = true
			wg.hasErrors = true
			return
		}

		if *skipTest {
			return
		}

		vprintf("=== TEST %s\n", pkg)
		args[0] = "test"
		cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
		wt := &bytes.Buffer{}
		cmd.Stderr = wt
		cmd.Stdout = wt
		if err := cmd.Run(); err != nil {
			fmt.Printf("--- FAIL: %s:\n\t%v\n\tError: %v\n", pkg, indent(wt), err)
			hasErrors = true
			wg.hasErrors = true
			return
		}
		vprintf("--- SUCCESS: %s\n\t%v\n", pkg, indent(w))
		fmt.Print(wt.String())
	}()
	return &wg
}

// copyExported copies a package in x/text/internal/export to the
// destination repository.
func copyExported(p string) {
	copyPackage(
		filepath.Join("internal", "export", path.Base(p)),
		filepath.Join("..", filepath.FromSlash(p[len("golang.org/x"):])),
		"golang.org/x/text/internal/export/"+path.Base(p),
		p)
}

// goGenRE is used to remove go:generate lines.
var goGenRE = regexp.MustCompile("//go:generate[^\n]*\n")

// copyPackage copies relevant files from a directory in x/text to the
// destination package directory. The destination package is assumed to have
// the same name. For each copied file go:generate lines are removed and
// package comments are rewritten to the new path.
func copyPackage(dirSrc, dirDst, search, replace string) {
	err := filepath.Walk(dirSrc, func(file string, info os.FileInfo, err error) error {
		base := filepath.Base(file)
		if err != nil || info.IsDir() ||
			!strings.HasSuffix(base, ".go") ||
			strings.HasSuffix(base, "_test.go") ||
			// Don't process subdirectories.
			filepath.Dir(file) != dirSrc {
			return nil
		}
		b, err := ioutil.ReadFile(file)
		if err != nil || bytes.Contains(b, []byte("\n// +build ignore")) {
			return err
		}
		// Fix paths.
		b = bytes.Replace(b, []byte(search), []byte(replace), -1)
		b = bytes.Replace(b, []byte("internal/export"), []byte(""), -1)
		// Remove go:generate lines.
		b = goGenRE.ReplaceAllLiteral(b, nil)
		comment := "// Code generated by running \"go generate\" in golang.org/x/text. DO NOT EDIT.\n\n"
		if !bytes.HasPrefix(b, []byte(comment)) {
			b = append([]byte(comment), b...)
		}
		if b, err = format.Source(b); err != nil {
			fmt.Println("Failed to format file:", err)
			os.Exit(1)
		}
		file = filepath.Join(dirDst, base)
		vprintf("=== COPY %s\n", file)
		return ioutil.WriteFile(file, b, 0666)
	})
	if err != nil {
		fmt.Println("Copying exported files failed:", err)
		os.Exit(1)
	}
}

func contains(a []string, s string) bool {
	for _, e := range a {
		if s == e {
			return true
		}
	}
	return false
}

func indent(b *bytes.Buffer) string {
	return strings.Replace(strings.TrimSpace(b.String()), "\n", "\n\t", -1)
}
<|endoftext|>


<commit_before>package qtpm

import (
	"archive/zip"
	"bytes"
	"errors"
	"fmt"
	"git"
	"github.com/fatih/color"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func Get(packageName string, update, useGit bool) {
	parentConfig, _ := LoadConfig(".", true)

	var packages []*PackageConfig
	var err error
	if packageName == "" && parentConfig != nil {
		packages, err = getRecursively(parentConfig, parentConfig.Requires, true, false, update, useGit)
	} else if packageName != "" {
		packages, err = getRecursively(parentConfig, []string{packageName}, true, parentConfig != nil, update, useGit)
	} else {
		fmt.Println("qtpackage.toml or package name argument are needed")
		return
	}
	if err != nil {
		log.Fatalln(err)
	}
	if parentConfig != nil {
		parentConfig.SaveIfDirty()
	}
	var dir string
	if parentConfig != nil {
		dir = parentConfig.Dir
	} else {
		dir, _ = filepath.Abs(".")
	}
	os.MkdirAll(filepath.Join(dir, "qtresources", "translations"), 0755)
	if len(packages) == 0 {
		color.Red("%s package has no dependent packages.\n", parentConfig.Name)
		os.Exit(1)
	}
	for _, packageConfig := range packages[:len(packages)-1] {
		_, err = BuildPackage(parentConfig, packageConfig, update, false, true, false)
		if err != nil {
			continue
		}
	}
}

func getRecursively(rootPackage *PackageConfig, packages []string, download, save, update, useGit bool) ([]*PackageConfig, error) {
	loadedPackages := make(map[string]*PackageConfig)
	if rootPackage != nil {
		rootPackage.Children = loadedPackages
	}
	var edges [][2]string
	for _, packageName := range packages {
		edges = append(edges, [2]string{packageName, "*"})
	}
	nextPackages := packages
	for len(nextPackages) > 0 {
		waitingPackage := []string{}
		for _, packageName := range nextPackages {
			child, err := getSinglePackage(rootPackage, packageName, download, save, update, useGit)
			if err != nil {
				return nil, err
			}
			loadedPackages[packageName] = child
			child.PackageName = packageName
			for _, neededPackage := range child.Requires {
				edges = append(edges, [2]string{neededPackage, packageName})
				if _, ok := loadedPackages[neededPackage]; !ok && !has(waitingPackage, neededPackage) {
					waitingPackage = append(waitingPackage, neededPackage)
				}
			}
		}
		nextPackages = waitingPackage
	}
	packageNamesInBuildOrder, loops := TopologicalSort(edges)
	if len(loops) > 0 {
		var buffer bytes.Buffer
		buffer.WriteString("Circular reference issue. The following dependencies are :\n")
		for _, loop := range loops {
			fmt.Fprintf(&buffer, "  * %s -> %s\n", loop[0], loop[1])
		}
		return nil, errors.New(buffer.String())
	}
	var result []*PackageConfig
	for _, packageName := range packageNamesInBuildOrder {
		if packageName == "*" {
			if rootPackage != nil {
				result = append(result, rootPackage)
			}
		} else {
			result = append(result, loadedPackages[packageName])
		}
	}
	return result, nil
}

func has(list []string, entry string) bool {
	for _, listItem := range list {
		if listItem == entry {
			return true
		}
	}
	return false
}

func getSinglePackage(rootConfig *PackageConfig, packageName string, download, save, update, useGit bool) (*PackageConfig, error) {
	fmt.Println("getSinglePackage", packageName)
	// download from git
	paths := strings.Split(packageName, "/")
	if len(paths) != 3 {
		return nil, fmt.Errorf("qtpm supports only github.com/author/repository style path: %s", packageName)
	}
	var parentDir string
	if rootConfig != nil {
		parentDir = rootConfig.Dir
	} else {
		parentDir, _ = filepath.Abs(".")
	}
	dirName := strings.Join(paths, "___")
	installDir := filepath.Join(parentDir, "vendor", dirName)
	workDir := filepath.Dir(installDir)
	os.MkdirAll(workDir, 0755)

	if _, err := os.Stat(installDir); os.IsNotExist(err) {
		fmt.Println("not found(1)", installDir)
		if !download {
			fmt.Println("not found(2)", installDir)
			installDir = filepath.Clean(filepath.Join(parentDir, "..", dirName))
			workDir = filepath.Dir(installDir)
			if _, err := os.Stat(installDir); os.IsNotExist(err) {
				return nil, fmt.Errorf("%s is not downloaded yet. Run qtpm get first.", packageName)
			}
		}
		if useGit || paths[0] != "github.com" {
			fmt.Println("git clone", installDir)
			err = git.CloneWithoutFilters([]string{
				"--depth", "1", fmt.Sprintf("git@%s:%s/%s.git", paths[0], paths[1], paths[2]), dirName,
			}, workDir)
		} else {
			fmt.Println("download zip", installDir)
			err = DownloadZip(installDir, paths[1], paths[2])
		}
		if err != nil {
			return nil, err
		}
	} else if update {
		_, err := os.Stat(filepath.Join(installDir, ".git"))
		if os.IsNotExist(err) {
			os.RemoveAll(installDir)
			err = DownloadZip(installDir, paths[1], paths[2])
		} else {
			cmd := Command("get", installDir, "pull", "--ff-only")
			err = cmd.Run()
		}
		if err != nil {
			return nil, err
		}
	}
	if save && !has(rootConfig.Requires, packageName) {
		rootConfig.Requires = append(rootConfig.Requires, packageName)
		rootConfig.DirtyFlag = true
	}
	return LoadConfig(installDir, false)
}

func DownloadZip(installDir, orgName, repoName string) error {
	os.MkdirAll(installDir, 0755)
	resp, err := http.Get(fmt.Sprintf("https://codeload.github.com/%s/%s/zip/master", orgName, repoName))
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	rawReader := bytes.NewReader(body)
	reader, err := zip.NewReader(rawReader, int64(len(body)))
	if err != nil {
		return err
	}
	var prefixLength int
	for i, file := range reader.File {
		if i == 0 {
			prefixLength = len(file.Name)
			continue
		}
		path := filepath.Join(installDir, file.Name[prefixLength:])
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, 0755)
			continue
		}
		outFile, err := os.Create(path)
		if err != nil {
			return err
		}
		inFile, err := file.Open()
		if err != nil {
			outFile.Close()
			return err
		}
		io.Copy(outFile, inFile)
		outFile.Close()
		inFile.Close()
	}
	return nil
}
<commit_msg>remove debug print<commit_after>package qtpm

import (
	"archive/zip"
	"bytes"
	"errors"
	"fmt"
	"git"
	"github.com/fatih/color"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func Get(packageName string, update, useGit bool) {
	parentConfig, _ := LoadConfig(".", true)

	var packages []*PackageConfig
	var err error
	if packageName == "" && parentConfig != nil {
		packages, err = getRecursively(parentConfig, parentConfig.Requires, true, false, update, useGit)
	} else if packageName != "" {
		packages, err = getRecursively(parentConfig, []string{packageName}, true, parentConfig != nil, update, useGit)
	} else {
		fmt.Println("qtpackage.toml or package name argument are needed")
		return
	}
	if err != nil {
		log.Fatalln(err)
	}
	if parentConfig != nil {
		parentConfig.SaveIfDirty()
	}
	var dir string
	if parentConfig != nil {
		dir = parentConfig.Dir
	} else {
		dir, _ = filepath.Abs(".")
	}
	os.MkdirAll(filepath.Join(dir, "qtresources", "translations"), 0755)
	if len(packages) == 0 {
		color.Red("%s package has no dependent packages.\n", parentConfig.Name)
		os.Exit(1)
	}
	for _, packageConfig := range packages[:len(packages)-1] {
		_, err = BuildPackage(parentConfig, packageConfig, update, false, true, false)
		if err != nil {
			continue
		}
	}
}

func getRecursively(rootPackage *PackageConfig, packages []string, download, save, update, useGit bool) ([]*PackageConfig, error) {
	loadedPackages := make(map[string]*PackageConfig)
	if rootPackage != nil {
		rootPackage.Children = loadedPackages
	}
	var edges [][2]string
	for _, packageName := range packages {
		edges = append(edges, [2]string{packageName, "*"})
	}
	nextPackages := packages
	for len(nextPackages) > 0 {
		waitingPackage := []string{}
		for _, packageName := range nextPackages {
			child, err := getSinglePackage(rootPackage, packageName, download, save, update, useGit)
			if err != nil {
				return nil, err
			}
			loadedPackages[packageName] = child
			child.PackageName = packageName
			for _, neededPackage := range child.Requires {
				edges = append(edges, [2]string{neededPackage, packageName})
				if _, ok := loadedPackages[neededPackage]; !ok && !has(waitingPackage, neededPackage) {
					waitingPackage = append(waitingPackage, neededPackage)
				}
			}
		}
		nextPackages = waitingPackage
	}
	packageNamesInBuildOrder, loops := TopologicalSort(edges)
	if len(loops) > 0 {
		var buffer bytes.Buffer
		buffer.WriteString("Circular reference issue. The following dependencies are :\n")
		for _, loop := range loops {
			fmt.Fprintf(&buffer, "  * %s -> %s\n", loop[0], loop[1])
		}
		return nil, errors.New(buffer.String())
	}
	var result []*PackageConfig
	for _, packageName := range packageNamesInBuildOrder {
		if packageName == "*" {
			if rootPackage != nil {
				result = append(result, rootPackage)
			}
		} else {
			result = append(result, loadedPackages[packageName])
		}
	}
	return result, nil
}

func has(list []string, entry string) bool {
	for _, listItem := range list {
		if listItem == entry {
			return true
		}
	}
	return false
}

func getSinglePackage(rootConfig *PackageConfig, packageName string, download, save, update, useGit bool) (*PackageConfig, error) {
	// download from git
	paths := strings.Split(packageName, "/")
	if len(paths) != 3 {
		return nil, fmt.Errorf("qtpm supports only github.com/author/repository style path: %s", packageName)
	}
	var parentDir string
	if rootConfig != nil {
		parentDir = rootConfig.Dir
	} else {
		parentDir, _ = filepath.Abs(".")
	}
	dirName := strings.Join(paths, "___")
	installDir := filepath.Join(parentDir, "vendor", dirName)
	workDir := filepath.Dir(installDir)
	os.MkdirAll(workDir, 0755)

	if _, err := os.Stat(installDir); os.IsNotExist(err) {
		if !download {
			installDir = filepath.Clean(filepath.Join(parentDir, "..", dirName))
			workDir = filepath.Dir(installDir)
			if _, err := os.Stat(installDir); os.IsNotExist(err) {
				return nil, fmt.Errorf("%s is not downloaded yet. Run qtpm get first.", packageName)
			}
		}
		if useGit || paths[0] != "github.com" {
			err = git.CloneWithoutFilters([]string{
				"--depth", "1", fmt.Sprintf("git@%s:%s/%s.git", paths[0], paths[1], paths[2]), dirName,
			}, workDir)
		} else {
			err = DownloadZip(installDir, paths[1], paths[2])
		}
		if err != nil {
			return nil, err
		}
	} else if update {
		_, err := os.Stat(filepath.Join(installDir, ".git"))
		if os.IsNotExist(err) {
			os.RemoveAll(installDir)
			err = DownloadZip(installDir, paths[1], paths[2])
		} else {
			cmd := Command("get", installDir, "pull", "--ff-only")
			err = cmd.Run()
		}
		if err != nil {
			return nil, err
		}
	}
	if save && !has(rootConfig.Requires, packageName) {
		rootConfig.Requires = append(rootConfig.Requires, packageName)
		rootConfig.DirtyFlag = true
	}
	return LoadConfig(installDir, false)
}

func DownloadZip(installDir, orgName, repoName string) error {
	os.MkdirAll(installDir, 0755)
	resp, err := http.Get(fmt.Sprintf("https://codeload.github.com/%s/%s/zip/master", orgName, repoName))
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	rawReader := bytes.NewReader(body)
	reader, err := zip.NewReader(rawReader, int64(len(body)))
	if err != nil {
		return err
	}
	var prefixLength int
	for i, file := range reader.File {
		if i == 0 {
			prefixLength = len(file.Name)
			continue
		}
		path := filepath.Join(installDir, file.Name[prefixLength:])
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, 0755)
			continue
		}
		outFile, err := os.Create(path)
		if err != nil {
			return err
		}
		inFile, err := file.Open()
		if err != nil {
			outFile.Close()
			return err
		}
		io.Copy(outFile, inFile)
		outFile.Close()
		inFile.Close()
	}
	return nil
}
<|endoftext|>