<commit_before>//
// grump reader parses a country specific file and generate a config file of bodies
//
// for each cell of the country specifc file, this program generate bodies per cells according to
// the population count in the cell
//
// the arrangement of circle in each cell is taken from a outsite source (csq something) up to 200 circles
//
// usage grump-reader -country=xxx where xxx isthe 3 small letter ISO 3166 code for the country (for instance "fra")
//
package main

import "flag"
import "math"
// import "math/rand"
import "fmt"
import "os"
import "log"
import "bufio"
import "path/filepath"
import "github.com/thomaspeugeot/tkv/barnes-hut"
import "github.com/thomaspeugeot/tkv/quadtree"
import "github.com/thomaspeugeot/tkv/grump"

// store country code
type country struct {
	Name string
	NCols, NRows, XllCorner, YllCorner int
}

// coordinates of arrangement of circle packing in a square
type circleCoord struct {
	x,y float64
}

var targetMaxBodies = 400000

var maxCirclePerCell = 1000

// storage of circle arrangement per number of circle in the square
type arrangementsStore [][]circleCoord

//
// on the PC
//  go run grump-reader.go -tkvdata="C:\Users\peugeot\tkv-data"
func main() {

	// flag "country"
	countryPtr := flag.String("country","fra","iso 3166 country code")

	// get the directory containing tkv data through the flag "tkvdata"
	dirTKVDataPtr := flag.String("tkvdata","/Users/thomaspeugeot/the-mapping-data/","directory containing input tkv data")

	var country country

	flag.Parse()
	grump.Info.Println( "country to parse", *countryPtr)
	country.Name = *countryPtr
	fmt.Println( "directory containing tkv data", *dirTKVDataPtr)
	dirTKVData := *dirTKVDataPtr

	// create the path to the agragate country count
	grumpFilePath := fmt.Sprintf( "%s/%s_grumpv1_pcount_00_ascii_30/%sup00ag.asc", dirTKVData, *countryPtr, *countryPtr )
	fmt.Println("relative path ", filepath.Clean( grumpFilePath))
	var grumpFile *os.File
	var err error
	grumpFile, err = os.Open( filepath.Clean( grumpFilePath))
	if err != nil {
		log.Fatal(err)
	}

	// parse the grump
	var word int
	scanner := bufio.NewScanner( grumpFile)
	scanner.Split(bufio.ScanWords)

	// scan 8 first lines
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NCols)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NRows)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.XllCorner)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.YllCorner)

	fmt.Println( country )

	// scan the reamining header
	for word < 4 {
		scanner.Scan()
		word++
		fmt.Println( fmt.Sprintf("item %d : %s", word, scanner.Text()))
	}
	rowLatWidth := 0.0083333333333
	colLngWidth := 0.0083333333333

	// prepare the count matrix
	countMatrix := make([]float64, country.NRows * country.NCols)

	popTotal := 0.0
	// scan the file and store result in countMatrix
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			scanner.Scan()
			// lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			var count float64
			fmt.Sscanf( scanner.Text(), "%f", &count)
			popTotal += count

			countMatrix[ (country.NRows-row-1)*country.NCols + col ] = count
		}
		fmt.Printf("\rrow %5d lat %2.3f total %f", row, lat, popTotal)
	}
	fmt.Println("")
	grumpFile.Close()

	// get the arrangement
	arrangements := make( arrangementsStore, maxCirclePerCell)
	for nbCircles := 1; nbCircles < maxCirclePerCell; nbCircles++ {

		fmt.Printf("\rgetting arrangement for %3d circles", nbCircles)

		arrangements[nbCircles] = make( []circleCoord, nbCircles)

		// open the reference file
		circlePackingFilePath := fmt.Sprintf( "%s/csq_coords/csq%d.txt", dirTKVData, nbCircles )
		var circlePackingFile *os.File
		var errCirclePackingFile error
		circlePackingFile, errCirclePackingFile = os.Open( filepath.Clean( circlePackingFilePath))
		if errCirclePackingFile != nil {
			log.Fatal(err)
		}

		// prepare scanner
		scannerCircle := bufio.NewScanner( circlePackingFile)
		scannerCircle.Split(bufio.ScanWords)

		// one line per circle
		for circle := 0; circle < nbCircles; circle++ {

			// scan the id of the circle
			scannerCircle.Scan();

			// scan X coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].x))
			// scan Y coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].y))
			// fmt.Printf("getting arrangement for %d circle %f %f\n", nbCircles, arrangements[nbCircles][circle].x, arrangements[nbCircles][circle].y)
		}
		circlePackingFile.Close()
	}

	// prepare the output density file
	var bodies []quadtree.Body
	bodiesInCellMax := 0

	grump.Info.Printf("Preparing the ouput")
	cumulativePopTotal := 0.0
	bodiesNb :=0
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			// compute relative coordinate of the cell
			relX := (lng - float64(country.XllCorner)) / (float64(country.NCols) * colLngWidth)
			relY := (lat - float64(country.YllCorner)) / (float64(country.NRows) * rowLatWidth)

			// fetch count of the cell
			count := countMatrix[ row*country.NCols + col ]

			// how many bodies ? it is maxBodies *( count / country.PCount)
			bodiesInCell := int( math.Floor( float64( targetMaxBodies) * (count/popTotal)))
			if bodiesInCell > bodiesInCellMax { bodiesInCellMax = bodiesInCell}

			// initiate the bodies
			for i :=0; i<bodiesInCell; i++ {
				var body quadtree.Body
				// angle := float64(i) * 2.0 * math.Pi / float64(bodiesInCell)
				body.X = relX + (1.0/float64(country.NCols))*(0.5 + arrangements[bodiesInCell][i].x)
				body.Y = relY + (1.0/float64(country.NRows))*(0.5 + arrangements[bodiesInCell][i].y)
				// body.M = count/float64(bodiesInCell)
				body.M = float64(targetMaxBodies)
				bodies = append( bodies,  body)
			}
			cumulativePopTotal += count
			bodiesNb += bodiesInCell
		}
	}

	// var quadtree quadtree.Quadtree
	// quadtree.Init( &bodies)
	fmt.Println("bodies in cell max ", bodiesInCellMax)
	fmt.Println("cumulative pop ", cumulativePopTotal)
	fmt.Println("nb of bodies ", bodiesNb)

	var run barnes_hut.Run
	run.Init( & bodies)
	run.SetCountry( country.Name)

	run.CaptureConfig()
}<commit_msg>solved haiti issue<commit_after>//
// grump reader parses a country specific file and generate a config file of bodies
//
// for each cell of the country specifc file, this program generate bodies per cells according to
// the population count in the cell
//
// the arrangement of circle in each cell is taken from a outsite source (csq something) up to 200 circles
//
// usage grump-reader -country=xxx where xxx isthe 3 small letter ISO 3166 code for the country (for instance "fra")
//
package main

import "flag"
import "math"
// import "math/rand"
import "fmt"
import "os"
import "log"
import "bufio"
import "path/filepath"
import "github.com/thomaspeugeot/tkv/barnes-hut"
import "github.com/thomaspeugeot/tkv/quadtree"
import "github.com/thomaspeugeot/tkv/grump"

// store country code
type country struct {
	Name string
	NCols, NRows, XllCorner, YllCorner int
}

// coordinates of arrangement of circle packing in a square
type circleCoord struct {
	x,y float64
}

var targetMaxBodies = 400000

var maxCirclePerCell = 750

// storage of circle arrangement per number of circle in the square
type arrangementsStore [][]circleCoord

//
// on the PC
//  go run grump-reader.go -tkvdata="C:\Users\peugeot\tkv-data"
func main() {

	// flag "country"
	countryPtr := flag.String("country","fra","iso 3166 country code")

	// get the directory containing tkv data through the flag "tkvdata"
	dirTKVDataPtr := flag.String("tkvdata","/Users/thomaspeugeot/the-mapping-data/","directory containing input tkv data")

	var country country

	flag.Parse()
	grump.Info.Printf( "country to parse %s", *countryPtr)
	country.Name = *countryPtr
	grump.Info.Printf("directory containing tkv data %s", *dirTKVDataPtr)
	dirTKVData := *dirTKVDataPtr

	// create the path to the agragate country count
	grumpFilePath := fmt.Sprintf( "%s/%s_grumpv1_pcount_00_ascii_30/%sup00ag.asc", dirTKVData, *countryPtr, *countryPtr )
	grump.Info.Printf("relative path %s", filepath.Clean( grumpFilePath))
	var grumpFile *os.File
	var err error
	grumpFile, err = os.Open( filepath.Clean( grumpFilePath))
	if err != nil {
		log.Fatal(err)
	}

	// parse the grump
	var word int
	scanner := bufio.NewScanner( grumpFile)
	scanner.Split(bufio.ScanWords)

	// scan 8 first lines
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NCols)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NRows)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.XllCorner)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.YllCorner)

	grump.Info.Println("country struct content is ", country )

	// scan the reamining header
	for word < 4 {
		scanner.Scan()
		word++
		fmt.Println( fmt.Sprintf("item %d : %s", word, scanner.Text()))
	}
	rowLatWidth := 0.0083333333333
	colLngWidth := 0.0083333333333

	// prepare the count matrix
	countMatrix := make([]float64, country.NRows * country.NCols)

	popTotal := 0.0
	// scan the file and store result in countMatrix
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			scanner.Scan()
			// lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			var count float64
			fmt.Sscanf( scanner.Text(), "%f", &count)
			popTotal += count

			countMatrix[ (country.NRows-row-1)*country.NCols + col ] = count

		}
		fmt.Printf("\rrow %5d lat %2.3f total %f", row, lat, popTotal)
	}
	fmt.Printf("\n")
	grump.Info.Printf("reading grump file is over, closing")
	grumpFile.Close()

	// get the arrangement
	arrangements := make( arrangementsStore, maxCirclePerCell)
	for nbCircles := 1; nbCircles < maxCirclePerCell; nbCircles++ {

		fmt.Printf("\rgetting arrangement for %3d circles", nbCircles)

		arrangements[nbCircles] = make( []circleCoord, nbCircles)

		// open the reference file
		circlePackingFilePath := fmt.Sprintf( "%s/csq_coords/csq%d.txt", dirTKVData, nbCircles )
		var circlePackingFile *os.File
		var errCirclePackingFile error
		circlePackingFile, errCirclePackingFile = os.Open( filepath.Clean( circlePackingFilePath))
		if errCirclePackingFile != nil {
			log.Fatal(err)
		}

		// prepare scanner
		scannerCircle := bufio.NewScanner( circlePackingFile)
		scannerCircle.Split(bufio.ScanWords)

		// one line per circle
		for circle := 0; circle < nbCircles; circle++ {

			// scan the id of the circle
			scannerCircle.Scan();

			// scan X coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].x))
			// scan Y coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].y))
			// fmt.Printf("getting arrangement for %d circle %3d, coord %f %f\n", nbCircles, circle, arrangements[nbCircles][circle].x, arrangements[nbCircles][circle].y)
		}
		circlePackingFile.Close()
	}
	grump.Info.Printf("reading circle packing files is over")

	// prepare the output density file
	var bodies []quadtree.Body
	bodiesInCellMax := 0

	grump.Info.Printf("Preparing the ouput")
	cumulativePopTotal := 0.0
	bodiesNb :=0
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			// compute relative coordinate of the cell
			relX := (lng - float64(country.XllCorner)) / (float64(country.NCols) * colLngWidth)
			relY := (lat - float64(country.YllCorner)) / (float64(country.NRows) * rowLatWidth)

			// fetch count of the cell
			count := countMatrix[ row*country.NCols + col ]

			// how many bodies ? it is maxBodies *( count / country.PCount)
			bodiesInCell := int( math.Floor( float64( targetMaxBodies) * (count/popTotal)))
			if bodiesInCell > bodiesInCellMax { bodiesInCellMax = bodiesInCell}

			if (bodiesInCell > maxCirclePerCell ) {
				grump.Error.Printf("bodiesInCell %d superior to maxCirclePerCell %d", bodiesInCell, maxCirclePerCell)
			}

			// initiate the bodies
			for i :=0; i<bodiesInCell; i++ {
				var body quadtree.Body
				// angle := float64(i) * 2.0 * math.Pi / float64(bodiesInCell)
				body.X = relX + (1.0/float64(country.NCols))*(0.5 + arrangements[bodiesInCell][i].x)
				body.Y = relY + (1.0/float64(country.NRows))*(0.5 + arrangements[bodiesInCell][i].y)
				// body.M = count/float64(bodiesInCell)
				body.M = float64(targetMaxBodies)
				bodies = append( bodies,  body)
			}
			cumulativePopTotal += count
			bodiesNb += bodiesInCell
		}
	}

	// var quadtree quadtree.Quadtree
	// quadtree.Init( &bodies)
	fmt.Println("bodies in cell max ", bodiesInCellMax)
	fmt.Println("cumulative pop ", cumulativePopTotal)
	fmt.Println("nb of bodies ", bodiesNb)

	var run barnes_hut.Run
	run.Init( & bodies)
	run.SetCountry( country.Name)

	run.CaptureConfig()
}<|endoftext|>


<commit_before>package slack

import (
	"fmt"
	"time"
)

// XXX: Need to implement
type UserPrefs struct {
	// "highlight_words":"",
	// "user_colors":"",
	// "color_names_in_list":true,
	// "growls_enabled":true,
	// "tz":"Europe\/London",
	// "push_dm_alert":true,
	// "push_mention_alert":true,
	// "push_everything":true,
	// "push_idle_wait":2,
	// "push_sound":"b2.mp3",
	// "push_loud_channels":"",
	// "push_mention_channels":"",
	// "push_loud_channels_set":"",
	// "email_alerts":"instant",
	// "email_alerts_sleep_until":0,
	// "email_misc":false,
	// "email_weekly":true,
	// "welcome_message_hidden":false,
	// "all_channels_loud":true,
	// "loud_channels":"",
	// "never_channels":"",
	// "loud_channels_set":"",
	// "show_member_presence":true,
	// "search_sort":"timestamp",
	// "expand_inline_imgs":true,
	// "expand_internal_inline_imgs":true,
	// "expand_snippets":false,
	// "posts_formatting_guide":true,
	// "seen_welcome_2":true,
	// "seen_ssb_prompt":false,
	// "search_only_my_channels":false,
	// "emoji_mode":"default",
	// "has_invited":true,
	// "has_uploaded":false,
	// "has_created_channel":true,
	// "search_exclude_channels":"",
	// "messages_theme":"default",
	// "webapp_spellcheck":true,
	// "no_joined_overlays":false,
	// "no_created_overlays":true,
	// "dropbox_enabled":false,
	// "seen_user_menu_tip_card":true,
	// "seen_team_menu_tip_card":true,
	// "seen_channel_menu_tip_card":true,
	// "seen_message_input_tip_card":true,
	// "seen_channels_tip_card":true,
	// "seen_domain_invite_reminder":false,
	// "seen_member_invite_reminder":false,
	// "seen_flexpane_tip_card":true,
	// "seen_search_input_tip_card":true,
	// "mute_sounds":false,
	// "arrow_history":false,
	// "tab_ui_return_selects":true,
	// "obey_inline_img_limit":true,
	// "new_msg_snd":"knock_brush.mp3",
	// "collapsible":false,
	// "collapsible_by_click":true,
	// "require_at":false,
	// "mac_ssb_bounce":"",
	// "mac_ssb_bullet":true,
	// "win_ssb_bullet":true,
	// "expand_non_media_attachments":true,
	// "show_typing":true,
	// "pagekeys_handled":true,
	// "last_snippet_type":"",
	// "display_real_names_override":0,
	// "time24":false,
	// "enter_is_special_in_tbt":false,
	// "graphic_emoticons":false,
	// "convert_emoticons":true,
	// "autoplay_chat_sounds":true,
	// "ss_emojis":true,
	// "sidebar_behavior":"",
	// "mark_msgs_read_immediately":true,
	// "start_scroll_at_oldest":true,
	// "snippet_editor_wrap_long_lines":false,
	// "ls_disabled":false,
	// "sidebar_theme":"default",
	// "sidebar_theme_custom_values":"",
	// "f_key_search":false,
	// "k_key_omnibox":true,
	// "speak_growls":false,
	// "mac_speak_voice":"com.apple.speech.synthesis.voice.Alex",
	// "mac_speak_speed":250,
	// "comma_key_prefs":false,
	// "at_channel_suppressed_channels":"",
	// "push_at_channel_suppressed_channels":"",
	// "prompted_for_email_disabling":false,
	// "full_text_extracts":false,
	// "no_text_in_notifications":false,
	// "muted_channels":"",
	// "no_macssb1_banner":false,
	// "privacy_policy_seen":true,
	// "search_exclude_bots":false,
	// "fuzzy_matching":false
}

// UserDetails contains user details coming in the initial response from StartRTM
type UserDetails struct {
	Id             string    `json:"id"`
	Name           string    `json:"name"`
	Created        JSONTime  `json:"created"`
	ManualPresence string    `json:"manual_presence"`
	Prefs          UserPrefs `json:"prefs"`
}

// JSONTime exists so that we can have a String method converting the date
type JSONTime int64

// String converts the unix timestamp into a string
func (t JSONTime) String() string {
	tm := time.Unix(int64(t), 0)
	return fmt.Sprintf("\"%s\"", tm.Format("Mon Jan _2"))
}

// Team contains details about a team
type Team struct {
	Id     string `json:"id"`
	Name   string `json:"name"`
	Domain string `json:"name"`
}

// Icons XXX: needs further investigation
type Icons struct {
	Image48 string `json:"image_48"`
}

// Bot contains information about a bot
type Bot struct {
	Id      string `json:"id"`
	Name    string `json:"name"`
	Deleted bool   `json:"deleted"`
	Icons   Icons  `json:"icons"`
}

// Info contains various details about Users, Channels, Bots and the authenticated user
// It is returned by StartRTM
type Info struct {
	Url      string       `json:"url,omitempty"`
	User     *UserDetails `json:"self,omitempty"`
	Team     *Team        `json:"team,omitempty"`
	Users    []User       `json:"users,omitempty"`
	Channels []Channel    `json:"channels,omitempty"`
	Groups   []Group      `json:"groups,omitempty"`
	Bots     []Bot        `json:"bots,omitempty"`
	IMs      []IM         `json:"ims,omitempty"`
}

type infoResponseFull struct {
	Info
	SlackWSResponse
}

// GetBotById returns a bot given a bot id
func (info Info) GetBotById(botId string) *Bot {
	for _, bot := range info.Bots {
		if bot.Id == botId {
			return &bot
		}
	}
	return nil
}

// GetUserById returns a user given a user id
func (info Info) GetUserById(userId string) *User {
	for _, user := range info.Users {
		if user.Id == userId {
			return &user
		}
	}
	return nil
}

// GetChannelById returns a channel given a channel id
func (info Info) GetChannelById(channelId string) *Channel {
	for _, channel := range info.Channels {
		if channel.Id == channelId {
			return &channel
		}
	}
	return nil
}
<commit_msg>Added info.GetGroupById method.<commit_after>package slack

import (
	"fmt"
	"time"
)

// XXX: Need to implement
type UserPrefs struct {
	// "highlight_words":"",
	// "user_colors":"",
	// "color_names_in_list":true,
	// "growls_enabled":true,
	// "tz":"Europe\/London",
	// "push_dm_alert":true,
	// "push_mention_alert":true,
	// "push_everything":true,
	// "push_idle_wait":2,
	// "push_sound":"b2.mp3",
	// "push_loud_channels":"",
	// "push_mention_channels":"",
	// "push_loud_channels_set":"",
	// "email_alerts":"instant",
	// "email_alerts_sleep_until":0,
	// "email_misc":false,
	// "email_weekly":true,
	// "welcome_message_hidden":false,
	// "all_channels_loud":true,
	// "loud_channels":"",
	// "never_channels":"",
	// "loud_channels_set":"",
	// "show_member_presence":true,
	// "search_sort":"timestamp",
	// "expand_inline_imgs":true,
	// "expand_internal_inline_imgs":true,
	// "expand_snippets":false,
	// "posts_formatting_guide":true,
	// "seen_welcome_2":true,
	// "seen_ssb_prompt":false,
	// "search_only_my_channels":false,
	// "emoji_mode":"default",
	// "has_invited":true,
	// "has_uploaded":false,
	// "has_created_channel":true,
	// "search_exclude_channels":"",
	// "messages_theme":"default",
	// "webapp_spellcheck":true,
	// "no_joined_overlays":false,
	// "no_created_overlays":true,
	// "dropbox_enabled":false,
	// "seen_user_menu_tip_card":true,
	// "seen_team_menu_tip_card":true,
	// "seen_channel_menu_tip_card":true,
	// "seen_message_input_tip_card":true,
	// "seen_channels_tip_card":true,
	// "seen_domain_invite_reminder":false,
	// "seen_member_invite_reminder":false,
	// "seen_flexpane_tip_card":true,
	// "seen_search_input_tip_card":true,
	// "mute_sounds":false,
	// "arrow_history":false,
	// "tab_ui_return_selects":true,
	// "obey_inline_img_limit":true,
	// "new_msg_snd":"knock_brush.mp3",
	// "collapsible":false,
	// "collapsible_by_click":true,
	// "require_at":false,
	// "mac_ssb_bounce":"",
	// "mac_ssb_bullet":true,
	// "win_ssb_bullet":true,
	// "expand_non_media_attachments":true,
	// "show_typing":true,
	// "pagekeys_handled":true,
	// "last_snippet_type":"",
	// "display_real_names_override":0,
	// "time24":false,
	// "enter_is_special_in_tbt":false,
	// "graphic_emoticons":false,
	// "convert_emoticons":true,
	// "autoplay_chat_sounds":true,
	// "ss_emojis":true,
	// "sidebar_behavior":"",
	// "mark_msgs_read_immediately":true,
	// "start_scroll_at_oldest":true,
	// "snippet_editor_wrap_long_lines":false,
	// "ls_disabled":false,
	// "sidebar_theme":"default",
	// "sidebar_theme_custom_values":"",
	// "f_key_search":false,
	// "k_key_omnibox":true,
	// "speak_growls":false,
	// "mac_speak_voice":"com.apple.speech.synthesis.voice.Alex",
	// "mac_speak_speed":250,
	// "comma_key_prefs":false,
	// "at_channel_suppressed_channels":"",
	// "push_at_channel_suppressed_channels":"",
	// "prompted_for_email_disabling":false,
	// "full_text_extracts":false,
	// "no_text_in_notifications":false,
	// "muted_channels":"",
	// "no_macssb1_banner":false,
	// "privacy_policy_seen":true,
	// "search_exclude_bots":false,
	// "fuzzy_matching":false
}

// UserDetails contains user details coming in the initial response from StartRTM
type UserDetails struct {
	Id             string    `json:"id"`
	Name           string    `json:"name"`
	Created        JSONTime  `json:"created"`
	ManualPresence string    `json:"manual_presence"`
	Prefs          UserPrefs `json:"prefs"`
}

// JSONTime exists so that we can have a String method converting the date
type JSONTime int64

// String converts the unix timestamp into a string
func (t JSONTime) String() string {
	tm := time.Unix(int64(t), 0)
	return fmt.Sprintf("\"%s\"", tm.Format("Mon Jan _2"))
}

// Team contains details about a team
type Team struct {
	Id     string `json:"id"`
	Name   string `json:"name"`
	Domain string `json:"name"`
}

// Icons XXX: needs further investigation
type Icons struct {
	Image48 string `json:"image_48"`
}

// Bot contains information about a bot
type Bot struct {
	Id      string `json:"id"`
	Name    string `json:"name"`
	Deleted bool   `json:"deleted"`
	Icons   Icons  `json:"icons"`
}

// Info contains various details about Users, Channels, Bots and the authenticated user
// It is returned by StartRTM
type Info struct {
	Url      string       `json:"url,omitempty"`
	User     *UserDetails `json:"self,omitempty"`
	Team     *Team        `json:"team,omitempty"`
	Users    []User       `json:"users,omitempty"`
	Channels []Channel    `json:"channels,omitempty"`
	Groups   []Group      `json:"groups,omitempty"`
	Bots     []Bot        `json:"bots,omitempty"`
	IMs      []IM         `json:"ims,omitempty"`
}

type infoResponseFull struct {
	Info
	SlackWSResponse
}

// GetBotById returns a bot given a bot id
func (info Info) GetBotById(botId string) *Bot {
	for _, bot := range info.Bots {
		if bot.Id == botId {
			return &bot
		}
	}
	return nil
}

// GetUserById returns a user given a user id
func (info Info) GetUserById(userId string) *User {
	for _, user := range info.Users {
		if user.Id == userId {
			return &user
		}
	}
	return nil
}

// GetChannelById returns a channel given a channel id
func (info Info) GetChannelById(channelId string) *Channel {
	for _, channel := range info.Channels {
		if channel.Id == channelId {
			return &channel
		}
	}
	return nil
}

// GetGroupById returns a group given a group id
func (info Info) GetGroupById(groupId string) *Group {
	for _, group := range info.Groups {
		if group.Id == groupId {
			return &group
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"bytes"
	"context"
	"fmt"
	"log"
	"math"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"github.com/hashicorp/packer/helper/enumflag"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template"
	"golang.org/x/sync/semaphore"

	"github.com/posener/complete"
)

type BuildCommand struct {
	Meta
}

func (c *BuildCommand) Run(args []string) int {
	var cfgColor, cfgDebug, cfgForce, cfgTimestamp, cfgParallel bool
	var cfgParallelBuilds int64
	var cfgOnError string
	flags := c.Meta.FlagSet("build", FlagSetBuildFilter|FlagSetVars)
	flags.Usage = func() { c.Ui.Say(c.Help()) }
	flags.BoolVar(&cfgColor, "color", true, "")
	flags.BoolVar(&cfgDebug, "debug", false, "")
	flags.BoolVar(&cfgForce, "force", false, "")
	flags.BoolVar(&cfgTimestamp, "timestamp-ui", false, "")
	flagOnError := enumflag.New(&cfgOnError, "cleanup", "abort", "ask")
	flags.Var(flagOnError, "on-error", "")
	flags.BoolVar(&cfgParallel, "parallel", true, "")
	flags.Int64Var(&cfgParallelBuilds, "parallel-builds", 0, "")
	if err := flags.Parse(args); err != nil {
		return 1
	}

	args = flags.Args()
	if len(args) != 1 {
		flags.Usage()
		return 1
	}

	// Parse the template
	var tpl *template.Template
	var err error
	tpl, err = template.ParseFile(args[0])
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to parse template: %s", err))
		return 1
	}

	// Get the core
	core, err := c.Meta.Core(tpl)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	// Get the builds we care about
	buildNames := c.Meta.BuildNames(core)
	builds := make([]packer.Build, 0, len(buildNames))
	for _, n := range buildNames {
		b, err := core.Build(n)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Failed to initialize build '%s': %s",
				n, err))
			continue
		}

		builds = append(builds, b)
	}

	if cfgDebug {
		c.Ui.Say("Debug mode enabled. Builds will not be parallelized.")
	}

	// Compile all the UIs for the builds
	colors := [5]packer.UiColor{
		packer.UiColorGreen,
		packer.UiColorCyan,
		packer.UiColorMagenta,
		packer.UiColorYellow,
		packer.UiColorBlue,
	}
	buildUis := make(map[string]packer.Ui)
	for i, b := range buildNames {
		var ui packer.Ui
		ui = c.Ui
		if cfgColor {
			ui = &packer.ColoredUi{
				Color: colors[i%len(colors)],
				Ui:    ui,
			}
			if _, ok := c.Ui.(*packer.MachineReadableUi); !ok {
				ui.Say(fmt.Sprintf("%s output will be in this color.", b))
				if i+1 == len(buildNames) {
					// Add a newline between the color output and the actual output
					c.Ui.Say("")
				}
				// Now add timestamps if requested
				if cfgTimestamp {
					ui = &packer.TimestampedUi{
						Ui: ui,
					}
				}
			}
		}

		buildUis[b] = ui
	}

	log.Printf("Build debug mode: %v", cfgDebug)
	log.Printf("Force build: %v", cfgForce)
	log.Printf("On error: %v", cfgOnError)

	// Set the debug and force mode and prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s", b.Name())
		b.SetDebug(cfgDebug)
		b.SetForce(cfgForce)
		b.SetOnError(cfgOnError)

		warnings, err := b.Prepare()
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
		if len(warnings) > 0 {
			ui := buildUis[b.Name()]
			ui.Say(fmt.Sprintf("Warnings for build '%s':\n", b.Name()))
			for _, warning := range warnings {
				ui.Say(fmt.Sprintf("* %s", warning))
			}
			ui.Say("")
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var interruptWg, wg sync.WaitGroup
	interrupted := false
	var artifacts = struct {
		sync.RWMutex
		m map[string][]packer.Artifact
	}{m: make(map[string][]packer.Artifact)}
	errors := make(map[string]error)
	// ctx := context.Background()
	if cfgParallel == false {
		cfgParallelBuilds = 1
	}
	if cfgParallelBuilds < 1 {
		cfgParallelBuilds = math.MaxInt64
	}

	buildCtx, cancelCtx := context.WithCancel(context.Background())
	// Handle interrupts for this build
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
	defer signal.Stop(sigCh)
	go func() {
		sig := <-sigCh
		interruptWg.Add(1)
		defer interruptWg.Done()
		interrupted = true

		cancelCtx()
		c.Ui.Error(fmt.Sprintf("Cancelling build after receiving %s", sig))
	}()

	limitParallel := semaphore.NewWeighted(cfgParallelBuilds)
	for i := range builds {
		b := builds[i]
		name := b.Name()
		ui := buildUis[name]
		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)
		if err := limitParallel.Acquire(buildCtx, 1); err != nil {
			ui.Error(fmt.Sprintf("Build '%s' failed to acquire semaphore: %s", name, err))
			errors[name] = err
			break
		}

		// Run the build in a goroutine
		go func() {
			defer wg.Done()

			defer limitParallel.Release(1)

			log.Printf("Starting build run: %s", name)
			runArtifacts, err := b.Run(buildCtx, ui)

			if err != nil {
				ui.Error(fmt.Sprintf("Build '%s' errored: %s", name, err))
				errors[name] = err
			} else {
				ui.Say(fmt.Sprintf("Build '%s' finished.", name))
				artifacts.Lock()
				artifacts.m[name] = runArtifacts
				artifacts.Unlock()
			}
		}()

		if cfgDebug {
			log.Printf("Debug enabled, so waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if cfgParallelBuilds == 1 {
			log.Printf("Parallelization disabled, waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if interrupted {
			log.Println("Interrupted, not going to start any more builds.")
			break
		}
	}

	// Wait for both the builds to complete and the interrupt handler,
	// if it is interrupted.
	log.Printf("Waiting on builds to complete...")
	wg.Wait()

	log.Printf("Builds completed. Waiting on interrupt barrier...")
	interruptWg.Wait()

	if interrupted {
		c.Ui.Say("Cleanly cancelled builds after being interrupted.")
		return 1
	}

	if len(errors) > 0 {
		c.Ui.Machine("error-count", strconv.FormatInt(int64(len(errors)), 10))

		c.Ui.Error("\n==> Some builds didn't complete successfully and had errors:")
		for name, err := range errors {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			ui.Machine("error", err.Error())

			c.Ui.Error(fmt.Sprintf("--> %s: %s", name, err))
		}
	}

	if len(artifacts.m) > 0 {
		c.Ui.Say("\n==> Builds finished. The artifacts of successful builds are:")
		for name, buildArtifacts := range artifacts.m {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			// Machine-readable helpful
			ui.Machine("artifact-count", strconv.FormatInt(int64(len(buildArtifacts)), 10))

			for i, artifact := range buildArtifacts {
				var message bytes.Buffer
				fmt.Fprintf(&message, "--> %s: ", name)

				if artifact != nil {
					fmt.Fprint(&message, artifact.String())
				} else {
					fmt.Fprint(&message, "<nothing>")
				}

				iStr := strconv.FormatInt(int64(i), 10)
				if artifact != nil {
					ui.Machine("artifact", iStr, "builder-id", artifact.BuilderId())
					ui.Machine("artifact", iStr, "id", artifact.Id())
					ui.Machine("artifact", iStr, "string", artifact.String())

					files := artifact.Files()
					ui.Machine("artifact",
						iStr,
						"files-count", strconv.FormatInt(int64(len(files)), 10))
					for fi, file := range files {
						fiStr := strconv.FormatInt(int64(fi), 10)
						ui.Machine("artifact", iStr, "file", fiStr, file)
					}
				} else {
					ui.Machine("artifact", iStr, "nil")
				}

				ui.Machine("artifact", iStr, "end")
				c.Ui.Say(message.String())
			}
		}
	} else {
		c.Ui.Say("\n==> Builds finished but no artifacts were created.")
	}

	if len(errors) > 0 {
		// If any errors occurred, exit with a non-zero exit status
		return 1
	}

	return 0
}

func (*BuildCommand) Help() string {
	helpText := `
Usage: packer build [options] TEMPLATE

  Will execute multiple builds in parallel as defined in the template.
  The various artifacts created by the template will be outputted.

Options:

  -color=false                  Disable color output. (Default: color)
  -debug                        Debug mode enabled for builds.
  -except=foo,bar,baz           Run all builds and post-procesors other than these.
  -only=foo,bar,baz             Build only the specified builds.
  -force                        Force a build to continue if artifacts exist, deletes existing artifacts.
  -machine-readable             Produce machine-readable output.
  -on-error=[cleanup|abort|ask] If the build fails do: clean up (default), abort, or ask.
  -parallel=false               Disable parallelization. (Default: true)
  -parallel-builds=1            Number of builds to run in parallel. 0 means no limit (Default: 0)
  -timestamp-ui                 Enable prefixing of each ui output with an RFC3339 timestamp.
  -var 'key=value'              Variable for templates, can be used multiple times.
  -var-file=path                JSON file containing user variables.
`

	return strings.TrimSpace(helpText)
}

func (*BuildCommand) Synopsis() string {
	return "build image(s) from template"
}

func (*BuildCommand) AutocompleteArgs() complete.Predictor {
	return complete.PredictNothing
}

func (*BuildCommand) AutocompleteFlags() complete.Flags {
	return complete.Flags{
		"-color":            complete.PredictNothing,
		"-debug":            complete.PredictNothing,
		"-except":           complete.PredictNothing,
		"-only":             complete.PredictNothing,
		"-force":            complete.PredictNothing,
		"-machine-readable": complete.PredictNothing,
		"-on-error":         complete.PredictNothing,
		"-parallel":         complete.PredictNothing,
		"-timestamp-ui":     complete.PredictNothing,
		"-var":              complete.PredictNothing,
		"-var-file":         complete.PredictNothing,
	}
}
<commit_msg>be true to docs<commit_after>package command

import (
	"bytes"
	"context"
	"fmt"
	"log"
	"math"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"github.com/hashicorp/packer/helper/enumflag"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template"
	"golang.org/x/sync/semaphore"

	"github.com/posener/complete"
)

type BuildCommand struct {
	Meta
}

func (c *BuildCommand) Run(args []string) int {
	var cfgColor, cfgDebug, cfgForce, cfgTimestamp, cfgParallel bool
	var cfgParallelBuilds int64
	var cfgOnError string
	flags := c.Meta.FlagSet("build", FlagSetBuildFilter|FlagSetVars)
	flags.Usage = func() { c.Ui.Say(c.Help()) }
	flags.BoolVar(&cfgColor, "color", true, "")
	flags.BoolVar(&cfgDebug, "debug", false, "")
	flags.BoolVar(&cfgForce, "force", false, "")
	flags.BoolVar(&cfgTimestamp, "timestamp-ui", false, "")
	flagOnError := enumflag.New(&cfgOnError, "cleanup", "abort", "ask")
	flags.Var(flagOnError, "on-error", "")
	flags.BoolVar(&cfgParallel, "parallel", true, "")
	flags.Int64Var(&cfgParallelBuilds, "parallel-builds", 0, "")
	if err := flags.Parse(args); err != nil {
		return 1
	}

	args = flags.Args()
	if len(args) != 1 {
		flags.Usage()
		return 1
	}

	// Parse the template
	var tpl *template.Template
	var err error
	tpl, err = template.ParseFile(args[0])
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to parse template: %s", err))
		return 1
	}

	// Get the core
	core, err := c.Meta.Core(tpl)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	// Get the builds we care about
	buildNames := c.Meta.BuildNames(core)
	builds := make([]packer.Build, 0, len(buildNames))
	for _, n := range buildNames {
		b, err := core.Build(n)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Failed to initialize build '%s': %s",
				n, err))
			continue
		}

		builds = append(builds, b)
	}

	if cfgDebug {
		c.Ui.Say("Debug mode enabled. Builds will not be parallelized.")
	}

	// Compile all the UIs for the builds
	colors := [5]packer.UiColor{
		packer.UiColorGreen,
		packer.UiColorCyan,
		packer.UiColorMagenta,
		packer.UiColorYellow,
		packer.UiColorBlue,
	}
	buildUis := make(map[string]packer.Ui)
	for i, b := range buildNames {
		var ui packer.Ui
		ui = c.Ui
		if cfgColor {
			ui = &packer.ColoredUi{
				Color: colors[i%len(colors)],
				Ui:    ui,
			}
			if _, ok := c.Ui.(*packer.MachineReadableUi); !ok {
				ui.Say(fmt.Sprintf("%s output will be in this color.", b))
				if i+1 == len(buildNames) {
					// Add a newline between the color output and the actual output
					c.Ui.Say("")
				}
				// Now add timestamps if requested
				if cfgTimestamp {
					ui = &packer.TimestampedUi{
						Ui: ui,
					}
				}
			}
		}

		buildUis[b] = ui
	}

	log.Printf("Build debug mode: %v", cfgDebug)
	log.Printf("Force build: %v", cfgForce)
	log.Printf("On error: %v", cfgOnError)

	// Set the debug and force mode and prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s", b.Name())
		b.SetDebug(cfgDebug)
		b.SetForce(cfgForce)
		b.SetOnError(cfgOnError)

		warnings, err := b.Prepare()
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
		if len(warnings) > 0 {
			ui := buildUis[b.Name()]
			ui.Say(fmt.Sprintf("Warnings for build '%s':\n", b.Name()))
			for _, warning := range warnings {
				ui.Say(fmt.Sprintf("* %s", warning))
			}
			ui.Say("")
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var interruptWg, wg sync.WaitGroup
	interrupted := false
	var artifacts = struct {
		sync.RWMutex
		m map[string][]packer.Artifact
	}{m: make(map[string][]packer.Artifact)}
	errors := make(map[string]error)
	if cfgParallelBuilds < 1 {
		cfgParallelBuilds = math.MaxInt64
	}
	if cfgParallel == false && cfgParallelBuilds == 0 {
		cfgParallelBuilds = 1
	}

	buildCtx, cancelCtx := context.WithCancel(context.Background())
	// Handle interrupts for this build
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
	defer signal.Stop(sigCh)
	go func() {
		sig := <-sigCh
		interruptWg.Add(1)
		defer interruptWg.Done()
		interrupted = true

		cancelCtx()
		c.Ui.Error(fmt.Sprintf("Cancelling build after receiving %s", sig))
	}()

	limitParallel := semaphore.NewWeighted(cfgParallelBuilds)
	for i := range builds {
		b := builds[i]
		name := b.Name()
		ui := buildUis[name]
		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)
		if err := limitParallel.Acquire(buildCtx, 1); err != nil {
			ui.Error(fmt.Sprintf("Build '%s' failed to acquire semaphore: %s", name, err))
			errors[name] = err
			break
		}

		// Run the build in a goroutine
		go func() {
			defer wg.Done()

			defer limitParallel.Release(1)

			log.Printf("Starting build run: %s", name)
			runArtifacts, err := b.Run(buildCtx, ui)

			if err != nil {
				ui.Error(fmt.Sprintf("Build '%s' errored: %s", name, err))
				errors[name] = err
			} else {
				ui.Say(fmt.Sprintf("Build '%s' finished.", name))
				artifacts.Lock()
				artifacts.m[name] = runArtifacts
				artifacts.Unlock()
			}
		}()

		if cfgDebug {
			log.Printf("Debug enabled, so waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if cfgParallelBuilds == 1 {
			log.Printf("Parallelization disabled, waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if interrupted {
			log.Println("Interrupted, not going to start any more builds.")
			break
		}
	}

	// Wait for both the builds to complete and the interrupt handler,
	// if it is interrupted.
	log.Printf("Waiting on builds to complete...")
	wg.Wait()

	log.Printf("Builds completed. Waiting on interrupt barrier...")
	interruptWg.Wait()

	if interrupted {
		c.Ui.Say("Cleanly cancelled builds after being interrupted.")
		return 1
	}

	if len(errors) > 0 {
		c.Ui.Machine("error-count", strconv.FormatInt(int64(len(errors)), 10))

		c.Ui.Error("\n==> Some builds didn't complete successfully and had errors:")
		for name, err := range errors {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			ui.Machine("error", err.Error())

			c.Ui.Error(fmt.Sprintf("--> %s: %s", name, err))
		}
	}

	if len(artifacts.m) > 0 {
		c.Ui.Say("\n==> Builds finished. The artifacts of successful builds are:")
		for name, buildArtifacts := range artifacts.m {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			// Machine-readable helpful
			ui.Machine("artifact-count", strconv.FormatInt(int64(len(buildArtifacts)), 10))

			for i, artifact := range buildArtifacts {
				var message bytes.Buffer
				fmt.Fprintf(&message, "--> %s: ", name)

				if artifact != nil {
					fmt.Fprint(&message, artifact.String())
				} else {
					fmt.Fprint(&message, "<nothing>")
				}

				iStr := strconv.FormatInt(int64(i), 10)
				if artifact != nil {
					ui.Machine("artifact", iStr, "builder-id", artifact.BuilderId())
					ui.Machine("artifact", iStr, "id", artifact.Id())
					ui.Machine("artifact", iStr, "string", artifact.String())

					files := artifact.Files()
					ui.Machine("artifact",
						iStr,
						"files-count", strconv.FormatInt(int64(len(files)), 10))
					for fi, file := range files {
						fiStr := strconv.FormatInt(int64(fi), 10)
						ui.Machine("artifact", iStr, "file", fiStr, file)
					}
				} else {
					ui.Machine("artifact", iStr, "nil")
				}

				ui.Machine("artifact", iStr, "end")
				c.Ui.Say(message.String())
			}
		}
	} else {
		c.Ui.Say("\n==> Builds finished but no artifacts were created.")
	}

	if len(errors) > 0 {
		// If any errors occurred, exit with a non-zero exit status
		return 1
	}

	return 0
}

func (*BuildCommand) Help() string {
	helpText := `
Usage: packer build [options] TEMPLATE

  Will execute multiple builds in parallel as defined in the template.
  The various artifacts created by the template will be outputted.

Options:

  -color=false                  Disable color output. (Default: color)
  -debug                        Debug mode enabled for builds.
  -except=foo,bar,baz           Run all builds and post-procesors other than these.
  -only=foo,bar,baz             Build only the specified builds.
  -force                        Force a build to continue if artifacts exist, deletes existing artifacts.
  -machine-readable             Produce machine-readable output.
  -on-error=[cleanup|abort|ask] If the build fails do: clean up (default), abort, or ask.
  -parallel=false               Disable parallelization. (Default: true)
  -parallel-builds=1            Number of builds to run in parallel. 0 means no limit (Default: 0)
  -timestamp-ui                 Enable prefixing of each ui output with an RFC3339 timestamp.
  -var 'key=value'              Variable for templates, can be used multiple times.
  -var-file=path                JSON file containing user variables.
`

	return strings.TrimSpace(helpText)
}

func (*BuildCommand) Synopsis() string {
	return "build image(s) from template"
}

func (*BuildCommand) AutocompleteArgs() complete.Predictor {
	return complete.PredictNothing
}

func (*BuildCommand) AutocompleteFlags() complete.Flags {
	return complete.Flags{
		"-color":            complete.PredictNothing,
		"-debug":            complete.PredictNothing,
		"-except":           complete.PredictNothing,
		"-only":             complete.PredictNothing,
		"-force":            complete.PredictNothing,
		"-machine-readable": complete.PredictNothing,
		"-on-error":         complete.PredictNothing,
		"-parallel":         complete.PredictNothing,
		"-timestamp-ui":     complete.PredictNothing,
		"-var":              complete.PredictNothing,
		"-var-file":         complete.PredictNothing,
	}
}
<|endoftext|>


<commit_before>package desc

import (
	"fmt"
	"path/filepath"
	"reflect"
	"strings"
	"sync"

	"github.com/golang/protobuf/proto"
	dpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
)

var (
	globalImportPathConf map[string]string
	globalImportPathMu   sync.RWMutex
)

// RegisterImportPath registers an alternate import path for a given registered
// proto file path. For more details on why alternate import paths may need to
// be configured, see ImportResolver.
//
// This method panics if provided invalid input. An empty importPath is invalid.
// An un-registered registerPath is also invalid. For example, if an attempt is
// made to register the import path "foo/bar.proto" as "bar.proto", but there is
// no "bar.proto" registered in the Go protobuf runtime, this method will panic.
// This method also panics if an attempt is made to register the same import
// path more than once.
//
// This function works globally, applying to all descriptors loaded by this
// package. If you instead want more granular support for handling alternate
// import paths -- such as for a single invocation of a function in this
// package or when the alternate path is only used from one file (so you don't
// want the alternate path used when loading every other file), use an
// ImportResolver instead.
func RegisterImportPath(registerPath, importPath string) {
	if len(importPath) == 0 {
		panic("import path cannot be empty")
	}
	desc := proto.FileDescriptor(registerPath)
	if len(desc) == 0 {
		panic(fmt.Sprintf("path %q is not a registered proto file", registerPath))
	}
	globalImportPathMu.Lock()
	defer globalImportPathMu.Unlock()
	if reg := globalImportPathConf[importPath]; reg != "" {
		panic(fmt.Sprintf("import path %q already registered for %s", importPath, reg))
	}
	if globalImportPathConf == nil {
		globalImportPathConf = map[string]string{}
	}
	globalImportPathConf[importPath] = registerPath
}

// ResolveImport resolves the given import path. If it has been registered as an
// alternate via RegisterImportPath, the registered path is returned. Otherwise,
// the given import path is returned unchanged.
func ResolveImport(importPath string) string {
	importPath = clean(importPath)
	globalImportPathMu.RLock()
	defer globalImportPathMu.RUnlock()
	reg := globalImportPathConf[importPath]
	if reg == "" {
		return importPath
	}
	return reg
}

// ImportResolver lets you work-around linking issues that are caused by
// mismatches between how a particular proto source file is registered in the Go
// protobuf runtime and how that same file is imported by other files. The file
// is registered using the same relative path given to protoc when the file is
// compiled (i.e. when Go code is generated). So if any file tries to import
// that source file, but using a different relative path, then a link error will
// occur when this package tries to load a descriptor for the importing file.
//
// For example, let's say we have two proto source files: "foo/bar.proto" and
// "fubar/baz.proto". The latter imports the former using a line like so:
//    import "foo/bar.proto";
// However, when protoc is invoked, the command-line args looks like so:
//    protoc -Ifoo/ --go_out=foo/ bar.proto
//    protoc -I./ -Ifubar/ --go_out=fubar/ baz.proto
// Because the path given to protoc is just "bar.proto" and "baz.proto", this is
// how they are registered in the Go protobuf runtime. So, when loading the
// descriptor for "fubar/baz.proto", we'll see an import path of "foo/bar.proto"
// but will find no file registered with that path:
//    fd, err := desc.LoadFileDescriptor("baz.proto")
//    // err will be non-nil, complaining that there is no such file
//    // found named "foo/bar.proto"
//
// This can be remedied by registering alternate import paths using an
// ImportResolver. Continuing with the example above, the code below would fix
// any link issue:
//    var r desc.ImportResolver
//    r.RegisterImportPath("bar.proto", "foo/bar.proto")
//    fd, err := r.LoadFileDescriptor("baz.proto")
//    // err will be nil; descriptor successfully loaded!
//
// If there are files that are *always* imported using a different relative
// path then how they are registered, consider using the global
// RegisterImportPath function, so you don't have to use an ImportResolver for
// every file that imports it.
type ImportResolver struct {
	children    map[string]*ImportResolver
	importPaths map[string]string
}

// ResolveImport resolves the given import path in the context of the given
// source file. If a matching alternate has been registered with this resolver
// via a call to RegisterImportPath or RegisterImportPathFrom, then the
// registered path is returned. Otherwise, the given import path is returned
// unchanged.
func (r *ImportResolver) ResolveImport(source, importPath string) string {
	if r != nil {
		res := r.resolveImport(clean(source), clean(importPath))
		if res != "" {
			return res
		}
	}
	return ResolveImport(importPath)
}

func (r *ImportResolver) resolveImport(source, importPath string) string {
	if source == "" {
		return r.importPaths[importPath]
	}
	var car, cdr string
	idx := strings.IndexRune(source, filepath.Separator)
	if idx < 0 {
		car, cdr = source, ""
	} else {
		car, cdr = source[:idx], source[idx+1:]
	}
	ch := r.children[car]
	if ch != nil {
		if reg := ch.resolveImport(cdr, importPath); reg != "" {
			return reg
		}
	}
	return r.importPaths[importPath]
}

// RegisterImportPath registers an alternate import path for a given registered
// proto file path with this resolver. Any appearance of the given import path
// when linking files will instead try to link the given registered path. If the
// registered path cannot be located, then linking will fallback to the actual
// imported path.
//
// This method will panic if given an empty path or if the same import path is
// registered more than once.
//
// To constrain the contexts where the given import path is to be re-written,
// use RegisterImportPathFrom instead.
func (r *ImportResolver) RegisterImportPath(registerPath, importPath string) {
	r.RegisterImportPathFrom(registerPath, importPath, "")
}

// RegisterImportPathFrom registers an alternate import path for a given
// registered proto file path with this resolver, but only for imports in the
// specified source context.
//
// The source context can be the name of a folder or a proto source file. Any
// appearance of the given import path in that context will instead try to link
// the given registered path. To be in context, the file that is being linked
// (i.e. the one whose import statement is being resolved) must be the same
// relative path of the source context or be a sub-path (i.e. a descendant of
// the source folder).
//
// If the registered path cannot be located, then linking will fallback to the
// actual imported path.
//
// This method will panic if given an empty path. The source context, on the
// other hand, is allowed to be blank. A blank source matches all files. This
// method also panics if the same import path is registered in the same source
// context more than once.
func (r *ImportResolver) RegisterImportPathFrom(registerPath, importPath, source string) {
	importPath = clean(importPath)
	if len(importPath) == 0 {
		panic("import path cannot be empty")
	}
	registerPath = clean(registerPath)
	if len(registerPath) == 0 {
		panic("registered path cannot be empty")
	}
	r.registerImportPathFrom(registerPath, importPath, clean(source))
}

func (r *ImportResolver) registerImportPathFrom(registerPath, importPath, source string) {
	if source == "" {
		if r.importPaths == nil {
			r.importPaths = map[string]string{}
		} else if reg := r.importPaths[importPath]; reg != "" {
			panic(fmt.Sprintf("already registered import path %q as %q", importPath, registerPath))
		}
		r.importPaths[importPath] = registerPath
		return
	}
	var car, cdr string
	idx := strings.IndexRune(source, filepath.Separator)
	if idx < 0 {
		car, cdr = source, ""
	} else {
		car, cdr = source[:idx], source[idx+1:]
	}
	ch := r.children[car]
	if ch == nil {
		if r.children == nil {
			r.children = map[string]*ImportResolver{}
		}
		ch = &ImportResolver{}
		r.children[car] = ch
	}
	ch.registerImportPathFrom(registerPath, importPath, cdr)
}

// LoadFileDescriptor is the same as the package function of the same name, but
// any alternate paths configured in this resolver are used when linking the
// given descriptor proto.
func (r *ImportResolver) LoadFileDescriptor(filePath string) (*FileDescriptor, error) {
	return loadFileDescriptor(filePath, r)
}

// LoadMessageDescriptor is the same as the package function of the same name,
// but any alternate paths configured in this resolver are used when linking
// files for the returned descriptor.
func (r *ImportResolver) LoadMessageDescriptor(msgName string) (*MessageDescriptor, error) {
	return loadMessageDescriptor(msgName, r)
}

// LoadMessageDescriptorForMessage is the same as the package function of the
// same name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadMessageDescriptorForMessage(msg proto.Message) (*MessageDescriptor, error) {
	return loadMessageDescriptorForMessage(msg, r)
}

// LoadMessageDescriptorForType is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadMessageDescriptorForType(msgType reflect.Type) (*MessageDescriptor, error) {
	return loadMessageDescriptorForType(msgType, r)
}

// LoadEnumDescriptorForEnum is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadEnumDescriptorForEnum(enum protoEnum) (*EnumDescriptor, error) {
	return loadEnumDescriptorForEnum(enum, r)
}

// LoadEnumDescriptorForType is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadEnumDescriptorForType(enumType reflect.Type) (*EnumDescriptor, error) {
	return loadEnumDescriptorForType(enumType, r)
}

// LoadFieldDescriptorForExtension is the same as the package function of the
// same name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadFieldDescriptorForExtension(ext *proto.ExtensionDesc) (*FieldDescriptor, error) {
	return loadFieldDescriptorForExtension(ext, r)
}

// CreateFileDescriptor is the same as the package function of the same name,
// but any alternate paths configured in this resolver are used when linking the
// given descriptor proto.
func (r *ImportResolver) CreateFileDescriptor(fdp *dpb.FileDescriptorProto, deps ...*FileDescriptor) (*FileDescriptor, error) {
	return createFileDescriptor(fdp, deps, r)
}

// CreateFileDescriptors is the same as the package function of the same name,
// but any alternate paths configured in this resolver are used when linking the
// given descriptor protos.
func (r *ImportResolver) CreateFileDescriptors(fds []*dpb.FileDescriptorProto) (map[string]*FileDescriptor, error) {
	return createFileDescriptors(fds, r)
}

// CreateFileDescriptorFromSet is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking the descriptor protos in the given set.
func (r *ImportResolver) CreateFileDescriptorFromSet(fds *dpb.FileDescriptorSet) (*FileDescriptor, error) {
	return createFileDescriptorFromSet(fds, r)
}

const dotPrefix = "." + string(filepath.Separator)

func clean(path string) string {
	if path == "" {
		return ""
	}
	path = filepath.Clean(path)
	if path == "." {
		return ""
	}
	return strings.TrimPrefix(path, dotPrefix)
}
<commit_msg>ImportResolver flag to skip global rules<commit_after>package desc

import (
	"fmt"
	"path/filepath"
	"reflect"
	"strings"
	"sync"

	"github.com/golang/protobuf/proto"
	dpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
)

var (
	globalImportPathConf map[string]string
	globalImportPathMu   sync.RWMutex
)

// RegisterImportPath registers an alternate import path for a given registered
// proto file path. For more details on why alternate import paths may need to
// be configured, see ImportResolver.
//
// This method panics if provided invalid input. An empty importPath is invalid.
// An un-registered registerPath is also invalid. For example, if an attempt is
// made to register the import path "foo/bar.proto" as "bar.proto", but there is
// no "bar.proto" registered in the Go protobuf runtime, this method will panic.
// This method also panics if an attempt is made to register the same import
// path more than once.
//
// This function works globally, applying to all descriptors loaded by this
// package. If you instead want more granular support for handling alternate
// import paths -- such as for a single invocation of a function in this
// package or when the alternate path is only used from one file (so you don't
// want the alternate path used when loading every other file), use an
// ImportResolver instead.
func RegisterImportPath(registerPath, importPath string) {
	if len(importPath) == 0 {
		panic("import path cannot be empty")
	}
	desc := proto.FileDescriptor(registerPath)
	if len(desc) == 0 {
		panic(fmt.Sprintf("path %q is not a registered proto file", registerPath))
	}
	globalImportPathMu.Lock()
	defer globalImportPathMu.Unlock()
	if reg := globalImportPathConf[importPath]; reg != "" {
		panic(fmt.Sprintf("import path %q already registered for %s", importPath, reg))
	}
	if globalImportPathConf == nil {
		globalImportPathConf = map[string]string{}
	}
	globalImportPathConf[importPath] = registerPath
}

// ResolveImport resolves the given import path. If it has been registered as an
// alternate via RegisterImportPath, the registered path is returned. Otherwise,
// the given import path is returned unchanged.
func ResolveImport(importPath string) string {
	importPath = clean(importPath)
	globalImportPathMu.RLock()
	defer globalImportPathMu.RUnlock()
	reg := globalImportPathConf[importPath]
	if reg == "" {
		return importPath
	}
	return reg
}

// ImportResolver lets you work-around linking issues that are caused by
// mismatches between how a particular proto source file is registered in the Go
// protobuf runtime and how that same file is imported by other files. The file
// is registered using the same relative path given to protoc when the file is
// compiled (i.e. when Go code is generated). So if any file tries to import
// that source file, but using a different relative path, then a link error will
// occur when this package tries to load a descriptor for the importing file.
//
// For example, let's say we have two proto source files: "foo/bar.proto" and
// "fubar/baz.proto". The latter imports the former using a line like so:
//    import "foo/bar.proto";
// However, when protoc is invoked, the command-line args looks like so:
//    protoc -Ifoo/ --go_out=foo/ bar.proto
//    protoc -I./ -Ifubar/ --go_out=fubar/ baz.proto
// Because the path given to protoc is just "bar.proto" and "baz.proto", this is
// how they are registered in the Go protobuf runtime. So, when loading the
// descriptor for "fubar/baz.proto", we'll see an import path of "foo/bar.proto"
// but will find no file registered with that path:
//    fd, err := desc.LoadFileDescriptor("baz.proto")
//    // err will be non-nil, complaining that there is no such file
//    // found named "foo/bar.proto"
//
// This can be remedied by registering alternate import paths using an
// ImportResolver. Continuing with the example above, the code below would fix
// any link issue:
//    var r desc.ImportResolver
//    r.RegisterImportPath("bar.proto", "foo/bar.proto")
//    fd, err := r.LoadFileDescriptor("baz.proto")
//    // err will be nil; descriptor successfully loaded!
//
// If there are files that are *always* imported using a different relative
// path then how they are registered, consider using the global
// RegisterImportPath function, so you don't have to use an ImportResolver for
// every file that imports it.
type ImportResolver struct {
	children    map[string]*ImportResolver
	importPaths map[string]string

	// By default, an ImportResolver will fallback to consulting any paths
	// registered via the top-level RegisterImportPath function. Setting this
	// field to true will cause the ImportResolver to skip that fallback and
	// only examine its own locally registered paths.
	SkipFallbackRules bool
}

// ResolveImport resolves the given import path in the context of the given
// source file. If a matching alternate has been registered with this resolver
// via a call to RegisterImportPath or RegisterImportPathFrom, then the
// registered path is returned. Otherwise, the given import path is returned
// unchanged.
func (r *ImportResolver) ResolveImport(source, importPath string) string {
	if r != nil {
		res := r.resolveImport(clean(source), clean(importPath))
		if res != "" {
			return res
		}
	}
	if r.SkipFallbackRules {
		return importPath
	}
	return ResolveImport(importPath)
}

func (r *ImportResolver) resolveImport(source, importPath string) string {
	if source == "" {
		return r.importPaths[importPath]
	}
	var car, cdr string
	idx := strings.IndexRune(source, filepath.Separator)
	if idx < 0 {
		car, cdr = source, ""
	} else {
		car, cdr = source[:idx], source[idx+1:]
	}
	ch := r.children[car]
	if ch != nil {
		if reg := ch.resolveImport(cdr, importPath); reg != "" {
			return reg
		}
	}
	return r.importPaths[importPath]
}

// RegisterImportPath registers an alternate import path for a given registered
// proto file path with this resolver. Any appearance of the given import path
// when linking files will instead try to link the given registered path. If the
// registered path cannot be located, then linking will fallback to the actual
// imported path.
//
// This method will panic if given an empty path or if the same import path is
// registered more than once.
//
// To constrain the contexts where the given import path is to be re-written,
// use RegisterImportPathFrom instead.
func (r *ImportResolver) RegisterImportPath(registerPath, importPath string) {
	r.RegisterImportPathFrom(registerPath, importPath, "")
}

// RegisterImportPathFrom registers an alternate import path for a given
// registered proto file path with this resolver, but only for imports in the
// specified source context.
//
// The source context can be the name of a folder or a proto source file. Any
// appearance of the given import path in that context will instead try to link
// the given registered path. To be in context, the file that is being linked
// (i.e. the one whose import statement is being resolved) must be the same
// relative path of the source context or be a sub-path (i.e. a descendant of
// the source folder).
//
// If the registered path cannot be located, then linking will fallback to the
// actual imported path.
//
// This method will panic if given an empty path. The source context, on the
// other hand, is allowed to be blank. A blank source matches all files. This
// method also panics if the same import path is registered in the same source
// context more than once.
func (r *ImportResolver) RegisterImportPathFrom(registerPath, importPath, source string) {
	importPath = clean(importPath)
	if len(importPath) == 0 {
		panic("import path cannot be empty")
	}
	registerPath = clean(registerPath)
	if len(registerPath) == 0 {
		panic("registered path cannot be empty")
	}
	r.registerImportPathFrom(registerPath, importPath, clean(source))
}

func (r *ImportResolver) registerImportPathFrom(registerPath, importPath, source string) {
	if source == "" {
		if r.importPaths == nil {
			r.importPaths = map[string]string{}
		} else if reg := r.importPaths[importPath]; reg != "" {
			panic(fmt.Sprintf("already registered import path %q as %q", importPath, registerPath))
		}
		r.importPaths[importPath] = registerPath
		return
	}
	var car, cdr string
	idx := strings.IndexRune(source, filepath.Separator)
	if idx < 0 {
		car, cdr = source, ""
	} else {
		car, cdr = source[:idx], source[idx+1:]
	}
	ch := r.children[car]
	if ch == nil {
		if r.children == nil {
			r.children = map[string]*ImportResolver{}
		}
		ch = &ImportResolver{}
		r.children[car] = ch
	}
	ch.registerImportPathFrom(registerPath, importPath, cdr)
}

// LoadFileDescriptor is the same as the package function of the same name, but
// any alternate paths configured in this resolver are used when linking the
// given descriptor proto.
func (r *ImportResolver) LoadFileDescriptor(filePath string) (*FileDescriptor, error) {
	return loadFileDescriptor(filePath, r)
}

// LoadMessageDescriptor is the same as the package function of the same name,
// but any alternate paths configured in this resolver are used when linking
// files for the returned descriptor.
func (r *ImportResolver) LoadMessageDescriptor(msgName string) (*MessageDescriptor, error) {
	return loadMessageDescriptor(msgName, r)
}

// LoadMessageDescriptorForMessage is the same as the package function of the
// same name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadMessageDescriptorForMessage(msg proto.Message) (*MessageDescriptor, error) {
	return loadMessageDescriptorForMessage(msg, r)
}

// LoadMessageDescriptorForType is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadMessageDescriptorForType(msgType reflect.Type) (*MessageDescriptor, error) {
	return loadMessageDescriptorForType(msgType, r)
}

// LoadEnumDescriptorForEnum is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadEnumDescriptorForEnum(enum protoEnum) (*EnumDescriptor, error) {
	return loadEnumDescriptorForEnum(enum, r)
}

// LoadEnumDescriptorForType is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadEnumDescriptorForType(enumType reflect.Type) (*EnumDescriptor, error) {
	return loadEnumDescriptorForType(enumType, r)
}

// LoadFieldDescriptorForExtension is the same as the package function of the
// same name, but any alternate paths configured in this resolver are used when
// linking files for the returned descriptor.
func (r *ImportResolver) LoadFieldDescriptorForExtension(ext *proto.ExtensionDesc) (*FieldDescriptor, error) {
	return loadFieldDescriptorForExtension(ext, r)
}

// CreateFileDescriptor is the same as the package function of the same name,
// but any alternate paths configured in this resolver are used when linking the
// given descriptor proto.
func (r *ImportResolver) CreateFileDescriptor(fdp *dpb.FileDescriptorProto, deps ...*FileDescriptor) (*FileDescriptor, error) {
	return createFileDescriptor(fdp, deps, r)
}

// CreateFileDescriptors is the same as the package function of the same name,
// but any alternate paths configured in this resolver are used when linking the
// given descriptor protos.
func (r *ImportResolver) CreateFileDescriptors(fds []*dpb.FileDescriptorProto) (map[string]*FileDescriptor, error) {
	return createFileDescriptors(fds, r)
}

// CreateFileDescriptorFromSet is the same as the package function of the same
// name, but any alternate paths configured in this resolver are used when
// linking the descriptor protos in the given set.
func (r *ImportResolver) CreateFileDescriptorFromSet(fds *dpb.FileDescriptorSet) (*FileDescriptor, error) {
	return createFileDescriptorFromSet(fds, r)
}

const dotPrefix = "." + string(filepath.Separator)

func clean(path string) string {
	if path == "" {
		return ""
	}
	path = filepath.Clean(path)
	if path == "." {
		return ""
	}
	return strings.TrimPrefix(path, dotPrefix)
}
<|endoftext|>


<commit_before>package server

import (
	"math"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/tidwall/geojson"
	"github.com/tidwall/rhh"
	"github.com/tidwall/tile38/core"
	"github.com/tidwall/tile38/internal/collection"
	"github.com/tidwall/tile38/internal/log"
)

const maxkeys = 8
const maxids = 32
const maxchunk = 4 * 1024 * 1024

func (server *Server) aofshrink() {
	if server.aof == nil {
		return
	}
	start := time.Now()
	server.mu.Lock()
	if server.shrinking {
		server.mu.Unlock()
		return
	}
	server.shrinking = true
	server.shrinklog = nil
	server.mu.Unlock()

	defer func() {
		server.mu.Lock()
		server.shrinking = false
		server.shrinklog = nil
		server.mu.Unlock()
		log.Infof("aof shrink ended %v", time.Now().Sub(start))
		return
	}()

	err := func() error {
		f, err := os.Create(core.AppendFileName + "-shrink")
		if err != nil {
			return err
		}
		defer f.Close()
		var aofbuf []byte
		var values []string
		var keys []string
		var nextkey string
		var keysdone bool
		for {
			if len(keys) == 0 {
				// load more keys
				if keysdone {
					break
				}
				keysdone = true
				func() {
					server.mu.Lock()
					defer server.mu.Unlock()
					server.scanGreaterOrEqual(nextkey, func(key string, col *collection.Collection) bool {
						if len(keys) == maxkeys {
							keysdone = false
							nextkey = key
							return false
						}
						keys = append(keys, key)
						return true
					})
				}()
				continue
			}

			var idsdone bool
			var nextid string
			for {
				if idsdone {
					keys = keys[1:]
					break
				}

				// load more objects
				func() {
					idsdone = true
					server.mu.Lock()
					defer server.mu.Unlock()
					col := server.getCol(keys[0])
					if col == nil {
						return
					}
					var fnames = col.FieldArr() // reload an array of field names to match each object
					var exm *rhh.Map            // the expiration map
					if value, ok := server.expires.Get(keys[0]); ok {
						exm = value.(*rhh.Map)
					}
					var now = time.Now().UnixNano() // used for expiration
					var count = 0                   // the object count
					col.ScanGreaterOrEqual(nextid, false, nil, nil,
						func(id string, obj geojson.Object, fields []float64) bool {
							if count == maxids {
								// we reached the max number of ids for one batch
								nextid = id
								idsdone = false
								return false
							}

							// here we fill the values array with a new command
							values = values[:0]
							values = append(values, "set")
							values = append(values, keys[0])
							values = append(values, id)
							for i, fvalue := range fields {
								if fvalue != 0 {
									values = append(values, "field")
									values = append(values, fnames[i])
									values = append(values, strconv.FormatFloat(fvalue, 'f', -1, 64))
								}
							}
							if exm != nil {
								if at, ok := exm.Get(id); ok {
									expires := at.(int64) - now
									if expires > 0 {
										values = append(values, "ex")
										values = append(values, strconv.FormatFloat(math.Floor(float64(expires)/float64(time.Second)*10)/10, 'f', -1, 64))
									}
								}
							}
							if objIsSpatial(obj) {
								values = append(values, "object")
								values = append(values, string(obj.AppendJSON(nil)))
							} else {
								values = append(values, "string")
								values = append(values, obj.String())
							}

							// append the values to the aof buffer
							aofbuf = append(aofbuf, '*')
							aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...)
							aofbuf = append(aofbuf, '\r', '\n')
							for _, value := range values {
								aofbuf = append(aofbuf, '$')
								aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...)
								aofbuf = append(aofbuf, '\r', '\n')
								aofbuf = append(aofbuf, value...)
								aofbuf = append(aofbuf, '\r', '\n')
							}

							// increment the object count
							count++
							return true
						},
					)

				}()
				if len(aofbuf) > maxchunk {
					if _, err := f.Write(aofbuf); err != nil {
						return err
					}
					aofbuf = aofbuf[:0]
				}
			}
		}

		// load hooks
		// first load the names of the hooks
		var hnames []string
		func() {
			server.mu.Lock()
			defer server.mu.Unlock()
			for name := range server.hooks {
				hnames = append(hnames, name)
			}
		}()
		// sort the names for consistency
		sort.Strings(hnames)
		for _, name := range hnames {
			func() {
				server.mu.Lock()
				defer server.mu.Unlock()
				hook := server.hooks[name]
				if hook == nil {
					return
				}
				hook.cond.L.Lock()
				defer hook.cond.L.Unlock()

				var values []string
				if hook.channel {
					values = append(values, "setchan", name)
				} else {
					values = append(values, "sethook", name,
						strings.Join(hook.Endpoints, ","))
					values = append(values)
				}
				for _, meta := range hook.Metas {
					values = append(values, "meta", meta.Name, meta.Value)
				}
				if !hook.expires.IsZero() {
					ex := float64(hook.expires.Sub(time.Now())) /
						float64(time.Second)
					values = append(values, "ex",
						strconv.FormatFloat(ex, 'f', 1, 64))
				}
				for _, value := range hook.Message.Args {
					values = append(values, value)
				}
				// append the values to the aof buffer
				aofbuf = append(aofbuf, '*')
				aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...)
				aofbuf = append(aofbuf, '\r', '\n')
				for _, value := range values {
					aofbuf = append(aofbuf, '$')
					aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...)
					aofbuf = append(aofbuf, '\r', '\n')
					aofbuf = append(aofbuf, value...)
					aofbuf = append(aofbuf, '\r', '\n')
				}
			}()
		}
		if len(aofbuf) > 0 {
			if _, err := f.Write(aofbuf); err != nil {
				return err
			}
			aofbuf = aofbuf[:0]
		}
		if err := f.Sync(); err != nil {
			return err
		}

		// finally grab any new data that may have been written since
		// the aofshrink has started and swap out the files.
		return func() error {
			server.mu.Lock()
			defer server.mu.Unlock()

			// flush the aof buffer
			server.flushAOF(false)

			aofbuf = aofbuf[:0]
			for _, values := range server.shrinklog {
				// append the values to the aof buffer
				aofbuf = append(aofbuf, '*')
				aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...)
				aofbuf = append(aofbuf, '\r', '\n')
				for _, value := range values {
					aofbuf = append(aofbuf, '$')
					aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...)
					aofbuf = append(aofbuf, '\r', '\n')
					aofbuf = append(aofbuf, value...)
					aofbuf = append(aofbuf, '\r', '\n')
				}
			}
			if _, err := f.Write(aofbuf); err != nil {
				return err
			}
			if err := f.Sync(); err != nil {
				return err
			}
			// we now have a shrunken aof file that is fully in-sync with
			// the current dataset. let's swap out the on disk files and
			// point to the new file.

			// anything below this point is unrecoverable. just log and exit process
			// back up the live aof, just in case of fatal error
			if err := server.aof.Close(); err != nil {
				log.Fatalf("shrink live aof close fatal operation: %v", err)
			}
			if err := f.Close(); err != nil {
				log.Fatalf("shrink new aof close fatal operation: %v", err)
			}
			if err := os.Rename(core.AppendFileName, core.AppendFileName+"-bak"); err != nil {
				log.Fatalf("shrink backup fatal operation: %v", err)
			}
			if err := os.Rename(core.AppendFileName+"-shrink", core.AppendFileName); err != nil {
				log.Fatalf("shrink rename fatal operation: %v", err)
			}
			server.aof, err = os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600)
			if err != nil {
				log.Fatalf("shrink openfile fatal operation: %v", err)
			}
			var n int64
			n, err = server.aof.Seek(0, 2)
			if err != nil {
				log.Fatalf("shrink seek end fatal operation: %v", err)
			}
			server.aofsz = int(n)

			os.Remove(core.AppendFileName + "-bak") // ignore error

			// kill all followers connections
			for conn := range server.aofconnM {
				conn.Close()
			}
			return nil
		}()
	}()
	if err != nil {
		log.Errorf("aof shrink failed: %v", err)
		return
	}
}
<commit_msg>Fixed fields being shuffled after AOFSHRINK<commit_after>package server

import (
	"math"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/tidwall/geojson"
	"github.com/tidwall/rhh"
	"github.com/tidwall/tile38/core"
	"github.com/tidwall/tile38/internal/collection"
	"github.com/tidwall/tile38/internal/log"
)

const maxkeys = 8
const maxids = 32
const maxchunk = 4 * 1024 * 1024

func (server *Server) aofshrink() {
	if server.aof == nil {
		return
	}
	start := time.Now()
	server.mu.Lock()
	if server.shrinking {
		server.mu.Unlock()
		return
	}
	server.shrinking = true
	server.shrinklog = nil
	server.mu.Unlock()

	defer func() {
		server.mu.Lock()
		server.shrinking = false
		server.shrinklog = nil
		server.mu.Unlock()
		log.Infof("aof shrink ended %v", time.Now().Sub(start))
		return
	}()

	err := func() error {
		f, err := os.Create(core.AppendFileName + "-shrink")
		if err != nil {
			return err
		}
		defer f.Close()
		var aofbuf []byte
		var values []string
		var keys []string
		var nextkey string
		var keysdone bool
		for {
			if len(keys) == 0 {
				// load more keys
				if keysdone {
					break
				}
				keysdone = true
				func() {
					server.mu.Lock()
					defer server.mu.Unlock()
					server.scanGreaterOrEqual(nextkey, func(key string, col *collection.Collection) bool {
						if len(keys) == maxkeys {
							keysdone = false
							nextkey = key
							return false
						}
						keys = append(keys, key)
						return true
					})
				}()
				continue
			}

			var idsdone bool
			var nextid string
			for {
				if idsdone {
					keys = keys[1:]
					break
				}

				// load more objects
				func() {
					idsdone = true
					server.mu.Lock()
					defer server.mu.Unlock()
					col := server.getCol(keys[0])
					if col == nil {
						return
					}
					var fnames = col.FieldArr() // reload an array of field names to match each object
					var fmap = col.FieldMap()   //
					var exm *rhh.Map            // the expiration map
					if value, ok := server.expires.Get(keys[0]); ok {
						exm = value.(*rhh.Map)
					}
					var now = time.Now().UnixNano() // used for expiration
					var count = 0                   // the object count
					col.ScanGreaterOrEqual(nextid, false, nil, nil,
						func(id string, obj geojson.Object, fields []float64) bool {
							if count == maxids {
								// we reached the max number of ids for one batch
								nextid = id
								idsdone = false
								return false
							}

							// here we fill the values array with a new command
							values = values[:0]
							values = append(values, "set")
							values = append(values, keys[0])
							values = append(values, id)
							if len(fields) > 0 {
								fvs := orderFields(fmap, fnames, fields)
								for _, fv := range fvs {
									if fv.value != 0 {
										values = append(values, "field")
										values = append(values, fv.field)
										values = append(values, strconv.FormatFloat(fv.value, 'f', -1, 64))
									}
								}
							}
							if exm != nil {
								if at, ok := exm.Get(id); ok {
									expires := at.(int64) - now
									if expires > 0 {
										values = append(values, "ex")
										values = append(values, strconv.FormatFloat(math.Floor(float64(expires)/float64(time.Second)*10)/10, 'f', -1, 64))
									}
								}
							}
							if objIsSpatial(obj) {
								values = append(values, "object")
								values = append(values, string(obj.AppendJSON(nil)))
							} else {
								values = append(values, "string")
								values = append(values, obj.String())
							}

							// append the values to the aof buffer
							aofbuf = append(aofbuf, '*')
							aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...)
							aofbuf = append(aofbuf, '\r', '\n')
							for _, value := range values {
								aofbuf = append(aofbuf, '$')
								aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...)
								aofbuf = append(aofbuf, '\r', '\n')
								aofbuf = append(aofbuf, value...)
								aofbuf = append(aofbuf, '\r', '\n')
							}

							// increment the object count
							count++
							return true
						},
					)

				}()
				if len(aofbuf) > maxchunk {
					if _, err := f.Write(aofbuf); err != nil {
						return err
					}
					aofbuf = aofbuf[:0]
				}
			}
		}

		// load hooks
		// first load the names of the hooks
		var hnames []string
		func() {
			server.mu.Lock()
			defer server.mu.Unlock()
			for name := range server.hooks {
				hnames = append(hnames, name)
			}
		}()
		// sort the names for consistency
		sort.Strings(hnames)
		for _, name := range hnames {
			func() {
				server.mu.Lock()
				defer server.mu.Unlock()
				hook := server.hooks[name]
				if hook == nil {
					return
				}
				hook.cond.L.Lock()
				defer hook.cond.L.Unlock()

				var values []string
				if hook.channel {
					values = append(values, "setchan", name)
				} else {
					values = append(values, "sethook", name,
						strings.Join(hook.Endpoints, ","))
					values = append(values)
				}
				for _, meta := range hook.Metas {
					values = append(values, "meta", meta.Name, meta.Value)
				}
				if !hook.expires.IsZero() {
					ex := float64(hook.expires.Sub(time.Now())) /
						float64(time.Second)
					values = append(values, "ex",
						strconv.FormatFloat(ex, 'f', 1, 64))
				}
				for _, value := range hook.Message.Args {
					values = append(values, value)
				}
				// append the values to the aof buffer
				aofbuf = append(aofbuf, '*')
				aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...)
				aofbuf = append(aofbuf, '\r', '\n')
				for _, value := range values {
					aofbuf = append(aofbuf, '$')
					aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...)
					aofbuf = append(aofbuf, '\r', '\n')
					aofbuf = append(aofbuf, value...)
					aofbuf = append(aofbuf, '\r', '\n')
				}
			}()
		}
		if len(aofbuf) > 0 {
			if _, err := f.Write(aofbuf); err != nil {
				return err
			}
			aofbuf = aofbuf[:0]
		}
		if err := f.Sync(); err != nil {
			return err
		}

		// finally grab any new data that may have been written since
		// the aofshrink has started and swap out the files.
		return func() error {
			server.mu.Lock()
			defer server.mu.Unlock()

			// flush the aof buffer
			server.flushAOF(false)

			aofbuf = aofbuf[:0]
			for _, values := range server.shrinklog {
				// append the values to the aof buffer
				aofbuf = append(aofbuf, '*')
				aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...)
				aofbuf = append(aofbuf, '\r', '\n')
				for _, value := range values {
					aofbuf = append(aofbuf, '$')
					aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...)
					aofbuf = append(aofbuf, '\r', '\n')
					aofbuf = append(aofbuf, value...)
					aofbuf = append(aofbuf, '\r', '\n')
				}
			}
			if _, err := f.Write(aofbuf); err != nil {
				return err
			}
			if err := f.Sync(); err != nil {
				return err
			}
			// we now have a shrunken aof file that is fully in-sync with
			// the current dataset. let's swap out the on disk files and
			// point to the new file.

			// anything below this point is unrecoverable. just log and exit process
			// back up the live aof, just in case of fatal error
			if err := server.aof.Close(); err != nil {
				log.Fatalf("shrink live aof close fatal operation: %v", err)
			}
			if err := f.Close(); err != nil {
				log.Fatalf("shrink new aof close fatal operation: %v", err)
			}
			if err := os.Rename(core.AppendFileName, core.AppendFileName+"-bak"); err != nil {
				log.Fatalf("shrink backup fatal operation: %v", err)
			}
			if err := os.Rename(core.AppendFileName+"-shrink", core.AppendFileName); err != nil {
				log.Fatalf("shrink rename fatal operation: %v", err)
			}
			server.aof, err = os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600)
			if err != nil {
				log.Fatalf("shrink openfile fatal operation: %v", err)
			}
			var n int64
			n, err = server.aof.Seek(0, 2)
			if err != nil {
				log.Fatalf("shrink seek end fatal operation: %v", err)
			}
			server.aofsz = int(n)

			os.Remove(core.AppendFileName + "-bak") // ignore error

			// kill all followers connections
			for conn := range server.aofconnM {
				conn.Close()
			}
			return nil
		}()
	}()
	if err != nil {
		log.Errorf("aof shrink failed: %v", err)
		return
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"strings"
	"unicode/utf8"

	"github.com/zetamatta/go-mbcs"
)

func cat(ctx context.Context, r io.Reader, w io.Writer) bool {
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		if done := ctx.Done(); done != nil {
			select {
			case <-done:
				return false
			default:

			}
		}
		line := scanner.Bytes()
		var text string
		if utf8.Valid(line) {
			text = string(line)
		} else {
			var err error
			text, err = mbcs.AtoU(line)
			if err != nil {
				text = err.Error()
			}
		}
		text = strings.Replace(text, "\xEF\xBB\xBF", "", 1)
		fmt.Fprintln(w, text)
	}
	return true
}

func cmdType(ctx context.Context, cmd Param) (int, error) {
	if len(cmd.Args()) <= 1 {
		cat(ctx, cmd.In(), cmd.Out())
	} else {
		for _, arg1 := range cmd.Args()[1:] {
			r, err := os.Open(arg1)
			if err != nil {
				return 1, err
			}
			stat1, err := r.Stat()
			if err != nil {
				r.Close()
				return 2, err
			}
			if stat1.IsDir() {
				r.Close()
				return 3, fmt.Errorf("%s: Permission denied", arg1)
			}
			cont := cat(ctx, r, cmd.Out())
			r.Close()
			if !cont {
				return 0, nil
			}
		}
	}
	return 0, nil
}
<commit_msg>(#267) `type` supports UTF16 (requires go-texts package)<commit_after>package commands

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/zetamatta/go-texts/mbcs"
)

func cat(ctx context.Context, _r io.Reader, w io.Writer) bool {
	r := mbcs.NewAutoDetectReader(_r, mbcs.ConsoleCP())
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		if done := ctx.Done(); done != nil {
			select {
			case <-done:
				return false
			default:

			}
		}
		text := scanner.Text()
		text = strings.Replace(text, "\xEF\xBB\xBF", "", 1)
		fmt.Fprintln(w, text)
	}
	return true
}

func cmdType(ctx context.Context, cmd Param) (int, error) {
	if len(cmd.Args()) <= 1 {
		cat(ctx, cmd.In(), cmd.Out())
	} else {
		for _, arg1 := range cmd.Args()[1:] {
			r, err := os.Open(arg1)
			if err != nil {
				return 1, err
			}
			stat1, err := r.Stat()
			if err != nil {
				r.Close()
				return 2, err
			}
			if stat1.IsDir() {
				r.Close()
				return 3, fmt.Errorf("%s: Permission denied", arg1)
			}
			cont := cat(ctx, r, cmd.Out())
			r.Close()
			if !cont {
				return 0, nil
			}
		}
	}
	return 0, nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// PackerKeyEnv is used to specify the key interval (delay) between keystrokes
// sent to the VM, typically in boot commands. This is to prevent host CPU
// utilization from causing key presses to be skipped or repeated incorrectly.
const PackerKeyEnv = "PACKER_KEY_INTERVAL"

// PackerKeyDefault 100ms is appropriate for shared build infrastructure while a
// shorter delay (e.g. 10ms) can be used on a workstation. See PackerKeyEnv.
const PackerKeyDefault = 100 * time.Millisecond

// ScrubConfig is a helper that returns a string representation of
// any struct with the given values stripped out.
func ScrubConfig(target interface{}, values ...string) string {
	conf := fmt.Sprintf("Config: %+v", target)
	for _, value := range values {
		if value == "" {
			continue
		}
		conf = strings.Replace(conf, value, "<Filtered>", -1)
	}
	return conf
}

// ChooseString returns the first non-empty value.
func ChooseString(vals ...string) string {
	for _, el := range vals {
		if el != "" {
			return el
		}
	}

	return ""
}

// DownloadableURL processes a URL that may also be a file path and returns
// a completely valid URL. For example, the original URL might be "local/file.iso"
// which isn't a valid URL. DownloadableURL will return "file:///local/file.iso"
func DownloadableURL(original string) (string, error) {
	if runtime.GOOS == "windows" {
		// If the distance to the first ":" is just one character, assume
		// we're dealing with a drive letter and thus a file path.
		idx := strings.Index(original, ":")
		if idx == 1 {
			original = "file:///" + original
		}
	}

	url, err := url.Parse(original)
	if err != nil {
		return "", err
	}

	if url.Scheme == "" {
		url.Scheme = "file"
	}

	if url.Scheme == "file" {
		// Windows file handling is all sorts of tricky...
		if runtime.GOOS == "windows" {
			// If the path is using Windows-style slashes, URL parses
			// it into the host field.
			if url.Path == "" && strings.Contains(url.Host, `\`) {
				url.Path = url.Host
				url.Host = ""
			}

			// For Windows absolute file paths, remove leading / prior to processing
			// since net/url turns "C:/" into "/C:/"
			if len(url.Path) > 0 && url.Path[0] == '/' {
				url.Path = url.Path[1:len(url.Path)]
			}
		}

		// Only do the filepath transformations if the file appears
		// to actually exist.
		if _, err := os.Stat(url.Path); err == nil {
			url.Path, err = filepath.Abs(url.Path)
			if err != nil {
				return "", err
			}

			url.Path, err = filepath.EvalSymlinks(url.Path)
			if err != nil {
				return "", err
			}

			url.Path = filepath.Clean(url.Path)
		}

		if runtime.GOOS == "windows" {
			// Also replace all backslashes with forwardslashes since Windows
			// users are likely to do this but the URL should actually only
			// contain forward slashes.
			url.Path = strings.Replace(url.Path, `\`, `/`, -1)
		}
	}

	// Make sure it is lowercased
	url.Scheme = strings.ToLower(url.Scheme)

	// This is to work around issue #5927. This can safely be removed once
	// we distribute with a version of Go that fixes that bug.
	//
	// See: https://code.google.com/p/go/issues/detail?id=5927
	if url.Path != "" && url.Path[0] != '/' {
		url.Path = "/" + url.Path
	}

	// Verify that the scheme is something we support in our common downloader.
	supported := []string{"file", "http", "https"}
	found := false
	for _, s := range supported {
		if url.Scheme == s {
			found = true
			break
		}
	}

	if !found {
		return "", fmt.Errorf("Unsupported URL scheme: %s", url.Scheme)
	}

	return url.String(), nil
}
<commit_msg>clearly state that url is wrong at validation stage of build<commit_after>package common

import (
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// PackerKeyEnv is used to specify the key interval (delay) between keystrokes
// sent to the VM, typically in boot commands. This is to prevent host CPU
// utilization from causing key presses to be skipped or repeated incorrectly.
const PackerKeyEnv = "PACKER_KEY_INTERVAL"

// PackerKeyDefault 100ms is appropriate for shared build infrastructure while a
// shorter delay (e.g. 10ms) can be used on a workstation. See PackerKeyEnv.
const PackerKeyDefault = 100 * time.Millisecond

// ScrubConfig is a helper that returns a string representation of
// any struct with the given values stripped out.
func ScrubConfig(target interface{}, values ...string) string {
	conf := fmt.Sprintf("Config: %+v", target)
	for _, value := range values {
		if value == "" {
			continue
		}
		conf = strings.Replace(conf, value, "<Filtered>", -1)
	}
	return conf
}

// ChooseString returns the first non-empty value.
func ChooseString(vals ...string) string {
	for _, el := range vals {
		if el != "" {
			return el
		}
	}

	return ""
}

// DownloadableURL processes a URL that may also be a file path and returns
// a completely valid URL. For example, the original URL might be "local/file.iso"
// which isn't a valid URL. DownloadableURL will return "file:///local/file.iso"
func DownloadableURL(original string) (string, error) {
	if runtime.GOOS == "windows" {
		// If the distance to the first ":" is just one character, assume
		// we're dealing with a drive letter and thus a file path.
		idx := strings.Index(original, ":")
		if idx == 1 {
			original = "file:///" + original
		}
	}

	url, err := url.Parse(original)
	if err != nil {
		return "", err
	}

	if url.Scheme == "" {
		url.Scheme = "file"
	}

	if url.Scheme == "file" {
		// Windows file handling is all sorts of tricky...
		if runtime.GOOS == "windows" {
			// If the path is using Windows-style slashes, URL parses
			// it into the host field.
			if url.Path == "" && strings.Contains(url.Host, `\`) {
				url.Path = url.Host
				url.Host = ""
			}

			// For Windows absolute file paths, remove leading / prior to processing
			// since net/url turns "C:/" into "/C:/"
			if len(url.Path) > 0 && url.Path[0] == '/' {
				url.Path = url.Path[1:len(url.Path)]
			}
		}

		// Only do the filepath transformations if the file appears
		// to actually exist.
		if _, err := os.Stat(url.Path); err == nil {
			url.Path, err = filepath.Abs(url.Path)
			if err != nil {
				return "", err
			}

			url.Path, err = filepath.EvalSymlinks(url.Path)
			if err != nil {
				return "", err
			}

			url.Path = filepath.Clean(url.Path)
		}

		if runtime.GOOS == "windows" {
			// Also replace all backslashes with forwardslashes since Windows
			// users are likely to do this but the URL should actually only
			// contain forward slashes.
			url.Path = strings.Replace(url.Path, `\`, `/`, -1)
		}
	}

	// Make sure it is lowercased
	url.Scheme = strings.ToLower(url.Scheme)

	// This is to work around issue #5927. This can safely be removed once
	// we distribute with a version of Go that fixes that bug.
	//
	// See: https://code.google.com/p/go/issues/detail?id=5927
	if url.Path != "" && url.Path[0] != '/' {
		url.Path = "/" + url.Path
	}

	// Verify that the scheme is something we support in our common downloader.
	supported := []string{"file", "http", "https"}
	found := false
	for _, s := range supported {
		if url.Scheme == s {
			found = true
			break
		}
	}

	if !found {
		return "", fmt.Errorf("Unsupported URL scheme: %s", url.Scheme)
	}

	// if cleaned filepath does not exist, error out now.
	if url.Scheme == "file" {
		if _, err := os.Stat(url.Path); err != nil {
			return "", fmt.Errorf("The filepath doesn't exist either as a "+
				"relative or an absolute path: %s", err)
		}
	}
	return url.String(), nil
}
<|endoftext|>


<commit_before>package topic

import (
	"os"
	"sync"
	"testing"

	"gopkg.in/logex.v1"

	"github.com/chzyer/mmq/internal/utils"
	"github.com/chzyer/mmq/mmq"
)

var (
	c *Config
)

func init() {
	c = new(Config)
	c.ChunkBit = 22
	c.Root = "/data/mmq/test/topic"
	os.MkdirAll(c.Root, 0777)
	os.RemoveAll(c.Root)
}

func BenchmarkTopicPut(b *testing.B) {
	topic, err := New("bench-put", c)
	if err != nil {
		b.Fatal(err)
	}
	msg := mmq.NewMessageByData([]byte(utils.RandString(256)))
	reply := make(chan []error)
	var wg sync.WaitGroup
	go func() {
		for _ = range reply {
			wg.Done()
		}
	}()
	b.ResetTimer()
	buffer := []*mmq.Message{}
	for i := 0; i < b.N; i++ {
		m, _ := mmq.NewMessage(msg.Bytes(), true)
		buffer = append(buffer, m)
		if len(buffer) >= 100 {
			wg.Add(1)
			topic.Put(buffer, reply)
			buffer = nil
		}
	}
	wg.Wait()
}

func TestTopic(t *testing.T) {
	topic, err := New("topicTest", c)
	if err != nil {
		t.Fatal(err)
	}
	var wg sync.WaitGroup
	var testSource = [][]byte{
		[]byte("hello!"),
		[]byte("who are you"),
		[]byte("oo?"),
	}
	wg.Add(len(testSource))
	go func() {
		incoming := make(chan []*mmq.Message, len(testSource))
		errChan := make(chan error)
		topic.Get(0, len(testSource), incoming, errChan)
		idx := 0
		for {
			select {
			case msg := <-incoming:
				for _, m := range msg {
					if string(m.Data) != string(testSource[idx]) {
						t.Error("result not except", string(m.Data))
					}
					wg.Done()
					idx++
				}
			case err := <-errChan:
				logex.Error("get:", err)
			}
		}
	}()
	go func() {
		for _, m := range testSource {
			msg := mmq.NewMessageByData(m)
			errs := topic.PutSync([]*mmq.Message{msg})
			logex.Error(errs)
		}
	}()
	wg.Wait()

}
<commit_msg>add topic get benchmark<commit_after>package topic

import (
	"os"
	"sync"
	"testing"
	"time"

	"gopkg.in/logex.v1"

	"github.com/chzyer/mmq/internal/utils"
	"github.com/chzyer/mmq/mmq"
)

var (
	c *Config
)

func init() {
	c = new(Config)
	c.ChunkBit = 22
	c.Root = "/data/mmq/test/topic"
	os.MkdirAll(c.Root, 0777)
	os.RemoveAll(c.Root)
}

func BenchmarkSleep(b *testing.B) {
	time.Sleep(100 * time.Millisecond)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		time.Sleep(time.Microsecond)
	}
}

func BenchmarkTopicGet(b *testing.B) {
	n := b.N
	topic, err := New("bench-get", c)
	if err != nil {
		b.Fatal(err)
	}

	var wg2 sync.WaitGroup
	replyErrs := make(chan []error)
	go func() {
		for _ = range replyErrs {
			wg2.Done()
		}
	}()
	msg := mmq.NewMessageByData(mmq.NewMessageData([]byte(utils.RandString(256))))
	var buffer []*mmq.Message
	for i := 0; i < n; i++ {
		buffer = append(buffer, msg)
		if len(buffer) > 100 {
			wg2.Add(1)
			topic.Put(buffer, replyErrs)
			buffer = nil
		}
	}
	wg2.Wait()
	close(replyErrs)

	b.ResetTimer()
	reply := make(chan []*mmq.Message, 1024)

	size := 0
	off := int64(0)
	var wg sync.WaitGroup
	go func() {
		for msgs := range reply {
			wg.Add(1)
			for _, m := range msgs {
				off += int64(len(m.Bytes()))
				wg.Done()
			}
			size -= len(msgs)
			wg.Done()
			if size == 0 {
				continue
			}
		}
	}()

	for i := 0; i < n; i++ {
		if size < 100 {
			size++
			continue
		}

		wg.Add(size)
		if err := topic.GetSync(off, size, reply); err != nil {
			b.Fatal(err)
		}
		wg.Wait()
	}
	close(reply)
}

func BenchmarkTopicPut(b *testing.B) {
	topic, err := New("bench-put", c)
	if err != nil {
		b.Fatal(err)
	}
	msg := mmq.NewMessageByData(mmq.NewMessageData([]byte(utils.RandString(256))))
	reply := make(chan []error)
	var wg sync.WaitGroup
	go func() {
		for _ = range reply {
			wg.Done()
		}
	}()
	b.ResetTimer()
	buffer := []*mmq.Message{}
	for i := 0; i < b.N; i++ {
		m, _ := mmq.NewMessage(msg.Bytes(), true)
		buffer = append(buffer, m)
		if len(buffer) >= 100 {
			wg.Add(1)
			topic.Put(buffer, reply)
			buffer = nil
		}
	}
	wg.Wait()
}

func TestTopic(t *testing.T) {
	topic, err := New("topicTest", c)
	if err != nil {
		t.Fatal(err)
	}
	var wg sync.WaitGroup
	var testSource = [][]byte{
		[]byte("hello!"),
		[]byte("who are you"),
		[]byte("oo?"),
	}
	wg.Add(len(testSource))
	go func() {
		incoming := make(chan []*mmq.Message, len(testSource))
		errChan := make(chan error)
		topic.Get(0, len(testSource), incoming, errChan)
		idx := 0
		for {
			select {
			case msg := <-incoming:
				for _, m := range msg {
					if string(m.Data) != string(testSource[idx]) {
						t.Error("result not except", string(m.Data))
					}
					wg.Done()
					idx++
				}
			case err := <-errChan:
				logex.Error("get:", err)
			}
		}
	}()
	go func() {
		for _, m := range testSource {
			msg := mmq.NewMessageByData(mmq.NewMessageData(m))
			errs := topic.PutSync([]*mmq.Message{msg})
			logex.Error(errs)
		}
	}()
	wg.Wait()

}
<|endoftext|>