<commit_before>package objects

type EpochTime int64
type Timestamp float64

const (
	ButtonActionType = "button"
)

// Conversation is a structure that is never used by itself:
// it's re-used to describe a basic conversation profile
// by being embedded in other objects
type Conversation struct {
	ID                 string    `json:"id"`
	Created            EpochTime `json:"created"`
	IsOpen             bool      `json:"is_open"`
	LastRead           string    `json:"last_read,omitempty"`
	Latest             *Message  `json:"latest,omitempty"`
	UnreadCount        int       `json:"unread_count,omitempty"`
	UnreadCountDisplay int       `json:"unread_count_display,omitempty"`
}

type GroupConversation struct {
	Conversation
	Creator       string   `json:"creator"`
	IsArchived    bool     `json:"is_archived"`
	Members       []string `json:"members"`
	Name          string   `json:"name"`
	NumMembers    int      `json:"num_members,omitempty"`
	PreviousNames []string `json:"previous_names"`
	Purpose       Purpose  `json:"purpose"`
	Topic         Topic    `json:"topic"`
}

type Purpose struct {
	Value   string    `json:"value"`
	Creator string    `json:"creator"`
	LastSet EpochTime `json:"last_set"`
}

type Topic struct {
	Value   string    `json:"value"`
	Creator string    `json:"creator"`
	LastSet EpochTime `json:"last_set"`
}

// Action is used in conjunction with message buttons
type Action struct {
	Confirm         *Confirmation   `json:"confirm"`
	DataSource      string          `json:"data_source"`
	MinQueryLength  int             `json:"min_query_length"`
	Name            string          `json:"name"`
	OptionGroups    OptionGroupList `json:"option_groups"`
	Options         OptionList      `json:"options"`
	SelectedOptions OptionList      `json:"selected_options"`
	Style           string          `json:"style"`
	Text            string          `json:"text"`
	Type            string          `json:"type"`
	Value           string          `json:"value"`
}
type ActionList []*Action

type Attachment struct {
	Actions        ActionList          `json:"actions,omitempty"` // for buttons
	AttachmentType string              `json:"attachment_type"`
	AuthorName     string              `json:"author_name"`
	AuthorLink     string              `json:"author_link"`
	AuthorIcon     string              `json:"author_icon"`
	CallbackID     string              `json:"callback_id,omitempty"` // for buttons
	Color          string              `json:"color,omitempty"`
	Fallback       string              `json:"fallback"`
	Fields         AttachmentFieldList `json:"fields"`
	Footer         string              `json:"footer"`
	FooterIcon     string              `json:"footer_icon"`
	ImageURL       string              `json:"image_url"`
	ThumbURL       string              `json:"thumb_url"`
	Pretext        string              `json:"pretext,omitempty"`
	Text           string              `json:"text"`
	Timestamp      Timestamp           `json:"ts"`
	Title          string              `json:"title"`
	TitleLink      string              `json:"title_link"`
}
type AttachmentList []*Attachment

type Channel struct {
	GroupConversation
	IsChannel   bool `json:"is_channel"`
	IsGeneral   bool `json:"is_general"`
	IsMember    bool `json:"is_member"`
	IsOrgShared bool `json:"is_org_shared"`
	IsShared    bool `json:"is_shared"`
}

type ChannelList []*Channel

// Confirmation is used in conjunction with message buttons
type Confirmation struct {
	Title       string `json:"title"`
	Text        string `json:"text"`
	OkText      string `json:"ok_text"`
	DismissText string `json:"dismiss_text"`
}

type Comment struct {
	ID        string    `json:"id,omitempty"`
	Created   EpochTime `json:"created,omitempty"`
	Timestamp EpochTime `json:"timestamp,omitempty"`
	User      string    `json:"user,omitempty"`
	Comment   string    `json:"comment,omitempty"`
}

type Edited struct {
	Timestamp string `json:"ts"`
	User      string `json:"user"`
}

type AttachmentField struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short"`
}
type AttachmentFieldList []*AttachmentField

// Message is a representation of a message, as obtained
// by the RTM or Events API. This is NOT what you use when
// you are posting a message. See ChatService#PostMessage
// and MessageParams for that.
type Message struct {
	Attachments AttachmentList `json:"attachments"`
	Channel     string         `json:"channel"`
	Edited      *Edited        `json:"edited"`
	IsStarred   bool           `json:"is_starred"`
	PinnedTo    []string       `json:"pinned_to"`
	Text        string         `json:"text"`
	Timestamp   string         `json:"ts"`
	Type        string         `json:"type"`
	User        string         `json:"user"`

	// Message Subtypes
	Subtype string `json:"subtype"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// file_share, file_comment, file_mention
	//	File *File `json:"file,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`

	// reactions
	Reactions []ItemReaction `json:"reactions,omitempty"`
}

type MessageList []*Message

type IM struct {
	Conversation
	IsIM          bool   `json:"is_im"`
	User          string `json:"user"`
	IsUserDeleted bool   `json:"is_user_deleted"`
}

type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

type ItemReaction struct {
	Name  string   `json:"name"`
	Count int      `json:"count"`
	Users []string `json:"users"`
}

type UserProfile struct {
	AlwaysActive       bool   `json:"always_active"`
	AvatarHash         string `json:"avatar_hash"`
	Email              string `json:"email"`
	FirstName          string `json:"first_name"`
	Image24            string `json:"image_24"`
	Image32            string `json:"image_32"`
	Image48            string `json:"image_48"`
	Image72            string `json:"image_72"`
	Image192           string `json:"image_192"`
	Image512           string `json:"image_512"`
	LastName           string `json:"last_name"`
	RealName           string `json:"real_name"`
	RealNameNormalized string `json:"real_name_normalized"`
}

type User struct {
	Color             string      `json:"color"`
	Deleted           bool        `json:"deleted"`
	ID                string      `json:"id"`
	IsAdmin           bool        `json:"is_admin"`
	IsBot             bool        `json:"is_bot"`
	IsOwner           bool        `json:"is_owner"`
	IsPrimaryOwner    bool        `json:"is_primary_owner"`
	IsRestricted      bool        `json:"is_restricted"`
	IsUltraRestricted bool        `json:"is_ultra_restricted"`
	Name              string      `json:"name"`
	Profile           UserProfile `json:"profile"`
	RealName          string      `json:"real_name"`
	Status            string      `json:"status,omitempty"`
	TeamID            string      `json:"team_id"`
	TZ                string      `json:"tz,omitempty"`
	TZLabel           string      `json:"tz_label"`
	TZOffset          int         `json:"tz_offset"`
	Update            int         `json:"updated"`
}

// UserDetails is only provided by rtm.start response
type UserDetails struct {
	ID             string    `json:"id"`
	Name           string    `json:"name"`
	Created        EpochTime `json:"created"`
	ManualPresence string    `json:"manual_presence"`
	Prefs          UserPrefs `json:"prefs"`
}

type UserPrefs struct{} // TODO

type UserList []*User

type Presence string

const (
	Presencective Presence = "away"
	PresenceAway  Presence = "away"
)

type UserPresence struct {
	AutoAway        bool     `json:"auto_away,omitempty"`
	ConnectionCount int      `json:"connection_count,omitempty"`
	LastActivity    int      `json:"last_activity,omitempty"`
	ManualAway      bool     `json:"manual_away,omitempty"`
	Online          bool     `json:"online"`
	Presence        Presence `json:"presence"`
}

type Team struct {
	ID                string                 `json:"id"`
	Name              string                 `json:"name"`
	Domain            string                 `json:"domain"`
	EmailDomain       string                 `json:"email_domain"`
	EnterpriseID      string                 `json:"enterprise_id,omitempty"`
	EnterpriseName    string                 `json:"enterprise_name,omitempty"`
	Icon              map[string]interface{} `json:"icon"`
	MsgEditWindowMins int                    `json:"msg_edit_window_mins"`
	OverStorageLimit  bool                   `json:"over_storage_limit"`
	Prefs             interface{}            `json:"prefs"`
	Plan              string                 `json:"plan"`
}

type Group interface{}
type Bot struct {
	ID      string `json:"id"`
	AppID   string `json:"app_id"`
	Deleted bool   `json:"deleted"`
	Name    string `json:"name"`
	Icons   Icons  `json:"icons"`
}

type Icons struct {
	Image36 string `json:"image_36"`
	Image48 string `json:"image_48"`
	Image72 string `json:"image_72"`
}

// File represents a file object (https://api.slack.com/types/file)
type File struct {
	ID           string
	Name         string
	User         string
	Created      int    `json:"created"`   // The created property is a unix timestamp representing when the file was created.
	Timestamp    int    `json:"timestamp"` // Deprecated
	Updated      int    `json:"updated"`   // The updated property (for Post filetypes only) is a unix timestamp of when the Post was last edited.
	MimeType     string `json:"mimetype"`  // The mimetype and filetype props do not have a 1-to-1 mapping, as multiple different files types ('html', 'js', etc.) share the same mime type.
	FileType     string `json:"filetype"`
	PrettyType   string `json:"pretty_type"` // The pretty_type property contains a human-readable version of the type.
	Mode         string `json:"mode"`        // The mode property contains one of hosted, external, snippet or post.
	Editable     bool   `json:"editable"`    // The editable property indicates that files are stored in editable mode.
	IsExternal   bool   `json:"is_external"` // The is_external property indicates whether the master copy of a file is stored within the system or not. If the file is_external, then the url property will point to the externally-hosted master file. Further, the external_type property will indicate what kind of external file it is, e.g. dropbox or gdoc.
	ExternalType string `json:"external_type"`
	Size         int    `json:"size"` // The size parameter is the filesize in bytes. Snippets are limited to a maximum file size of 1 megabyte.

	URL                string `json:"url"`          // Deprecated - never set
	URLDownload        string `json:"url_download"` // Deprecated - never set
	URLPrivate         string `json:"url_private"`
	URLPrivateDownload string `json:"url_private_download"`

	ImageExifRotation string `json:"image_exif_rotation"`
	OriginalWidth     int    `json:"original_w"`
	OriginalHeight    int    `json:"original_h"`
	Thumb64           string `json:"thumb_64"`
	Thumb80           string `json:"thumb_80"`
	Thumb160          string `json:"thumb_160"`
	Thumb360          string `json:"thumb_360"`
	Thumb360Gif       string `json:"thumb_360_gif"`
	Thumb360W         int    `json:"thumb_360_w"`
	Thumb360H         int    `json:"thumb_360_h"`
	Thumb480          string `json:"thumb_480"`
	Thumb480W         int    `json:"thumb_480_w"`
	Thumb480H         int    `json:"thumb_480_h"`
	Thumb720          string `json:"thumb_720"`
	Thumb720W         int    `json:"thumb_720_w"`
	Thumb720H         int    `json:"thumb_720_h"`
	Thumb960          string `json:"thumb_960"`
	Thumb960W         int    `json:"thumb_960_w"`
	Thumb960H         int    `json:"thumb_960_h"`
	Thumb1024         string `json:"thumb_1024"`
	Thumb1024W        int    `json:"thumb_1024_w"`
	Thumb1024H        int    `json:"thumb_1024_h"`

	Permalink       string `json:"permalink"`
	PermalinkPublic string `json:"permalink_public"`

	EditLink         string `json:"edit_link"`
	Preview          string `json:"preview"`
	PreviewHighlight string `json:"preview_highlight"`
	Lines            int    `json:"lines"`
	LinesMore        int    `json:"lines_more"`

	IsPublic        bool     `json:"is_public"`
	PublicURLShared bool     `json:"public_url_shared"`
	Channels        []string `json:"channels"`
	Groups          []string `json:"groups"`
	IMs             []string `json:"ims"`
	InitialComment  Comment  `json:"initial_comment"`
	CommentsCount   int      `json:"comments_count"`
	NumStars        int      `json:"num_stars"`
	IsStarred       bool     `json:"is_starred"`
}

type Option struct {
	Text        string `json:"text"`
	Value       string `json:"value"`
	Description string `json:"description"`
}
type OptionList []*Option

type OptionGroup struct {
	Text    string     `json:"text"`
	Options OptionList `json:"options"`
}
type OptionGroupList []*OptionGroup
<commit_msg>specify omit empty<commit_after>package objects

type EpochTime int64
type Timestamp float64

const (
	ButtonActionType = "button"
)

// Conversation is a structure that is never used by itself:
// it's re-used to describe a basic conversation profile
// by being embedded in other objects
type Conversation struct {
	ID                 string    `json:"id"`
	Created            EpochTime `json:"created"`
	IsOpen             bool      `json:"is_open"`
	LastRead           string    `json:"last_read,omitempty"`
	Latest             *Message  `json:"latest,omitempty"`
	UnreadCount        int       `json:"unread_count,omitempty"`
	UnreadCountDisplay int       `json:"unread_count_display,omitempty"`
}

type GroupConversation struct {
	Conversation
	Creator       string   `json:"creator"`
	IsArchived    bool     `json:"is_archived"`
	Members       []string `json:"members"`
	Name          string   `json:"name"`
	NumMembers    int      `json:"num_members,omitempty"`
	PreviousNames []string `json:"previous_names"`
	Purpose       Purpose  `json:"purpose"`
	Topic         Topic    `json:"topic"`
}

type Purpose struct {
	Value   string    `json:"value"`
	Creator string    `json:"creator"`
	LastSet EpochTime `json:"last_set"`
}

type Topic struct {
	Value   string    `json:"value"`
	Creator string    `json:"creator"`
	LastSet EpochTime `json:"last_set"`
}

// Action is used in conjunction with message buttons
type Action struct {
	Confirm         *Confirmation   `json:"confirm,omitempty"`
	DataSource      string          `json:"data_source,omitempty"`
	MinQueryLength  int             `json:"min_query_length,omitempty"`
	Name            string          `json:"name"`
	OptionGroups    OptionGroupList `json:"option_groups,omitempty"`
	Options         OptionList      `json:"options,omitempty"`
	SelectedOptions OptionList      `json:"selected_options,omitempty"`
	Style           string          `json:"style,omitempty"`
	Text            string          `json:"text"`
	Type            string          `json:"type"`
	Value           string          `json:"value"`
}
type ActionList []*Action

type Attachment struct {
	Actions        ActionList          `json:"actions,omitempty"` // for buttons
	AttachmentType string              `json:"attachment_type"`
	AuthorName     string              `json:"author_name"`
	AuthorLink     string              `json:"author_link"`
	AuthorIcon     string              `json:"author_icon"`
	CallbackID     string              `json:"callback_id,omitempty"` // for buttons
	Color          string              `json:"color,omitempty"`
	Fallback       string              `json:"fallback"`
	Fields         AttachmentFieldList `json:"fields"`
	Footer         string              `json:"footer"`
	FooterIcon     string              `json:"footer_icon"`
	ImageURL       string              `json:"image_url"`
	ThumbURL       string              `json:"thumb_url"`
	Pretext        string              `json:"pretext,omitempty"`
	Text           string              `json:"text"`
	Timestamp      Timestamp           `json:"ts"`
	Title          string              `json:"title"`
	TitleLink      string              `json:"title_link"`
}
type AttachmentList []*Attachment

type Channel struct {
	GroupConversation
	IsChannel   bool `json:"is_channel"`
	IsGeneral   bool `json:"is_general"`
	IsMember    bool `json:"is_member"`
	IsOrgShared bool `json:"is_org_shared"`
	IsShared    bool `json:"is_shared"`
}

type ChannelList []*Channel

// Confirmation is used in conjunction with message buttons
type Confirmation struct {
	Title       string `json:"title"`
	Text        string `json:"text"`
	OkText      string `json:"ok_text"`
	DismissText string `json:"dismiss_text"`
}

type Comment struct {
	ID        string    `json:"id,omitempty"`
	Created   EpochTime `json:"created,omitempty"`
	Timestamp EpochTime `json:"timestamp,omitempty"`
	User      string    `json:"user,omitempty"`
	Comment   string    `json:"comment,omitempty"`
}

type Edited struct {
	Timestamp string `json:"ts"`
	User      string `json:"user"`
}

type AttachmentField struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short"`
}
type AttachmentFieldList []*AttachmentField

// Message is a representation of a message, as obtained
// by the RTM or Events API. This is NOT what you use when
// you are posting a message. See ChatService#PostMessage
// and MessageParams for that.
type Message struct {
	Attachments AttachmentList `json:"attachments"`
	Channel     string         `json:"channel"`
	Edited      *Edited        `json:"edited"`
	IsStarred   bool           `json:"is_starred"`
	PinnedTo    []string       `json:"pinned_to"`
	Text        string         `json:"text"`
	Timestamp   string         `json:"ts"`
	Type        string         `json:"type"`
	User        string         `json:"user"`

	// Message Subtypes
	Subtype string `json:"subtype"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// file_share, file_comment, file_mention
	//	File *File `json:"file,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`

	// reactions
	Reactions []ItemReaction `json:"reactions,omitempty"`
}

type MessageList []*Message

type IM struct {
	Conversation
	IsIM          bool   `json:"is_im"`
	User          string `json:"user"`
	IsUserDeleted bool   `json:"is_user_deleted"`
}

type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

type ItemReaction struct {
	Name  string   `json:"name"`
	Count int      `json:"count"`
	Users []string `json:"users"`
}

type UserProfile struct {
	AlwaysActive       bool   `json:"always_active"`
	AvatarHash         string `json:"avatar_hash"`
	Email              string `json:"email"`
	FirstName          string `json:"first_name"`
	Image24            string `json:"image_24"`
	Image32            string `json:"image_32"`
	Image48            string `json:"image_48"`
	Image72            string `json:"image_72"`
	Image192           string `json:"image_192"`
	Image512           string `json:"image_512"`
	LastName           string `json:"last_name"`
	RealName           string `json:"real_name"`
	RealNameNormalized string `json:"real_name_normalized"`
}

type User struct {
	Color             string      `json:"color"`
	Deleted           bool        `json:"deleted"`
	ID                string      `json:"id"`
	IsAdmin           bool        `json:"is_admin"`
	IsBot             bool        `json:"is_bot"`
	IsOwner           bool        `json:"is_owner"`
	IsPrimaryOwner    bool        `json:"is_primary_owner"`
	IsRestricted      bool        `json:"is_restricted"`
	IsUltraRestricted bool        `json:"is_ultra_restricted"`
	Name              string      `json:"name"`
	Profile           UserProfile `json:"profile"`
	RealName          string      `json:"real_name"`
	Status            string      `json:"status,omitempty"`
	TeamID            string      `json:"team_id"`
	TZ                string      `json:"tz,omitempty"`
	TZLabel           string      `json:"tz_label"`
	TZOffset          int         `json:"tz_offset"`
	Update            int         `json:"updated"`
}

// UserDetails is only provided by rtm.start response
type UserDetails struct {
	ID             string    `json:"id"`
	Name           string    `json:"name"`
	Created        EpochTime `json:"created"`
	ManualPresence string    `json:"manual_presence"`
	Prefs          UserPrefs `json:"prefs"`
}

type UserPrefs struct{} // TODO

type UserList []*User

type Presence string

const (
	Presencective Presence = "away"
	PresenceAway  Presence = "away"
)

type UserPresence struct {
	AutoAway        bool     `json:"auto_away,omitempty"`
	ConnectionCount int      `json:"connection_count,omitempty"`
	LastActivity    int      `json:"last_activity,omitempty"`
	ManualAway      bool     `json:"manual_away,omitempty"`
	Online          bool     `json:"online"`
	Presence        Presence `json:"presence"`
}

type Team struct {
	ID                string                 `json:"id"`
	Name              string                 `json:"name"`
	Domain            string                 `json:"domain"`
	EmailDomain       string                 `json:"email_domain"`
	EnterpriseID      string                 `json:"enterprise_id,omitempty"`
	EnterpriseName    string                 `json:"enterprise_name,omitempty"`
	Icon              map[string]interface{} `json:"icon"`
	MsgEditWindowMins int                    `json:"msg_edit_window_mins"`
	OverStorageLimit  bool                   `json:"over_storage_limit"`
	Prefs             interface{}            `json:"prefs"`
	Plan              string                 `json:"plan"`
}

type Group interface{}
type Bot struct {
	ID      string `json:"id"`
	AppID   string `json:"app_id"`
	Deleted bool   `json:"deleted"`
	Name    string `json:"name"`
	Icons   Icons  `json:"icons"`
}

type Icons struct {
	Image36 string `json:"image_36"`
	Image48 string `json:"image_48"`
	Image72 string `json:"image_72"`
}

// File represents a file object (https://api.slack.com/types/file)
type File struct {
	ID           string
	Name         string
	User         string
	Created      int    `json:"created"`   // The created property is a unix timestamp representing when the file was created.
	Timestamp    int    `json:"timestamp"` // Deprecated
	Updated      int    `json:"updated"`   // The updated property (for Post filetypes only) is a unix timestamp of when the Post was last edited.
	MimeType     string `json:"mimetype"`  // The mimetype and filetype props do not have a 1-to-1 mapping, as multiple different files types ('html', 'js', etc.) share the same mime type.
	FileType     string `json:"filetype"`
	PrettyType   string `json:"pretty_type"` // The pretty_type property contains a human-readable version of the type.
	Mode         string `json:"mode"`        // The mode property contains one of hosted, external, snippet or post.
	Editable     bool   `json:"editable"`    // The editable property indicates that files are stored in editable mode.
	IsExternal   bool   `json:"is_external"` // The is_external property indicates whether the master copy of a file is stored within the system or not. If the file is_external, then the url property will point to the externally-hosted master file. Further, the external_type property will indicate what kind of external file it is, e.g. dropbox or gdoc.
	ExternalType string `json:"external_type"`
	Size         int    `json:"size"` // The size parameter is the filesize in bytes. Snippets are limited to a maximum file size of 1 megabyte.

	URL                string `json:"url"`          // Deprecated - never set
	URLDownload        string `json:"url_download"` // Deprecated - never set
	URLPrivate         string `json:"url_private"`
	URLPrivateDownload string `json:"url_private_download"`

	ImageExifRotation string `json:"image_exif_rotation"`
	OriginalWidth     int    `json:"original_w"`
	OriginalHeight    int    `json:"original_h"`
	Thumb64           string `json:"thumb_64"`
	Thumb80           string `json:"thumb_80"`
	Thumb160          string `json:"thumb_160"`
	Thumb360          string `json:"thumb_360"`
	Thumb360Gif       string `json:"thumb_360_gif"`
	Thumb360W         int    `json:"thumb_360_w"`
	Thumb360H         int    `json:"thumb_360_h"`
	Thumb480          string `json:"thumb_480"`
	Thumb480W         int    `json:"thumb_480_w"`
	Thumb480H         int    `json:"thumb_480_h"`
	Thumb720          string `json:"thumb_720"`
	Thumb720W         int    `json:"thumb_720_w"`
	Thumb720H         int    `json:"thumb_720_h"`
	Thumb960          string `json:"thumb_960"`
	Thumb960W         int    `json:"thumb_960_w"`
	Thumb960H         int    `json:"thumb_960_h"`
	Thumb1024         string `json:"thumb_1024"`
	Thumb1024W        int    `json:"thumb_1024_w"`
	Thumb1024H        int    `json:"thumb_1024_h"`

	Permalink       string `json:"permalink"`
	PermalinkPublic string `json:"permalink_public"`

	EditLink         string `json:"edit_link"`
	Preview          string `json:"preview"`
	PreviewHighlight string `json:"preview_highlight"`
	Lines            int    `json:"lines"`
	LinesMore        int    `json:"lines_more"`

	IsPublic        bool     `json:"is_public"`
	PublicURLShared bool     `json:"public_url_shared"`
	Channels        []string `json:"channels"`
	Groups          []string `json:"groups"`
	IMs             []string `json:"ims"`
	InitialComment  Comment  `json:"initial_comment"`
	CommentsCount   int      `json:"comments_count"`
	NumStars        int      `json:"num_stars"`
	IsStarred       bool     `json:"is_starred"`
}

type Option struct {
	Text        string `json:"text"`
	Value       string `json:"value"`
	Description string `json:"description"`
}
type OptionList []*Option

type OptionGroup struct {
	Text    string     `json:"text"`
	Options OptionList `json:"options"`
}
type OptionGroupList []*OptionGroup
<|endoftext|>


<commit_before>package oci // import "github.com/docker/docker/oci"

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/devices"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

// Device transforms a libcontainer configs.Device to a specs.LinuxDevice object.
func Device(d *configs.Device) specs.LinuxDevice {
	return specs.LinuxDevice{
		Type:     string(d.Type),
		Path:     d.Path,
		Major:    d.Major,
		Minor:    d.Minor,
		FileMode: fmPtr(int64(d.FileMode)),
		UID:      u32Ptr(int64(d.Uid)),
		GID:      u32Ptr(int64(d.Gid)),
	}
}

func deviceCgroup(d *configs.Device) specs.LinuxDeviceCgroup {
	t := string(d.Type)
	return specs.LinuxDeviceCgroup{
		Allow:  true,
		Type:   t,
		Major:  &d.Major,
		Minor:  &d.Minor,
		Access: d.Permissions,
	}
}

// DevicesFromPath computes a list of devices and device permissions from paths (pathOnHost and pathInContainer) and cgroup permissions.
func DevicesFromPath(pathOnHost, pathInContainer, cgroupPermissions string) (devs []specs.LinuxDevice, devPermissions []specs.LinuxDeviceCgroup, err error) {
	resolvedPathOnHost := pathOnHost

	// check if it is a symbolic link
	if src, e := os.Lstat(pathOnHost); e == nil && src.Mode()&os.ModeSymlink == os.ModeSymlink {
		if linkedPathOnHost, e := filepath.EvalSymlinks(pathOnHost); e == nil {
			resolvedPathOnHost = linkedPathOnHost
		}
	}

	device, err := devices.DeviceFromPath(resolvedPathOnHost, cgroupPermissions)
	// if there was no error, return the device
	if err == nil {
		device.Path = pathInContainer
		return append(devs, Device(device)), append(devPermissions, deviceCgroup(device)), nil
	}

	// if the device is not a device node
	// try to see if it's a directory holding many devices
	if err == devices.ErrNotADevice {

		// check if it is a directory
		if src, e := os.Stat(resolvedPathOnHost); e == nil && src.IsDir() {

			// mount the internal devices recursively
			filepath.Walk(resolvedPathOnHost, func(dpath string, f os.FileInfo, e error) error {
				childDevice, e := devices.DeviceFromPath(dpath, cgroupPermissions)
				if e != nil {
					// ignore the device
					return nil
				}

				// add the device to userSpecified devices
				childDevice.Path = strings.Replace(dpath, resolvedPathOnHost, pathInContainer, 1)
				devs = append(devs, Device(childDevice))
				devPermissions = append(devPermissions, deviceCgroup(childDevice))

				return nil
			})
		}
	}

	if len(devs) > 0 {
		return devs, devPermissions, nil
	}

	return devs, devPermissions, fmt.Errorf("error gathering device information while adding custom device %q: %s", pathOnHost, err)
}
<commit_msg>oci: fix SA4009: argument e is overwritten before first use (staticcheck)<commit_after>package oci // import "github.com/docker/docker/oci"

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/devices"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

// Device transforms a libcontainer configs.Device to a specs.LinuxDevice object.
func Device(d *configs.Device) specs.LinuxDevice {
	return specs.LinuxDevice{
		Type:     string(d.Type),
		Path:     d.Path,
		Major:    d.Major,
		Minor:    d.Minor,
		FileMode: fmPtr(int64(d.FileMode)),
		UID:      u32Ptr(int64(d.Uid)),
		GID:      u32Ptr(int64(d.Gid)),
	}
}

func deviceCgroup(d *configs.Device) specs.LinuxDeviceCgroup {
	t := string(d.Type)
	return specs.LinuxDeviceCgroup{
		Allow:  true,
		Type:   t,
		Major:  &d.Major,
		Minor:  &d.Minor,
		Access: d.Permissions,
	}
}

// DevicesFromPath computes a list of devices and device permissions from paths (pathOnHost and pathInContainer) and cgroup permissions.
func DevicesFromPath(pathOnHost, pathInContainer, cgroupPermissions string) (devs []specs.LinuxDevice, devPermissions []specs.LinuxDeviceCgroup, err error) {
	resolvedPathOnHost := pathOnHost

	// check if it is a symbolic link
	if src, e := os.Lstat(pathOnHost); e == nil && src.Mode()&os.ModeSymlink == os.ModeSymlink {
		if linkedPathOnHost, e := filepath.EvalSymlinks(pathOnHost); e == nil {
			resolvedPathOnHost = linkedPathOnHost
		}
	}

	device, err := devices.DeviceFromPath(resolvedPathOnHost, cgroupPermissions)
	// if there was no error, return the device
	if err == nil {
		device.Path = pathInContainer
		return append(devs, Device(device)), append(devPermissions, deviceCgroup(device)), nil
	}

	// if the device is not a device node
	// try to see if it's a directory holding many devices
	if err == devices.ErrNotADevice {

		// check if it is a directory
		if src, e := os.Stat(resolvedPathOnHost); e == nil && src.IsDir() {

			// mount the internal devices recursively
			// TODO check if additional errors should be handled or logged
			_ = filepath.Walk(resolvedPathOnHost, func(dpath string, f os.FileInfo, _ error) error {
				childDevice, e := devices.DeviceFromPath(dpath, cgroupPermissions)
				if e != nil {
					// ignore the device
					return nil
				}

				// add the device to userSpecified devices
				childDevice.Path = strings.Replace(dpath, resolvedPathOnHost, pathInContainer, 1)
				devs = append(devs, Device(childDevice))
				devPermissions = append(devPermissions, deviceCgroup(childDevice))

				return nil
			})
		}
	}

	if len(devs) > 0 {
		return devs, devPermissions, nil
	}

	return devs, devPermissions, fmt.Errorf("error gathering device information while adding custom device %q: %s", pathOnHost, err)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
)

// コマンドの使い方
func usage() {
	cmd := os.Args[0]
	fmt.Fprintf(os.Stderr, "usage: %s [options] [file...]\n", filepath.Base(cmd))
	flag.PrintDefaults()
	os.Exit(0)
}

// Dockerfile の実行
func runDockerfile(path string) error {
	var file *os.File
	var err error

	if len(path) == 0 {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			// エラー処理をする
			return err
		}
		defer file.Close()
	}

	scanner := bufio.NewScanner(file)

	// 正規表現のコンパイル
	re := regexp.MustCompile("(?i)^ *(ONBUILD +)?([A-Z]+) +([^#]+)")
	re_args := regexp.MustCompile("(?i)^([^ ]+) +(.+)")

	var workdir string
	lineno := 0

	for scanner.Scan() {
		line := scanner.Text()
		lineno += 1
		match := re.FindStringSubmatch(line)
		if match != nil {
			//onbuild := match[1]
			instruction := match[2]
			args := match[3]

			switch instruction {
			case "FROM":
				// 何もしない
			case "MAINTAINER":
				// 何もしない
			case "RUN":
				// スクリプトを実行
				cmd := exec.Command("/bin/sh", "-c", args)
				cmd.Dir = workdir
				out, err := cmd.Output()
				if err != nil {
					fmt.Fprintf(os.Stderr, "%s:%d: %s : %s\n", path, lineno, err, args)
					return err
				}
				fmt.Printf("%s", out)
			case "CMD":
			case "EXPOSE":
				// iptables があれば書換える
			case "ENV":
				// 環境変数を設定
				match_args := re_args.FindStringSubmatch(args)
				if match_args != nil {
					os.Setenv(match_args[1], match_args[2])
				}
			case "ADD":
				// ファイルを追加
			case "ENTRYPOINT":
			case "VOLUME":
				// 何もしない
			case "USER":
				// ユーザを設定
			case "WORKDIR":
				// 作業ディレクトリを変更
				workdir = args
			}
		}
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	return err
}

func main() {
	help := flag.Bool("h", false, "help")
	flag.Parse()

	if *help {
		usage()
	}

	if len(flag.Args()) == 0 {
		runDockerfile("Dockerfile")
	} else {
		for _, v := range flag.Args() {
			runDockerfile(v)
		}
	}
}
<commit_msg>-H ホスト名 を指定すると RUN を ssh経由でリモート実行するようにした<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
)

type options struct {
	host string
}

// コマンドの使い方
func usage() {
	cmd := os.Args[0]
	fmt.Fprintf(os.Stderr, "usage: %s [options] [file...]\n", filepath.Base(cmd))
	flag.PrintDefaults()
	os.Exit(0)
}

// Dockerfile の実行
func runDockerfile(path string, opts *options) error {
	var file *os.File
	var err error

	if len(path) == 0 {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			// エラー処理をする
			return err
		}
		defer file.Close()
	}

	scanner := bufio.NewScanner(file)

	// 正規表現のコンパイル
	re := regexp.MustCompile("(?i)^ *(ONBUILD +)?([A-Z]+) +([^#]+)")
	re_args := regexp.MustCompile("(?i)^([^ ]+) +(.+)")

	var workdir string
	lineno := 0

	command := []string{"/bin/sh", "-c"}

	if 0 < len(opts.host) {
		ssh, err := exec.LookPath("ssh")
		if err != nil {
			return err
		}
		command = []string{ssh, opts.host}
	}

	for scanner.Scan() {
		line := scanner.Text()
		lineno += 1
		match := re.FindStringSubmatch(line)
		if match != nil {
			//onbuild := match[1]
			instruction := match[2]
			args := match[3]

			switch instruction {
			case "FROM":
				// 何もしない
			case "MAINTAINER":
				// 何もしない
			case "RUN":
				// スクリプトを実行
				if 0 < len(workdir) {
					args = fmt.Sprintf("cd %s; %s", workdir, args)
				}
				cmd := exec.Command(command[0], append(command[1:], args)...)
				out, err := cmd.Output()
				if err != nil {
					fmt.Fprintf(os.Stderr, "%s:%d: %s : %s\n", path, lineno, err, args)
					return err
				}
				fmt.Printf("%s", out)
			case "CMD":
			case "EXPOSE":
				// iptables があれば書換える
			case "ENV":
				// 環境変数を設定
				match_args := re_args.FindStringSubmatch(args)
				if match_args != nil {
					os.Setenv(match_args[1], match_args[2])
				}
			case "ADD":
				// ファイルを追加
			case "ENTRYPOINT":
			case "VOLUME":
				// 何もしない
			case "USER":
				// ユーザを設定
			case "WORKDIR":
				// 作業ディレクトリを変更
				workdir = args
			}
		}
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	return err
}

func main() {
	var opts options

	flag.StringVar(&opts.host, "H", "", "host")
	help := flag.Bool("h", false, "help")
	flag.Parse()

	if *help {
		usage()
	}

	if len(flag.Args()) == 0 {
		runDockerfile("Dockerfile", &opts)
	} else {
		for _, v := range flag.Args() {
			runDockerfile(v, &opts)
		}
	}
}
<|endoftext|>


<commit_before>package ogdatv21

import (
	"bufio"
	"code.google.com/p/go-uuid/uuid"
	"encoding/csv"
	"encoding/json"
	"github.com/the42/ogdat"
	"io"
	"log"
	"net/url"
	"os"
	"strconv"
	"time"
)

const OGDTimeSpecifier = "2006-01-02T15:04:05" // RFC 3339 = ISO 8601 ohne Zeitzone
const ogdatv21specfile = "ogdat_spec-2.1.csv"
const (
	OGDTime2 = time.RFC3339Nano
	OGDTime3 = time.RFC3339
	OGDTime1 = OGDTimeSpecifier
	OGDTimeUnknow
)

var specmap map[int]*ogdat.Beschreibung

type Kategorie struct {
	NumID       int `json:"-"`
	ID          string
	PrettyName  string `json:"-"`
	RDFProperty string `json:"-"`
}

func (kat *Kategorie) String() string {
	return kat.PrettyName
}

var (
	Arbeit           = Kategorie{NumID: 1, ID: "arbeit", PrettyName: "Arbeit", RDFProperty: ""}
	Bevoelkerung     = Kategorie{NumID: 2, ID: "bevölkerung", PrettyName: "Bevölkerung", RDFProperty: ""}
	BildungForschung = Kategorie{NumID: 3, ID: "bildung-und-forschung", PrettyName: "Bildung und Forschung", RDFProperty: ""}
	FinanzRW         = Kategorie{NumID: 4, ID: "finanzen-und-rechnungswesen", PrettyName: "Finanzen und Rechnungswesen", RDFProperty: ""}
	GeographPlanung  = Kategorie{NumID: 5, ID: "geographie-und-planung", PrettyName: "Geographie und Planung", RDFProperty: ""}
	GesellSoziales   = Kategorie{NumID: 6, ID: "gesellschaft-und-soziales", PrettyName: "Gesellschaft und Soziales", RDFProperty: ""}
	Gesundheit       = Kategorie{NumID: 7, ID: "gesundheit", PrettyName: "Gesundheit", RDFProperty: ""}
	KunstKultur      = Kategorie{NumID: 8, ID: "kunst-und-kultur", PrettyName: "Kunst und Kultur", RDFProperty: ""}
	LandFW           = Kategorie{NumID: 9, ID: "land-und-forstwirtschaft", PrettyName: "Land und Forstwirtschaft", RDFProperty: ""}
	SportFZ          = Kategorie{NumID: 10, ID: "sport-und-freizeit", PrettyName: "Sport und Freizeit", RDFProperty: ""}
	Umwelt           = Kategorie{NumID: 11, ID: "umwelt", PrettyName: "Umwelt", RDFProperty: ""}
	VerkehrTechnik   = Kategorie{NumID: 12, ID: "verkehr-und-technik", PrettyName: "Verkehr und Technik", RDFProperty: ""}
	VerwaltPol       = Kategorie{NumID: 13, ID: "verwaltung-und-politik", PrettyName: "Verwaltung und Politik", RDFProperty: ""}
	WirtTourism      = Kategorie{NumID: 14, ID: "wirtschaft-und-tourismus", PrettyName: "Wirtschaft und Tourismus", RDFProperty: ""}
)

var categories = []Kategorie{
	Arbeit,
	Bevoelkerung,
	BildungForschung,
	FinanzRW,
	GeographPlanung,
	GesellSoziales,
	Gesundheit,
	KunstKultur,
	LandFW,
	SportFZ,
	Umwelt,
	VerkehrTechnik,
	VerwaltPol,
	WirtTourism,
}

var categorymap = make(map[string]Kategorie)

type Tags string
type ResourceSpecifier string

type Cycle struct {
	NumID                       int
	DomainCode                  string
	MD_MaintenanceFrequencyCode string
	Name_DE                     string
}

var (
	CycCont     = Cycle{1, "001", "continual", "kontinuierlich"}
	CycDaily    = Cycle{2, "002", "daily", "täglich"}
	CycWeekly   = Cycle{3, "003", "weekly", "wöchentlich"}
	CycFortNly  = Cycle{4, "004", "fortnightly", "14-tägig"}
	CycMonthly  = Cycle{5, "005", "monthly", "monatlich"}
	CycQuart    = Cycle{6, "006", "quarterly", "quartalsweise"}
	CycBiAnn    = Cycle{7, "007", "biannually", "halbjährlich"}
	CycAnnually = Cycle{8, "008", "annually", "jährlich"}
	CycNeeded   = Cycle{9, "009", "asNeeded", "nach Bedarf"}
	CycIrreg    = Cycle{10, "010", "irregular", "unregelmäßig"}
	CycNP       = Cycle{11, "011", "notPlanned", "nicht geplant"}
	CycUnknown  = Cycle{12, "012", "unknown", "unbekannt"}
)

var cycles = []Cycle{
	CycCont,
	CycDaily,
	CycWeekly,
	CycFortNly,
	CycMonthly,
	CycQuart,
	CycBiAnn,
	CycAnnually,
	CycNeeded,
	CycIrreg,
	CycNP,
	CycUnknown,
}

type Url struct {
	*url.URL
	Raw string
}

type Identfier struct {
	*uuid.UUID
	Raw string
}

func (id *Identfier) String() string {
	return id.Raw
}

type Time struct {
	*time.Time
	Raw    string
	Format string
}

func (time *Time) String() string {
	return time.Raw
}

func (cyc *Cycle) String() string {
	return cyc.Name_DE
}

func cmpstrtocycle(raw string, cyc Cycle) bool {
	if raw == cyc.Name_DE || raw == cyc.DomainCode || raw == cyc.MD_MaintenanceFrequencyCode {
		return true
	}
	if len(raw) > 0 {
		if i, err := strconv.Atoi(raw); err == nil && i == cyc.NumID {
			return true
		}
	}
	return false
}

func (cyc *Cycle) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	var found bool
	var idx int
	var matchcyc Cycle

	for idx, matchcyc = range cycles {
		if found := cmpstrtocycle(raw, matchcyc); found == true {
			break
		}
	}

	if found {
		*cyc = cycles[idx]
	} else {
		cyc.NumID = -1
		cyc.Name_DE = "**** NON cycle spec **** - " + raw
		cyc.MD_MaintenanceFrequencyCode = cyc.Name_DE
	}
	return nil
}

func (ogdtime *Time) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	ogdtime.Raw = raw

	ogdtime.Format = OGDTime1
	t, err := time.Parse(ogdtime.Format, raw)
	if err != nil {
		ogdtime.Format = OGDTime2
		t, err = time.Parse(ogdtime.Format, raw)
		if err != nil {
			ogdtime.Format = OGDTime3
			t, err = time.Parse(ogdtime.Format, raw)
			if err != nil {
				ogdtime.Format = OGDTimeUnknow
			}
		}
	}
	ogdtime.Time = &t
	return nil
}

func (u *Url) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	u.Raw = raw
	url, _ := url.Parse(raw) // an actuall error is not important. If url can not be parsed, result will be nil, which is fine here
	u.URL = url
	return nil
}

func (id *Identfier) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	id.Raw = string(raw)
	if uuid := uuid.Parse(raw); uuid != nil {
		id.UUID = &uuid
	}
	return nil
}

func (kat *Kategorie) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	corecat, found := categorymap[raw]
	if !found {
		kat.NumID = -1
		kat.ID = raw
		kat.PrettyName = "**** NON core category **** - " + kat.ID
	} else {
		*kat = corecat
	}
	return nil
}

type Extras struct {
	// Core
	Metadata_Identifier Identfier   `json:"metadata_identifier"` // CKAN uses since API Version 2 a UUID V4, cf. https://github.com/okfn/ckan/blob/master/ckan/model/types.py
	Metadata_Modified   *Time       `json:"metadata_modified"`
	Categorization      []Kategorie `json:"categorization"`
	Begin_DateTime      *Time       `json:"begin_datetime"`

	// Optional
	Schema_Name           *string `json:"schema_name"`
	Schema_Language       *string `json:"schema_language"`     // always "ger"
	Schema_Characterset   *string `json:"schema_characterset"` // always "utf8", cf. https://www.ghrsst.org/files/download.php?m=documents&f=ISO%2019115%20.pdf
	MetaData_Linkage      []Url   `json:"metadata_linkage"`
	Attribute_Description *string `json:"attribute_description"`
	Maintainer_Link       *Url    `json:"maintainer_link"`
	Publisher             *string `json:"publisher"`
	Geographich_Toponym   *string `json:"geographic_toponym"`

	/*  ON/EN/ISO 19115:2003: westBL (344) & eastBL (345) & southBL (346) & northBL (347)
	 * TODO: Specifiaction says a WKT of POLYGON should be used, which would make a
	 * POLYGON ((-180.00 -90.00, 180.00 90.00)) but Example states
	 * POLYGON (-180.00 -90.00, 180.00 90.00)
	 * UNDER CLARIFICATION
	 */
	Geographic_BBox  *string `json:"geographic_bbox"`
	End_DateTime     *Time   `json:"end_datetime"`
	Update_Frequency *Cycle  `json:"update_frequency"`
	Lineage_Quality  *string `json:"lineage_quality"`
	EnTitleDesc      *string `json:"en_title_and_desc"`
}

type Resource struct {
	// Core
	URL    *Url              `json:"url"`
	Format ResourceSpecifier `json:"format"`

	// Optional
	Name         *string `json:"name"`
	Created      *Time   `json:"created"`
	LastModified *Time   `json:"last_modified"`

	/*
	 * dcat:bytes a rdf:Property, owl:DatatypeProperty;
	 * rdfs:isDefinedBy <http://www.w3.org/ns/dcat>;
	 * rdfs:label "size in bytes";
	 * rdfs:comment "describe size of resource in bytes";
	 * rdfs:domain dcat:Distribution;
	 * rdfs:range xsd:integer .
	 */
	Size             *string `json:"size"`
	License_Citation *string `json:"license_citation"`
	Language         *string `json:"language"`
	/* Here we have a problem in spec 2.1. which says "nach ISO\IEC 10646-1", which means utf-8, utf-16 and utf-32.
	 * We would certainly support more encodings, as eg.
	 * ISO 19115 / B.5.10 MD_CharacterSetCode<> or
	 * http://www.iana.org/assignments/character-sets/character-sets.xml
	 */
	Encoding *string `json:"characterset"`
}

type MetaData struct {
	// Core
	Title       string `json:"title"`
	Description string `json:"notes"`
	Schlagworte []Tags `json:"tags"`
	Maintainer  string `json:"maintainer"`
	License     string `json:"license"` // Sollte URI des Lizenzdokuments sein

	// nested structs
	Extras   `json:"extras"`
	Resource []Resource `json:"resources"`
}

func loadogdatv21spec(filename string) (specmap map[int]*ogdat.Beschreibung) {
	reader, err := os.Open(filename)

	if err == nil {
		defer reader.Close()
		specmap = make(map[int]*ogdat.Beschreibung)

		// skip the first line as it contains the field description
		bufio.NewReader(reader).ReadLine()

		csvreader := csv.NewReader(reader)
		csvreader.Comma = '|'

		for record, err := csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
			if len(record) < 12 {
				return nil
			}

			id, err := strconv.Atoi(record[0])
			if err != nil {
				return nil
			}
			var occ ogdat.Occurrence
			switch record[12][0] {
			case 'R':
				occ = ogdat.OccRequired
			case 'O':
				occ = ogdat.OccOptional
			}
			descrecord := ogdat.NewBeschreibung(id, occ, ogdat.Version21)
			descrecord.ID = id
			descrecord.Bezeichner = record[1]
			descrecord.OGD_Kurzname = record[2]
			descrecord.CKAN_Feld = record[3]
			descrecord.Anzahl = byte(record[4][0])
			descrecord.Definition_DE = record[5]
			descrecord.Erlauterung = record[6]
			descrecord.Beispiel = record[7]
			descrecord.ONA2270 = record[8]
			descrecord.ISO19115 = record[9]
			descrecord.RDFProperty = record[10]
			descrecord.Definition_EN = record[11]

			specmap[id] = descrecord
		}
		log.Printf("Info: Read %d %s specifiaction records", len(specmap), ogdat.Version21)
	} else {
		log.Printf("Warning: Can not read %s specification records", len(isolangfilemap))
	}
	return
}

func init() {
	for idx, val := range categories {
		categorymap[val.ID] = categories[idx]
	}
	specmap = loadogdatv21spec(ogdatv21specfile)
}
<commit_msg>get rid of bufio reader to read first line, simply use an empty CSV read allow lazy quotes in strings. This is important as strings in the spec file are not delimited but contain intermediate quotes<commit_after>package ogdatv21

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/csv"
	"encoding/json"
	"github.com/the42/ogdat"
	"io"
	"log"
	"net/url"
	"os"
	"strconv"
	"time"
)

const OGDTimeSpecifier = "2006-01-02T15:04:05" // RFC 3339 = ISO 8601 ohne Zeitzone
const ogdatv21specfile = "ogdat_spec-2.1.csv"
const (
	OGDTime2 = time.RFC3339Nano
	OGDTime3 = time.RFC3339
	OGDTime1 = OGDTimeSpecifier
	OGDTimeUnknow
)

var specmap map[int]*ogdat.Beschreibung

type Kategorie struct {
	NumID       int `json:"-"`
	ID          string
	PrettyName  string `json:"-"`
	RDFProperty string `json:"-"`
}

func (kat *Kategorie) String() string {
	return kat.PrettyName
}

var (
	Arbeit           = Kategorie{NumID: 1, ID: "arbeit", PrettyName: "Arbeit", RDFProperty: ""}
	Bevoelkerung     = Kategorie{NumID: 2, ID: "bevölkerung", PrettyName: "Bevölkerung", RDFProperty: ""}
	BildungForschung = Kategorie{NumID: 3, ID: "bildung-und-forschung", PrettyName: "Bildung und Forschung", RDFProperty: ""}
	FinanzRW         = Kategorie{NumID: 4, ID: "finanzen-und-rechnungswesen", PrettyName: "Finanzen und Rechnungswesen", RDFProperty: ""}
	GeographPlanung  = Kategorie{NumID: 5, ID: "geographie-und-planung", PrettyName: "Geographie und Planung", RDFProperty: ""}
	GesellSoziales   = Kategorie{NumID: 6, ID: "gesellschaft-und-soziales", PrettyName: "Gesellschaft und Soziales", RDFProperty: ""}
	Gesundheit       = Kategorie{NumID: 7, ID: "gesundheit", PrettyName: "Gesundheit", RDFProperty: ""}
	KunstKultur      = Kategorie{NumID: 8, ID: "kunst-und-kultur", PrettyName: "Kunst und Kultur", RDFProperty: ""}
	LandFW           = Kategorie{NumID: 9, ID: "land-und-forstwirtschaft", PrettyName: "Land und Forstwirtschaft", RDFProperty: ""}
	SportFZ          = Kategorie{NumID: 10, ID: "sport-und-freizeit", PrettyName: "Sport und Freizeit", RDFProperty: ""}
	Umwelt           = Kategorie{NumID: 11, ID: "umwelt", PrettyName: "Umwelt", RDFProperty: ""}
	VerkehrTechnik   = Kategorie{NumID: 12, ID: "verkehr-und-technik", PrettyName: "Verkehr und Technik", RDFProperty: ""}
	VerwaltPol       = Kategorie{NumID: 13, ID: "verwaltung-und-politik", PrettyName: "Verwaltung und Politik", RDFProperty: ""}
	WirtTourism      = Kategorie{NumID: 14, ID: "wirtschaft-und-tourismus", PrettyName: "Wirtschaft und Tourismus", RDFProperty: ""}
)

var categories = []Kategorie{
	Arbeit,
	Bevoelkerung,
	BildungForschung,
	FinanzRW,
	GeographPlanung,
	GesellSoziales,
	Gesundheit,
	KunstKultur,
	LandFW,
	SportFZ,
	Umwelt,
	VerkehrTechnik,
	VerwaltPol,
	WirtTourism,
}

var categorymap = make(map[string]Kategorie)

type Tags string
type ResourceSpecifier string

type Cycle struct {
	NumID                       int
	DomainCode                  string
	MD_MaintenanceFrequencyCode string
	Name_DE                     string
}

var (
	CycCont     = Cycle{1, "001", "continual", "kontinuierlich"}
	CycDaily    = Cycle{2, "002", "daily", "täglich"}
	CycWeekly   = Cycle{3, "003", "weekly", "wöchentlich"}
	CycFortNly  = Cycle{4, "004", "fortnightly", "14-tägig"}
	CycMonthly  = Cycle{5, "005", "monthly", "monatlich"}
	CycQuart    = Cycle{6, "006", "quarterly", "quartalsweise"}
	CycBiAnn    = Cycle{7, "007", "biannually", "halbjährlich"}
	CycAnnually = Cycle{8, "008", "annually", "jährlich"}
	CycNeeded   = Cycle{9, "009", "asNeeded", "nach Bedarf"}
	CycIrreg    = Cycle{10, "010", "irregular", "unregelmäßig"}
	CycNP       = Cycle{11, "011", "notPlanned", "nicht geplant"}
	CycUnknown  = Cycle{12, "012", "unknown", "unbekannt"}
)

var cycles = []Cycle{
	CycCont,
	CycDaily,
	CycWeekly,
	CycFortNly,
	CycMonthly,
	CycQuart,
	CycBiAnn,
	CycAnnually,
	CycNeeded,
	CycIrreg,
	CycNP,
	CycUnknown,
}

type Url struct {
	*url.URL
	Raw string
}

type Identfier struct {
	*uuid.UUID
	Raw string
}

func (id *Identfier) String() string {
	return id.Raw
}

type Time struct {
	*time.Time
	Raw    string
	Format string
}

func (time *Time) String() string {
	return time.Raw
}

func (cyc *Cycle) String() string {
	return cyc.Name_DE
}

func cmpstrtocycle(raw string, cyc Cycle) bool {
	if raw == cyc.Name_DE || raw == cyc.DomainCode || raw == cyc.MD_MaintenanceFrequencyCode {
		return true
	}
	if len(raw) > 0 {
		if i, err := strconv.Atoi(raw); err == nil && i == cyc.NumID {
			return true
		}
	}
	return false
}

func (cyc *Cycle) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	var found bool
	var idx int
	var matchcyc Cycle

	for idx, matchcyc = range cycles {
		if found := cmpstrtocycle(raw, matchcyc); found == true {
			break
		}
	}

	if found {
		*cyc = cycles[idx]
	} else {
		cyc.NumID = -1
		cyc.Name_DE = "**** NON cycle spec **** - " + raw
		cyc.MD_MaintenanceFrequencyCode = cyc.Name_DE
	}
	return nil
}

func (ogdtime *Time) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	ogdtime.Raw = raw

	ogdtime.Format = OGDTime1
	t, err := time.Parse(ogdtime.Format, raw)
	if err != nil {
		ogdtime.Format = OGDTime2
		t, err = time.Parse(ogdtime.Format, raw)
		if err != nil {
			ogdtime.Format = OGDTime3
			t, err = time.Parse(ogdtime.Format, raw)
			if err != nil {
				ogdtime.Format = OGDTimeUnknow
			}
		}
	}
	ogdtime.Time = &t
	return nil
}

func (u *Url) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	u.Raw = raw
	url, _ := url.Parse(raw) // an actuall error is not important. If url can not be parsed, result will be nil, which is fine here
	u.URL = url
	return nil
}

func (id *Identfier) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	id.Raw = string(raw)
	if uuid := uuid.Parse(raw); uuid != nil {
		id.UUID = &uuid
	}
	return nil
}

func (kat *Kategorie) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	corecat, found := categorymap[raw]
	if !found {
		kat.NumID = -1
		kat.ID = raw
		kat.PrettyName = "**** NON core category **** - " + kat.ID
	} else {
		*kat = corecat
	}
	return nil
}

type Extras struct {
	// Core
	Metadata_Identifier Identfier   `json:"metadata_identifier"` // CKAN uses since API Version 2 a UUID V4, cf. https://github.com/okfn/ckan/blob/master/ckan/model/types.py
	Metadata_Modified   *Time       `json:"metadata_modified"`
	Categorization      []Kategorie `json:"categorization"`
	Begin_DateTime      *Time       `json:"begin_datetime"`

	// Optional
	Schema_Name           *string `json:"schema_name"`
	Schema_Language       *string `json:"schema_language"`     // always "ger"
	Schema_Characterset   *string `json:"schema_characterset"` // always "utf8", cf. https://www.ghrsst.org/files/download.php?m=documents&f=ISO%2019115%20.pdf
	MetaData_Linkage      []Url   `json:"metadata_linkage"`
	Attribute_Description *string `json:"attribute_description"`
	Maintainer_Link       *Url    `json:"maintainer_link"`
	Publisher             *string `json:"publisher"`
	Geographich_Toponym   *string `json:"geographic_toponym"`

	/*  ON/EN/ISO 19115:2003: westBL (344) & eastBL (345) & southBL (346) & northBL (347)
	 * TODO: Specifiaction says a WKT of POLYGON should be used, which would make a
	 * POLYGON ((-180.00 -90.00, 180.00 90.00)) but Example states
	 * POLYGON (-180.00 -90.00, 180.00 90.00)
	 * UNDER CLARIFICATION
	 */
	Geographic_BBox  *string `json:"geographic_bbox"`
	End_DateTime     *Time   `json:"end_datetime"`
	Update_Frequency *Cycle  `json:"update_frequency"`
	Lineage_Quality  *string `json:"lineage_quality"`
	EnTitleDesc      *string `json:"en_title_and_desc"`
}

type Resource struct {
	// Core
	URL    *Url              `json:"url"`
	Format ResourceSpecifier `json:"format"`

	// Optional
	Name         *string `json:"name"`
	Created      *Time   `json:"created"`
	LastModified *Time   `json:"last_modified"`

	/*
	 * dcat:bytes a rdf:Property, owl:DatatypeProperty;
	 * rdfs:isDefinedBy <http://www.w3.org/ns/dcat>;
	 * rdfs:label "size in bytes";
	 * rdfs:comment "describe size of resource in bytes";
	 * rdfs:domain dcat:Distribution;
	 * rdfs:range xsd:integer .
	 */
	Size             *string `json:"size"`
	License_Citation *string `json:"license_citation"`
	Language         *string `json:"language"`
	/* Here we have a problem in spec 2.1. which says "nach ISO\IEC 10646-1", which means utf-8, utf-16 and utf-32.
	 * We would certainly support more encodings, as eg.
	 * ISO 19115 / B.5.10 MD_CharacterSetCode<> or
	 * http://www.iana.org/assignments/character-sets/character-sets.xml
	 */
	Encoding *string `json:"characterset"`
}

type MetaData struct {
	// Core
	Title       string `json:"title"`
	Description string `json:"notes"`
	Schlagworte []Tags `json:"tags"`
	Maintainer  string `json:"maintainer"`
	License     string `json:"license"` // Sollte URI des Lizenzdokuments sein

	// nested structs
	Extras   `json:"extras"`
	Resource []Resource `json:"resources"`
}

func loadogdatv21spec(filename string) (specmap map[int]*ogdat.Beschreibung) {
	reader, err := os.Open(filename)
	if err == nil {
		defer reader.Close()
		specmap = make(map[int]*ogdat.Beschreibung)
		csvreader := csv.NewReader(reader)
		csvreader.Comma = '|'
		csvreader.LazyQuotes = true

		// skip the first line as it contains the field description
		record, err := csvreader.Read()

		for record, err = csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
			id, _ := strconv.Atoi(record[0])
			var occ ogdat.Occurrence
			switch record[12][0] {
			case 'R':
				occ = ogdat.OccRequired
			case 'O':
				occ = ogdat.OccOptional
			}
			descrecord := ogdat.NewBeschreibung(id, occ, ogdat.Version21)

			descrecord.Bezeichner = record[1]
			descrecord.OGD_Kurzname = record[2]
			descrecord.CKAN_Feld = record[3]
			descrecord.Anzahl = byte(record[4][0])
			descrecord.Definition_DE = record[5]
			descrecord.Erlauterung = record[6]
			descrecord.Beispiel = record[7]
			descrecord.ONA2270 = record[8]
			descrecord.ISO19115 = record[9]
			descrecord.RDFProperty = record[10]
			descrecord.Definition_EN = record[11]

			specmap[id] = descrecord
		}
		log.Printf("Info: Read %d %s specifiaction records", len(specmap), ogdat.Version21)
	} else {
		log.Printf("Warning: Can not read %s specification records from file %s", ogdat.Version21, filename)
	}
	return
}

func init() {
	for idx, val := range categories {
		categorymap[val.ID] = categories[idx]
	}
	specmap = loadogdatv21spec(ogdatv21specfile)
}
<|endoftext|>


<commit_before>package ratelimit

import (
	"net/http"
	"time"
)

// TokenBucketStore is an interface for for any storage implementing
// Token Bucket algorithm.
type TokenBucketStore interface {
	InitRate(rate int, window time.Duration)
	Take(key string) (taken bool, remaining int, err error)
}

// HasResetTime is a TokenBucketStore implementation capable of returning
// timestamp of next expected reset time (next available token).
type HasResetTime interface {
	TokenBucketStore

	// TODO: Do we need "key" parameter too? Maybe we do.
	ResetTime() time.Time
}

// KeyFn is a function returning bucket key depending on request data.
type KeyFn func(r *http.Request) string
<commit_msg>Change API again for the sake of performance<commit_after>package ratelimit

import (
	"net/http"
	"time"
)

// TokenBucketStore is an interface for for any storage implementing
// Token Bucket algorithm.
type TokenBucketStore interface {
	InitRate(rate int, window time.Duration)
	Take(key string) (taken bool, remaining int, reset time.Time, err error)
}

// KeyFn is a function returning bucket key depending on request data.
type KeyFn func(r *http.Request) string
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"launchpad.net/gocheck"
)

type PlatformSuite struct{}

var _ = gocheck.Suite(&PlatformSuite{})

func (s *PlatformSuite) SetUpSuite(c *gocheck.C) {
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "platform_tests")
}

func (s *PlatformSuite) TestPlatforms(c *gocheck.C) {
	want := []Platform{
		{Name: "dea"},
		{Name: "pecuniae"},
		{Name: "money"},
		{Name: "raise"},
		{Name: "glass"},
	}
	conn, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	defer conn.Close()
	for _, p := range want {
		conn.Platforms().Insert(p)
		defer conn.Platforms().Remove(p)
	}
	got, err := Platforms()
	c.Assert(err, gocheck.IsNil)
	c.Assert(got, gocheck.DeepEquals, want)
}

func (s *PlatformSuite) TestPlatformsEmpty(c *gocheck.C) {
	got, err := Platforms()
	c.Assert(err, gocheck.IsNil)
	c.Assert(got, gocheck.HasLen, 0)
}

func (s *PlatformSuite) TestGetPlatform(c *gocheck.C) {
	conn, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	defer conn.Close()
	p := Platform{Name: "dea"}
	conn.Platforms().Insert(p)
	defer conn.Platforms().Remove(p)
	got, err := getPlatform(p.Name)
	c.Assert(err, gocheck.IsNil)
	c.Assert(*got, gocheck.DeepEquals, p)
	got, err = getPlatform("WAT")
	c.Assert(got, gocheck.IsNil)
	_, ok := err.(InvalidPlatformError)
	c.Assert(ok, gocheck.Equals, true)
}
<commit_msg>app: drop database on PlatformSuite.TearDownSuite<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"launchpad.net/gocheck"
)

type PlatformSuite struct{}

var _ = gocheck.Suite(&PlatformSuite{})

func (s *PlatformSuite) SetUpSuite(c *gocheck.C) {
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "platform_tests")
}

func (s *PlatformSuite) TearDownSuite(c *gocheck.C) {
	conn, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	conn.Apps().Database.DropDatabase()
	conn.Close()
}

func (s *PlatformSuite) TestPlatforms(c *gocheck.C) {
	want := []Platform{
		{Name: "dea"},
		{Name: "pecuniae"},
		{Name: "money"},
		{Name: "raise"},
		{Name: "glass"},
	}
	conn, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	defer conn.Close()
	for _, p := range want {
		conn.Platforms().Insert(p)
		defer conn.Platforms().Remove(p)
	}
	got, err := Platforms()
	c.Assert(err, gocheck.IsNil)
	c.Assert(got, gocheck.DeepEquals, want)
}

func (s *PlatformSuite) TestPlatformsEmpty(c *gocheck.C) {
	got, err := Platforms()
	c.Assert(err, gocheck.IsNil)
	c.Assert(got, gocheck.HasLen, 0)
}

func (s *PlatformSuite) TestGetPlatform(c *gocheck.C) {
	conn, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	defer conn.Close()
	p := Platform{Name: "dea"}
	conn.Platforms().Insert(p)
	defer conn.Platforms().Remove(p)
	got, err := getPlatform(p.Name)
	c.Assert(err, gocheck.IsNil)
	c.Assert(*got, gocheck.DeepEquals, p)
	got, err = getPlatform("WAT")
	c.Assert(got, gocheck.IsNil)
	_, ok := err.(InvalidPlatformError)
	c.Assert(ok, gocheck.Equals, true)
}
<|endoftext|>


<commit_before>package router

import (
	"io"
	"net/http"

	log "github.com/sirupsen/logrus"

	"fmt"

	"github.com/gorilla/mux"
	sns "github.com/p4tin/goaws/app/gosns"
	sqs "github.com/p4tin/goaws/app/gosqs"
)

// New returns a new router
func New() http.Handler {
	r := mux.NewRouter()

	r.HandleFunc("/", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/{account}", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/queue/{queueName}", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/{account}/{queueName}", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/SimpleNotificationService/{id}.pem", pemHandler).Methods("GET")
	r.HandleFunc("/health", health).Methods("GET")

	return r
}

var routingTable = map[string]http.HandlerFunc{
	// SQS
	"ListQueues":              sqs.ListQueues,
	"CreateQueue":             sqs.CreateQueue,
	"GetQueueAttributes":      sqs.GetQueueAttributes,
	"SetQueueAttributes":      sqs.SetQueueAttributes,
	"SendMessage":             sqs.SendMessage,
	"SendMessageBatch":        sqs.SendMessageBatch,
	"ReceiveMessage":          sqs.ReceiveMessage,
	"DeleteMessage":           sqs.DeleteMessage,
	"DeleteMessageBatch":      sqs.DeleteMessageBatch,
	"GetQueueUrl":             sqs.GetQueueUrl,
	"PurgeQueue":              sqs.PurgeQueue,
	"DeleteQueue":             sqs.DeleteQueue,
	"ChangeMessageVisibility": sqs.ChangeMessageVisibility,

	// SNS
	"ListTopics":                sns.ListTopics,
	"CreateTopic":               sns.CreateTopic,
	"DeleteTopic":               sns.DeleteTopic,
	"Subscribe":                 sns.Subscribe,
	"ConfirmSubscription":       sns.ConfirmSubscription,
	"SetSubscriptionAttributes": sns.SetSubscriptionAttributes,
	"GetSubscriptionAttributes": sns.GetSubscriptionAttributes,
	"ListSubscriptionsByTopic":  sns.ListSubscriptionsByTopic,
	"ListSubscriptions":         sns.ListSubscriptions,
	"Unsubscribe":               sns.Unsubscribe,
	"Publish":                   sns.Publish,
}

func health(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(200)
	fmt.Fprint(w, "OK")
}

func actionHandler(w http.ResponseWriter, req *http.Request) {
	log.WithFields(
		log.Fields{
			"action": req.FormValue("Action"),
			"url":    req.URL,
		}).Debug("Handling URL request")
	fn, ok := routingTable[req.FormValue("Action")]
	if !ok {
		log.Println("Bad Request - Action:", req.FormValue("Action"))
		w.WriteHeader(http.StatusBadRequest)
		io.WriteString(w, "Bad Request")
		return
	}

	http.HandlerFunc(fn).ServeHTTP(w, req)
}

func pemHandler(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Write(sns.PemKEY)
}
<commit_msg>Fix the registration order of health path<commit_after>package router

import (
	"io"
	"net/http"

	log "github.com/sirupsen/logrus"

	"fmt"

	"github.com/gorilla/mux"
	sns "github.com/p4tin/goaws/app/gosns"
	sqs "github.com/p4tin/goaws/app/gosqs"
)

// New returns a new router
func New() http.Handler {
	r := mux.NewRouter()

	r.HandleFunc("/", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/health", health).Methods("GET")
	r.HandleFunc("/{account}", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/queue/{queueName}", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/{account}/{queueName}", actionHandler).Methods("GET", "POST")
	r.HandleFunc("/SimpleNotificationService/{id}.pem", pemHandler).Methods("GET")

	return r
}

var routingTable = map[string]http.HandlerFunc{
	// SQS
	"ListQueues":              sqs.ListQueues,
	"CreateQueue":             sqs.CreateQueue,
	"GetQueueAttributes":      sqs.GetQueueAttributes,
	"SetQueueAttributes":      sqs.SetQueueAttributes,
	"SendMessage":             sqs.SendMessage,
	"SendMessageBatch":        sqs.SendMessageBatch,
	"ReceiveMessage":          sqs.ReceiveMessage,
	"DeleteMessage":           sqs.DeleteMessage,
	"DeleteMessageBatch":      sqs.DeleteMessageBatch,
	"GetQueueUrl":             sqs.GetQueueUrl,
	"PurgeQueue":              sqs.PurgeQueue,
	"DeleteQueue":             sqs.DeleteQueue,
	"ChangeMessageVisibility": sqs.ChangeMessageVisibility,

	// SNS
	"ListTopics":                sns.ListTopics,
	"CreateTopic":               sns.CreateTopic,
	"DeleteTopic":               sns.DeleteTopic,
	"Subscribe":                 sns.Subscribe,
	"ConfirmSubscription":       sns.ConfirmSubscription,
	"SetSubscriptionAttributes": sns.SetSubscriptionAttributes,
	"GetSubscriptionAttributes": sns.GetSubscriptionAttributes,
	"ListSubscriptionsByTopic":  sns.ListSubscriptionsByTopic,
	"ListSubscriptions":         sns.ListSubscriptions,
	"Unsubscribe":               sns.Unsubscribe,
	"Publish":                   sns.Publish,
}

func health(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(200)
	fmt.Fprint(w, "OK")
}

func actionHandler(w http.ResponseWriter, req *http.Request) {
	log.WithFields(
		log.Fields{
			"action": req.FormValue("Action"),
			"url":    req.URL,
		}).Debug("Handling URL request")
	fn, ok := routingTable[req.FormValue("Action")]
	if !ok {
		log.Println("Bad Request - Action:", req.FormValue("Action"))
		w.WriteHeader(http.StatusBadRequest)
		io.WriteString(w, "Bad Request")
		return
	}

	http.HandlerFunc(fn).ServeHTTP(w, req)
}

func pemHandler(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Write(sns.PemKEY)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
)

type NodeResponseOK struct {
	Status  string
	Message map[string]string
	Nonce   string
}

type DashboardServiceSender interface {
	Init() error
	Register() error
	DeRegister() error
	StartBeating() error
	StopBeating()
}

type HTTPDashboardHandler struct {
	RegistrationEndpoint   string
	DeRegistrationEndpoint string
	HeartBeatEndpoint      string
	Secret                 string

	heartBeatStopSentinel bool
}

func reLogin() {
	log.WithFields(logrus.Fields{
		"prefix": "main",
	}).Info("Registering node (again).")
	DashService.StopBeating()
	DashService.DeRegister()

	time.Sleep(30 * time.Second)

	if err := DashService.Register(); err != nil {
		log.WithFields(logrus.Fields{
			"prefix": "main",
		}).Error(err)
	} else {
		go DashService.StartBeating()
	}

	log.WithFields(logrus.Fields{
		"prefix": "main",
	}).Info("Recovering configurations, reloading...")
	doReload()
}

func (h *HTTPDashboardHandler) Init() error {
	h.RegistrationEndpoint = buildConnStr("/register/node")
	h.DeRegistrationEndpoint = buildConnStr("/system/node")
	h.HeartBeatEndpoint = buildConnStr("/register/ping")

	h.Secret = globalConf.NodeSecret
	return nil
}

func (h *HTTPDashboardHandler) Register() error {
	// Get the definitions

	endpoint := h.RegistrationEndpoint
	secret := h.Secret

	log.Debug("Calling: ", endpoint)
	newRequest, err := http.NewRequest("GET", endpoint, nil)
	if err != nil {
		log.Error("Failed to create request: ", err)
	}

	newRequest.Header.Set("authorization", secret)
	newRequest.Header.Set("x-tyk-hostname", HostDetails.Hostname)

	c := &http.Client{
		Timeout: 5 * time.Second,
	}
	resp, err := c.Do(newRequest)
	if err != nil {
		log.Error("Request failed: ", err)
		time.Sleep(time.Second * 5)
		return h.Register()
	}
	if resp.StatusCode != 200 {
		log.Error("Failed to register node, retrying in 5s")
		time.Sleep(time.Second * 5)
		return h.Register()
	}

	defer resp.Body.Close()
	val := NodeResponseOK{}
	if err := json.NewDecoder(resp.Body).Decode(&val); err != nil {
		log.Error("Failed to decode body: ", err)
		return err
	}

	// Set the NodeID
	var found bool
	NodeID, found = val.Message["NodeID"]
	if !found {
		log.Error("Failed to register node, retrying in 5s")
		time.Sleep(time.Second * 5)
		return h.Register()
	}

	log.WithFields(logrus.Fields{
		"prefix": "dashboard",
		"id":     NodeID,
	}).Info("Node registered")

	// Set the nonce
	ServiceNonce = val.Nonce
	log.Debug("Registration Finished: Nonce Set: ", ServiceNonce)

	return nil
}

func (h *HTTPDashboardHandler) StartBeating() error {
	for !h.heartBeatStopSentinel {
		failure := h.SendHeartBeat(h.HeartBeatEndpoint, h.Secret)
		if failure != nil {
			log.Warning(failure)
		}
		time.Sleep(time.Second * 2)
	}

	log.Info("Stopped Heartbeat")
	h.heartBeatStopSentinel = false
	return nil
}

func (h *HTTPDashboardHandler) StopBeating() {
	h.heartBeatStopSentinel = true
}

func (h *HTTPDashboardHandler) SendHeartBeat(endpoint, secret string) error {
	// Get the definitions
	log.Debug("Calling: ", endpoint)
	newRequest, err := http.NewRequest("GET", endpoint, nil)
	if err != nil {
		log.Error("Failed to create request: ", err)
	}

	newRequest.Header.Set("authorization", secret)
	newRequest.Header.Set("x-tyk-nodeid", NodeID)
	newRequest.Header.Set("x-tyk-hostname", HostDetails.Hostname)

	log.Debug("Sending Heartbeat as: ", NodeID)

	newRequest.Header.Set("x-tyk-nonce", ServiceNonce)

	c := &http.Client{
		Timeout: 5 * time.Second,
	}
	resp, err := c.Do(newRequest)
	if err != nil || resp.StatusCode != 200 {
		return errors.New("dashboard is down? Heartbeat is failing")
	}

	defer resp.Body.Close()
	val := NodeResponseOK{}
	if err := json.NewDecoder(resp.Body).Decode(&val); err != nil {
		log.Error("Failed to decode body: ", err)
		return err
	}

	// Set the nonce
	ServiceNonce = val.Nonce
	log.Debug("Heartbeat Finished: Nonce Set: ", ServiceNonce)

	return nil
}

func (h *HTTPDashboardHandler) DeRegister() error {
	// Get the definitions

	endpoint := h.DeRegistrationEndpoint
	secret := h.Secret

	log.Debug("Calling: ", endpoint)
	newRequest, err := http.NewRequest("DELETE", endpoint, nil)
	if err != nil {
		log.Error("Failed to create request: ", err)
	}

	newRequest.Header.Set("authorization", secret)
	newRequest.Header.Set("x-tyk-nodeid", NodeID)
	newRequest.Header.Set("x-tyk-hostname", HostDetails.Hostname)

	log.Info("De-registering: ", NodeID)

	newRequest.Header.Set("x-tyk-nonce", ServiceNonce)

	c := &http.Client{
		Timeout: 5 * time.Second,
	}
	resp, err := c.Do(newRequest)
	if err != nil {
		log.Error("Dashboard is down? Failed fo de-register: ", err)
		return err
	}

	if resp.StatusCode != 200 {
		log.Error("Dashboard is down? Failed fo de-register, incorrect status: ", resp.StatusCode)
		return errors.New("Incorrect status code")
	}

	defer resp.Body.Close()
	val := NodeResponseOK{}
	if err := json.NewDecoder(resp.Body).Decode(&val); err != nil {
		log.Error("Failed to decode body: ", err)
		return err
	}

	// Set the nonce
	ServiceNonce = val.Nonce
	log.Info("De-registered.")

	return nil
}
<commit_msg>dash: don't force a reload after a relogin<commit_after>package main

import (
	"encoding/json"
	"errors"
	"net/http"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

type NodeResponseOK struct {
	Status  string
	Message map[string]string
	Nonce   string
}

type DashboardServiceSender interface {
	Init() error
	Register() error
	DeRegister() error
	StartBeating() error
	StopBeating()
}

type HTTPDashboardHandler struct {
	RegistrationEndpoint   string
	DeRegistrationEndpoint string
	HeartBeatEndpoint      string
	Secret                 string

	heartBeatStopSentinel bool
}

func reLogin() {
	log.WithFields(logrus.Fields{
		"prefix": "main",
	}).Info("Registering node (again).")
	DashService.StopBeating()
	DashService.DeRegister()

	time.Sleep(30 * time.Second)

	if err := DashService.Register(); err != nil {
		log.WithFields(logrus.Fields{
			"prefix": "main",
		}).Error(err)
	} else {
		go DashService.StartBeating()
	}

	log.WithFields(logrus.Fields{
		"prefix": "main",
	}).Info("Recovering configurations, reloading...")
	var wg sync.WaitGroup
	wg.Add(1)
	reloadURLStructure(wg.Done)
	wg.Wait()
}

func (h *HTTPDashboardHandler) Init() error {
	h.RegistrationEndpoint = buildConnStr("/register/node")
	h.DeRegistrationEndpoint = buildConnStr("/system/node")
	h.HeartBeatEndpoint = buildConnStr("/register/ping")

	h.Secret = globalConf.NodeSecret
	return nil
}

func (h *HTTPDashboardHandler) Register() error {
	// Get the definitions

	endpoint := h.RegistrationEndpoint
	secret := h.Secret

	log.Debug("Calling: ", endpoint)
	newRequest, err := http.NewRequest("GET", endpoint, nil)
	if err != nil {
		log.Error("Failed to create request: ", err)
	}

	newRequest.Header.Set("authorization", secret)
	newRequest.Header.Set("x-tyk-hostname", HostDetails.Hostname)

	c := &http.Client{
		Timeout: 5 * time.Second,
	}
	resp, err := c.Do(newRequest)
	if err != nil {
		log.Error("Request failed: ", err)
		time.Sleep(time.Second * 5)
		return h.Register()
	}
	if resp.StatusCode != 200 {
		log.Error("Failed to register node, retrying in 5s")
		time.Sleep(time.Second * 5)
		return h.Register()
	}

	defer resp.Body.Close()
	val := NodeResponseOK{}
	if err := json.NewDecoder(resp.Body).Decode(&val); err != nil {
		log.Error("Failed to decode body: ", err)
		return err
	}

	// Set the NodeID
	var found bool
	NodeID, found = val.Message["NodeID"]
	if !found {
		log.Error("Failed to register node, retrying in 5s")
		time.Sleep(time.Second * 5)
		return h.Register()
	}

	log.WithFields(logrus.Fields{
		"prefix": "dashboard",
		"id":     NodeID,
	}).Info("Node registered")

	// Set the nonce
	ServiceNonce = val.Nonce
	log.Debug("Registration Finished: Nonce Set: ", ServiceNonce)

	return nil
}

func (h *HTTPDashboardHandler) StartBeating() error {
	for !h.heartBeatStopSentinel {
		failure := h.SendHeartBeat(h.HeartBeatEndpoint, h.Secret)
		if failure != nil {
			log.Warning(failure)
		}
		time.Sleep(time.Second * 2)
	}

	log.Info("Stopped Heartbeat")
	h.heartBeatStopSentinel = false
	return nil
}

func (h *HTTPDashboardHandler) StopBeating() {
	h.heartBeatStopSentinel = true
}

func (h *HTTPDashboardHandler) SendHeartBeat(endpoint, secret string) error {
	// Get the definitions
	log.Debug("Calling: ", endpoint)
	newRequest, err := http.NewRequest("GET", endpoint, nil)
	if err != nil {
		log.Error("Failed to create request: ", err)
	}

	newRequest.Header.Set("authorization", secret)
	newRequest.Header.Set("x-tyk-nodeid", NodeID)
	newRequest.Header.Set("x-tyk-hostname", HostDetails.Hostname)

	log.Debug("Sending Heartbeat as: ", NodeID)

	newRequest.Header.Set("x-tyk-nonce", ServiceNonce)

	c := &http.Client{
		Timeout: 5 * time.Second,
	}
	resp, err := c.Do(newRequest)
	if err != nil || resp.StatusCode != 200 {
		return errors.New("dashboard is down? Heartbeat is failing")
	}

	defer resp.Body.Close()
	val := NodeResponseOK{}
	if err := json.NewDecoder(resp.Body).Decode(&val); err != nil {
		log.Error("Failed to decode body: ", err)
		return err
	}

	// Set the nonce
	ServiceNonce = val.Nonce
	log.Debug("Heartbeat Finished: Nonce Set: ", ServiceNonce)

	return nil
}

func (h *HTTPDashboardHandler) DeRegister() error {
	// Get the definitions

	endpoint := h.DeRegistrationEndpoint
	secret := h.Secret

	log.Debug("Calling: ", endpoint)
	newRequest, err := http.NewRequest("DELETE", endpoint, nil)
	if err != nil {
		log.Error("Failed to create request: ", err)
	}

	newRequest.Header.Set("authorization", secret)
	newRequest.Header.Set("x-tyk-nodeid", NodeID)
	newRequest.Header.Set("x-tyk-hostname", HostDetails.Hostname)

	log.Info("De-registering: ", NodeID)

	newRequest.Header.Set("x-tyk-nonce", ServiceNonce)

	c := &http.Client{
		Timeout: 5 * time.Second,
	}
	resp, err := c.Do(newRequest)
	if err != nil {
		log.Error("Dashboard is down? Failed fo de-register: ", err)
		return err
	}

	if resp.StatusCode != 200 {
		log.Error("Dashboard is down? Failed fo de-register, incorrect status: ", resp.StatusCode)
		return errors.New("Incorrect status code")
	}

	defer resp.Body.Close()
	val := NodeResponseOK{}
	if err := json.NewDecoder(resp.Body).Decode(&val); err != nil {
		log.Error("Failed to decode body: ", err)
		return err
	}

	// Set the nonce
	ServiceNonce = val.Nonce
	log.Info("De-registered.")

	return nil
}
<|endoftext|>


<commit_before>package notify

import (
	"encoding/json"
	"fmt"
	"net/url"
	"sync"
	"time"

	"github.com/zouyx/agollo/v3/component"
	"github.com/zouyx/agollo/v3/env/config"

	"github.com/zouyx/agollo/v3/component/log"
	"github.com/zouyx/agollo/v3/env"
	"github.com/zouyx/agollo/v3/protocol/http"
	"github.com/zouyx/agollo/v3/storage"
	"github.com/zouyx/agollo/v3/utils"
)

const (
	longPollInterval = 2 * time.Second //2s

	//notify timeout
	nofityConnectTimeout = 10 * time.Minute //10m

	//同步链接时间
	syncNofityConnectTimeout = 3 * time.Second //3s

	defaultNotificationId = int64(-1)
)

var (
	allNotifications *notificationsMap
)

type notification struct {
	NamespaceName  string `json:"namespaceName"`
	NotificationID int64  `json:"notificationId"`
}

// map[string]int64
type notificationsMap struct {
	notifications sync.Map
}

type apolloNotify struct {
	NotificationID int64  `json:"notificationId"`
	NamespaceName  string `json:"namespaceName"`
}

//InitAllNotifications 初始化notificationsMap
func InitAllNotifications(callback func(namespace string)) {
	appConfig := env.GetPlainAppConfig()
	ns := env.SplitNamespaces(appConfig.NamespaceName, callback)
	allNotifications = &notificationsMap{
		notifications: ns,
	}
}

func (n *notificationsMap) setNotify(namespaceName string, notificationID int64) {
	n.notifications.Store(namespaceName, notificationID)
}

func (n *notificationsMap) getNotify(namespace string) int64 {
	value, ok := n.notifications.Load(namespace)
	if !ok || value == nil {
		return 0
	}
	return value.(int64)
}

func (n *notificationsMap) GetNotifyLen() int {
	s := n.notifications
	l := 0
	s.Range(func(k, v interface{}) bool {
		l++
		return true
	})
	return l
}

func (n *notificationsMap) getNotifies(namespace string) string {
	notificationArr := make([]*notification, 0)
	if namespace == "" {
		n.notifications.Range(func(key, value interface{}) bool {
			namespaceName := key.(string)
			notificationID := value.(int64)
			notificationArr = append(notificationArr,
				&notification{
					NamespaceName:  namespaceName,
					NotificationID: notificationID,
				})
			return true
		})
	} else {
		notify, _ := n.notifications.LoadOrStore(namespace, defaultNotificationId)

		notificationArr = append(notificationArr,
			&notification{
				NamespaceName:  namespace,
				NotificationID: notify.(int64),
			})
	}

	j, err := json.Marshal(notificationArr)

	if err != nil {
		return ""
	}

	return string(j)
}

//ConfigComponent 配置组件
type ConfigComponent struct {
}

//Start 启动配置组件定时器
func (c *ConfigComponent) Start() {
	t2 := time.NewTimer(longPollInterval)
	//long poll for sync
	for {
		select {
		case <-t2.C:
			AsyncConfigs()
			t2.Reset(longPollInterval)
		}
	}
}

//AsyncConfigs 异步同步所有配置文件中配置的namespace配置
func AsyncConfigs() error {
	return syncConfigs(utils.Empty, true)
}

//SyncConfigs 同步同步所有配置文件中配置的namespace配置
func SyncConfigs() error {
	return syncConfigs(utils.Empty, false)
}

//SyncNamespaceConfig 同步同步一个指定的namespace配置
func SyncNamespaceConfig(namespace string) error {
	return syncConfigs(namespace, false)
}

func syncConfigs(namespace string, isAsync bool) error {

	remoteConfigs, err := notifyRemoteConfig(nil, namespace, isAsync)

	if err != nil || len(namespace) == 0 {
		appConfig := env.GetPlainAppConfig()
		loadBackupConfig(appConfig.NamespaceName, appConfig)
	}

	if err != nil {
		return fmt.Errorf("notifySyncConfigServices: %s", err)
	}
	if len(remoteConfigs) == 0 {
		return fmt.Errorf("notifySyncConfigServices: empty remote config")
	}

	updateAllNotifications(remoteConfigs)

	//sync all config
	err = AutoSyncConfigServices(nil)

	if err != nil {
		if namespace != "" {
			return nil
		}
		//first sync fail then load config file
		appConfig := env.GetPlainAppConfig()
		loadBackupConfig(appConfig.NamespaceName, appConfig)
	}

	//sync all config
	return nil
}

func loadBackupConfig(namespace string, appConfig *config.AppConfig) {
	env.SplitNamespaces(namespace, func(namespace string) {
		config, _ := env.LoadConfigFile(appConfig.BackupConfigPath, namespace)
		if config != nil {
			storage.UpdateApolloConfig(config, false)
		}
	})
}

func toApolloConfig(resBody []byte) ([]*apolloNotify, error) {
	remoteConfig := make([]*apolloNotify, 0)

	err := json.Unmarshal(resBody, &remoteConfig)

	if err != nil {
		log.Error("Unmarshal Msg Fail,Error:", err)
		return nil, err
	}
	return remoteConfig, nil
}

func notifyRemoteConfig(newAppConfig *config.AppConfig, namespace string, isAsync bool) ([]*apolloNotify, error) {
	appConfig := env.GetAppConfig(newAppConfig)
	if appConfig == nil {
		panic("can not find apollo config!please confirm!")
	}
	urlSuffix := getNotifyURLSuffix(allNotifications.getNotifies(namespace), appConfig, newAppConfig)

	//seelog.Debugf("allNotifications.getNotifies():%s",allNotifications.getNotifies())

	connectConfig := &env.ConnectConfig{
		URI: urlSuffix,
	}
	if !isAsync {
		connectConfig.Timeout = syncNofityConnectTimeout
	} else {
		connectConfig.Timeout = nofityConnectTimeout
	}
	connectConfig.IsRetry = isAsync
	notifies, err := http.RequestRecovery(appConfig, connectConfig, &http.CallBack{
		SuccessCallBack: func(responseBody []byte) (interface{}, error) {
			return toApolloConfig(responseBody)
		},
		NotModifyCallBack: touchApolloConfigCache,
	})

	if notifies == nil {
		return nil, err
	}

	return notifies.([]*apolloNotify), err
}
func touchApolloConfigCache() error {
	return nil
}

func updateAllNotifications(remoteConfigs []*apolloNotify) {
	for _, remoteConfig := range remoteConfigs {
		if remoteConfig.NamespaceName == "" {
			continue
		}
		if allNotifications.getNotify(remoteConfig.NamespaceName) == 0 {
			continue
		}

		allNotifications.setNotify(remoteConfig.NamespaceName, remoteConfig.NotificationID)
	}
}

//AutoSyncConfigServicesSuccessCallBack 同步配置回调
func AutoSyncConfigServicesSuccessCallBack(responseBody []byte) (o interface{}, err error) {
	apolloConfig, err := env.CreateApolloConfigWithJSON(responseBody)

	if err != nil {
		log.Error("Unmarshal Msg Fail,Error:", err)
		return nil, err
	}
	appConfig := env.GetPlainAppConfig()

	storage.UpdateApolloConfig(apolloConfig, appConfig.GetIsBackupConfig())

	return nil, nil
}

//AutoSyncConfigServices 自动同步配置
func AutoSyncConfigServices(newAppConfig *config.AppConfig) error {
	return autoSyncNamespaceConfigServices(newAppConfig, allNotifications)
}

func autoSyncNamespaceConfigServices(newAppConfig *config.AppConfig, allNotifications *notificationsMap) error {
	appConfig := env.GetAppConfig(newAppConfig)
	if appConfig == nil {
		panic("can not find apollo config!please confirm!")
	}

	var err error
	allNotifications.notifications.Range(func(key, value interface{}) bool {
		namespace := key.(string)
		urlSuffix := component.GetConfigURLSuffix(appConfig, namespace)

		_, err = http.RequestRecovery(appConfig, &env.ConnectConfig{
			URI: urlSuffix,
		}, &http.CallBack{
			SuccessCallBack:   AutoSyncConfigServicesSuccessCallBack,
			NotModifyCallBack: touchApolloConfigCache,
		})
		if err != nil {
			return false
		}
		return true
	})
	return err
}

func getNotifyURLSuffix(notifications string, config *config.AppConfig, newConfig *config.AppConfig) string {
	c := config
	if newConfig != nil {
		c = newConfig
	}
	return fmt.Sprintf("notifications/v2?appId=%s&cluster=%s&notifications=%s",
		url.QueryEscape(c.AppID),
		url.QueryEscape(c.Cluster),
		url.QueryEscape(notifications))
}
<commit_msg>判断remote为空时<commit_after>package notify

import (
	"encoding/json"
	"fmt"
	"net/url"
	"sync"
	"time"

	"github.com/zouyx/agollo/v3/component"
	"github.com/zouyx/agollo/v3/env/config"

	"github.com/zouyx/agollo/v3/component/log"
	"github.com/zouyx/agollo/v3/env"
	"github.com/zouyx/agollo/v3/protocol/http"
	"github.com/zouyx/agollo/v3/storage"
	"github.com/zouyx/agollo/v3/utils"
)

const (
	longPollInterval = 2 * time.Second //2s

	//notify timeout
	nofityConnectTimeout = 10 * time.Minute //10m

	//同步链接时间
	syncNofityConnectTimeout = 3 * time.Second //3s

	defaultNotificationId = int64(-1)
)

var (
	allNotifications *notificationsMap
)

type notification struct {
	NamespaceName  string `json:"namespaceName"`
	NotificationID int64  `json:"notificationId"`
}

// map[string]int64
type notificationsMap struct {
	notifications sync.Map
}

type apolloNotify struct {
	NotificationID int64  `json:"notificationId"`
	NamespaceName  string `json:"namespaceName"`
}

//InitAllNotifications 初始化notificationsMap
func InitAllNotifications(callback func(namespace string)) {
	appConfig := env.GetPlainAppConfig()
	ns := env.SplitNamespaces(appConfig.NamespaceName, callback)
	allNotifications = &notificationsMap{
		notifications: ns,
	}
}

func (n *notificationsMap) setNotify(namespaceName string, notificationID int64) {
	n.notifications.Store(namespaceName, notificationID)
}

func (n *notificationsMap) getNotify(namespace string) int64 {
	value, ok := n.notifications.Load(namespace)
	if !ok || value == nil {
		return 0
	}
	return value.(int64)
}

func (n *notificationsMap) GetNotifyLen() int {
	s := n.notifications
	l := 0
	s.Range(func(k, v interface{}) bool {
		l++
		return true
	})
	return l
}

func (n *notificationsMap) getNotifies(namespace string) string {
	notificationArr := make([]*notification, 0)
	if namespace == "" {
		n.notifications.Range(func(key, value interface{}) bool {
			namespaceName := key.(string)
			notificationID := value.(int64)
			notificationArr = append(notificationArr,
				&notification{
					NamespaceName:  namespaceName,
					NotificationID: notificationID,
				})
			return true
		})
	} else {
		notify, _ := n.notifications.LoadOrStore(namespace, defaultNotificationId)

		notificationArr = append(notificationArr,
			&notification{
				NamespaceName:  namespace,
				NotificationID: notify.(int64),
			})
	}

	j, err := json.Marshal(notificationArr)

	if err != nil {
		return ""
	}

	return string(j)
}

//ConfigComponent 配置组件
type ConfigComponent struct {
}

//Start 启动配置组件定时器
func (c *ConfigComponent) Start() {
	t2 := time.NewTimer(longPollInterval)
	//long poll for sync
	for {
		select {
		case <-t2.C:
			AsyncConfigs()
			t2.Reset(longPollInterval)
		}
	}
}

//AsyncConfigs 异步同步所有配置文件中配置的namespace配置
func AsyncConfigs() error {
	return syncConfigs(utils.Empty, true)
}

//SyncConfigs 同步同步所有配置文件中配置的namespace配置
func SyncConfigs() error {
	return syncConfigs(utils.Empty, false)
}

//SyncNamespaceConfig 同步同步一个指定的namespace配置
func SyncNamespaceConfig(namespace string) error {
	return syncConfigs(namespace, false)
}

func syncConfigs(namespace string, isAsync bool) error {

	remoteConfigs, err := notifyRemoteConfig(nil, namespace, isAsync)

	if err != nil || len(remoteConfigs) == 0 {
		appConfig := env.GetPlainAppConfig()
		loadBackupConfig(appConfig.NamespaceName, appConfig)
	}

	if err != nil {
		return fmt.Errorf("notifySyncConfigServices: %s", err)
	}
	if len(remoteConfigs) == 0 {
		return fmt.Errorf("notifySyncConfigServices: empty remote config")
	}

	updateAllNotifications(remoteConfigs)

	//sync all config
	err = AutoSyncConfigServices(nil)

	if err != nil {
		if namespace != "" {
			return nil
		}
		//first sync fail then load config file
		appConfig := env.GetPlainAppConfig()
		loadBackupConfig(appConfig.NamespaceName, appConfig)
	}

	//sync all config
	return nil
}

func loadBackupConfig(namespace string, appConfig *config.AppConfig) {
	env.SplitNamespaces(namespace, func(namespace string) {
		config, _ := env.LoadConfigFile(appConfig.BackupConfigPath, namespace)
		if config != nil {
			storage.UpdateApolloConfig(config, false)
		}
	})
}

func toApolloConfig(resBody []byte) ([]*apolloNotify, error) {
	remoteConfig := make([]*apolloNotify, 0)

	err := json.Unmarshal(resBody, &remoteConfig)

	if err != nil {
		log.Error("Unmarshal Msg Fail,Error:", err)
		return nil, err
	}
	return remoteConfig, nil
}

func notifyRemoteConfig(newAppConfig *config.AppConfig, namespace string, isAsync bool) ([]*apolloNotify, error) {
	appConfig := env.GetAppConfig(newAppConfig)
	if appConfig == nil {
		panic("can not find apollo config!please confirm!")
	}
	urlSuffix := getNotifyURLSuffix(allNotifications.getNotifies(namespace), appConfig, newAppConfig)

	//seelog.Debugf("allNotifications.getNotifies():%s",allNotifications.getNotifies())

	connectConfig := &env.ConnectConfig{
		URI: urlSuffix,
	}
	if !isAsync {
		connectConfig.Timeout = syncNofityConnectTimeout
	} else {
		connectConfig.Timeout = nofityConnectTimeout
	}
	connectConfig.IsRetry = isAsync
	notifies, err := http.RequestRecovery(appConfig, connectConfig, &http.CallBack{
		SuccessCallBack: func(responseBody []byte) (interface{}, error) {
			return toApolloConfig(responseBody)
		},
		NotModifyCallBack: touchApolloConfigCache,
	})

	if notifies == nil {
		return nil, err
	}

	return notifies.([]*apolloNotify), err
}
func touchApolloConfigCache() error {
	return nil
}

func updateAllNotifications(remoteConfigs []*apolloNotify) {
	for _, remoteConfig := range remoteConfigs {
		if remoteConfig.NamespaceName == "" {
			continue
		}
		if allNotifications.getNotify(remoteConfig.NamespaceName) == 0 {
			continue
		}

		allNotifications.setNotify(remoteConfig.NamespaceName, remoteConfig.NotificationID)
	}
}

//AutoSyncConfigServicesSuccessCallBack 同步配置回调
func AutoSyncConfigServicesSuccessCallBack(responseBody []byte) (o interface{}, err error) {
	apolloConfig, err := env.CreateApolloConfigWithJSON(responseBody)

	if err != nil {
		log.Error("Unmarshal Msg Fail,Error:", err)
		return nil, err
	}
	appConfig := env.GetPlainAppConfig()

	storage.UpdateApolloConfig(apolloConfig, appConfig.GetIsBackupConfig())

	return nil, nil
}

//AutoSyncConfigServices 自动同步配置
func AutoSyncConfigServices(newAppConfig *config.AppConfig) error {
	return autoSyncNamespaceConfigServices(newAppConfig, allNotifications)
}

func autoSyncNamespaceConfigServices(newAppConfig *config.AppConfig, allNotifications *notificationsMap) error {
	appConfig := env.GetAppConfig(newAppConfig)
	if appConfig == nil {
		panic("can not find apollo config!please confirm!")
	}

	var err error
	allNotifications.notifications.Range(func(key, value interface{}) bool {
		namespace := key.(string)
		urlSuffix := component.GetConfigURLSuffix(appConfig, namespace)

		_, err = http.RequestRecovery(appConfig, &env.ConnectConfig{
			URI: urlSuffix,
		}, &http.CallBack{
			SuccessCallBack:   AutoSyncConfigServicesSuccessCallBack,
			NotModifyCallBack: touchApolloConfigCache,
		})
		if err != nil {
			return false
		}
		return true
	})
	return err
}

func getNotifyURLSuffix(notifications string, config *config.AppConfig, newConfig *config.AppConfig) string {
	c := config
	if newConfig != nil {
		c = newConfig
	}
	return fmt.Sprintf("notifications/v2?appId=%s&cluster=%s&notifications=%s",
		url.QueryEscape(c.AppID),
		url.QueryEscape(c.Cluster),
		url.QueryEscape(notifications))
}
<|endoftext|>


<commit_before>package internal

import (
	"encoding/json"
	"strings"

	"github.com/sirupsen/logrus"
)

// CommandValidator -- constructs a config.Value from a string
type CommandValidator struct{}

// Validate -- always returns nil
func (v CommandValidator) Validate(value string) error {
	return v.Value(value).Error
}

// Value -- parses the command config string into a config.Value
//
// if the string you pass to .Value() is a valid JSON string array, it'll be parsed and split into its components.
// otherwise it'll
func (v CommandValidator) Value(raw string) Value {
	var args []string

	if len(raw) == 0 {
		return Value{} // empty value -> use default
	}

	if err := json.Unmarshal([]byte(raw), &args); err == nil {
		// valid JSON list - use as is
		return Value{
			values: args,
		}
	} else if strings.HasPrefix(raw, "[") {
		logrus.WithField("cmd", raw).WithError(err).Error("your command looks like it's in JSON format but contains errors")
		return Value{Error: err}
	}

	// default behaviour: put it into the first slice
	return Value{
		values: []string{raw},
	}
}<commit_msg>updated CommandValidator to be more strict with validating JSON (any string starting with '[' will now be treated as JSON array)<commit_after>package internal

import (
	"encoding/json"
	"fmt"
	"strings"
)

// CommandValidator -- parses and validates Command values
//
// There are two ways to specify commands
// - literal string path
// - JSON arrays (any string starting with the character '[' will be treated as such)
//   invalid JSON strings will cause validation errors
//
// In the first case, the resulting Value will simply be an array with one single item: the string itself
//
// If you want ondevice to call the command in question with predefined arguments, use the JSON variant,
// e.g: `ssh -C` (compressing data) should be declared as `["ssh", "-C"]`.
//
// to use a command starting with '[', wrap it inside a JSON array: `["["]`
type CommandValidator struct{}

// Validate -- returns any errors found while parsing this Command Value
func (v CommandValidator) Validate(value string) error {
	return v.Value(value).Error
}

// Value -- parses the command config string into a config.Value
//
// if the string you pass to .Value() starts with '[', it will be parsed as JSON.
//
// Otherwise the whole string will be put inside the first (and only) array element
func (v CommandValidator) Value(raw string) Value {
	if len(raw) == 0 {
		return Value{} // empty value -> use default
	}

	var rc Value
	if strings.HasPrefix(raw, "[") {
		// the value starts with '[', assume it is JSON
		if err := json.Unmarshal([]byte(raw), &rc.values); err != nil {
			rc.Error = fmt.Errorf("failed to parse JSON command: %s", err.Error())
		}
	} else {
		// default behaviour: put it into the first slice
		rc.values = []string{raw}
	}
	return rc
}<|endoftext|>


<commit_before>package venom

import (
	"sort"
	"strings"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

func sanitize(s string) string {
	return strings.Replace(s, "-", "_", -1)
}

func allKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

func envKeyReplacer(flags *pflag.FlagSet) *strings.Replacer {
	replaceMap := make(map[string]string, flags.NFlag())
	flags.VisitAll(func(f *pflag.Flag) {
		name := strings.ToUpper(f.Name)
		replaceMap[name] = sanitize(name)
	})

	keys := allKeys(replaceMap)

	// Reverse sort keys, this is to make sure foo-bar comes before foo. This is to prevent
	// foo being triggered when foo-bar is given to string replacer.
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	values := make([]string, 0, 2*len(keys))
	for _, k := range keys {
		values = append(values, k, replaceMap[k])
	}

	return strings.NewReplacer(values...)
}

//
// Better version of viper.AutomaticEnv that searches FOO_BAR for every --foo-bar key in
// addition to the default FOO-BAR.
//
// Note that it must be called *after* all flags are added.
//
func AutomaticEnv(flags *pflag.FlagSet, v *viper.Viper) {
	v.SetEnvKeyReplacer(envKeyReplacer(flags))
	v.AutomaticEnv()
}
<commit_msg>Add comment<commit_after>package venom

import (
	"sort"
	"strings"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

func sanitize(s string) string {
	return strings.Replace(s, "-", "_", -1)
}

func allKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

func envKeyReplacer(flags *pflag.FlagSet) *strings.Replacer {
	replaceMap := make(map[string]string, flags.NFlag())
	flags.VisitAll(func(f *pflag.Flag) {
		name := strings.ToUpper(f.Name)
		replaceMap[name] = sanitize(name)
	})

	keys := allKeys(replaceMap)

	// Reverse sort keys, this is to make sure foo-bar comes before foo. This is to prevent
	// foo being triggered when foo-bar is given to string replacer.
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	values := make([]string, 0, 2*len(keys))
	for _, k := range keys {
		values = append(values, k, replaceMap[k])
	}

	return strings.NewReplacer(values...)
}

//
// Better version of viper.AutomaticEnv that searches FOO_BAR for every --foo-bar key in
// addition (?) to the default FOO-BAR.
//
// Note that it must be called *after* all flags are added.
//
func AutomaticEnv(flags *pflag.FlagSet, v *viper.Viper) {
	v.SetEnvKeyReplacer(envKeyReplacer(flags))
	v.AutomaticEnv()
}
<|endoftext|>