<commit_before>package goinsta

import (
	"encoding/json"
	"fmt"
)

// InboxItem is any conversation message.
type InboxItem struct {
	ID            string `json:"item_id"`
	UserID        int64  `json:"user_id"`
	Timestamp     int64  `json:"timestamp"`
	ClientContext string `json:"client_context"`

	// Type there are a few types:
	// text, like, raven_media
	Type string `json:"item_type"`

	// Text is message text.
	Text string `json:"text"`

	// InboxItemLike is the heart that your girlfriend send to you.
	// (or in my case: the heart that my fans sends to me hehe)

	Like string `json:"like"`

	// Media is image or video
	Media struct {
		ID                   string `json:"id"`
		Images               Images `json:"image_versions2"`
		OriginalWidth        int    `json:"original_width"`
		OriginalHeight       int    `json:"original_height"`
		MediaType            int    `json:"media_type"`
		MediaID              int64  `json:"media_id"`
		PlaybackDurationSecs int    `json:"playback_duration_secs"`
		URLExpireAtSecs      int    `json:"url_expire_at_secs"`
		OrganicTrackingToken string `json:"organic_tracking_token"`
	}
}

// Inbox is the direct message inbox.
type Inbox struct {
	inst *Instagram

	Conversations []Conversation `json:"threads"`

	HasNewer            bool  `json:"has_newer"` // TODO
	HasOlder            bool  `json:"has_older"`
	UnseenCount         int   `json:"unseen_count"`
	UnseenCountTs       int64 `json:"unseen_count_ts"`
	BlendedInboxEnabled bool  `json:"blended_inbox_enabled"`
	// this fields are copied from response
	SeqID                int   `json:"seq_id"`
	PendingRequestsTotal int   `json:"pending_requests_total"`
	SnapshotAtMs         int64 `json:"snapshot_at_ms"`
}

type inboxResp struct {
	Inbox                Inbox  `json:"inbox"`
	SeqID                int    `json:"seq_id"`
	PendingRequestsTotal int    `json:"pending_requests_total"`
	SnapshotAtMs         int64  `json:"snapshot_at_ms"`
	Status               string `json:"status"`
}

func newInbox(inst *Instagram) *Inbox {
	return &Inbox{inst: inst}
}

// Sync updates inbox messages.
//
// See example: examples/inbox/sync.go
func (inbox *Inbox) Sync() error {
	insta := inbox.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: urlInbox,
			Query: map[string]string{
				"persistentBadging": "true",
				"use_unified_inbox": "true",
			},
		},
	)
	if err == nil {
		resp := inboxResp{}
		err = json.Unmarshal(body, &resp)
		if err == nil {
			*inbox = resp.Inbox
			inbox.inst = insta
			inbox.SeqID = resp.Inbox.SeqID
			inbox.PendingRequestsTotal = resp.Inbox.PendingRequestsTotal
			inbox.SnapshotAtMs = resp.Inbox.SnapshotAtMs
			for i := range inbox.Conversations {
				inbox.Conversations[i].inst = insta
				inbox.Conversations[i].firstRun = true
			}
		}
	}
	return err
}

type Conversation struct {
	inst     *Instagram
	err      error
	firstRun bool

	ID   string `json:"thread_id"`
	V2ID int64  `json:"thread_v2_id"`
	// Items can be of many types.
	Items                     []InboxItem `json:"items"`
	Title                     string      `json:"thread_title"`
	Users                     []User      `json:"users"`
	LeftUsers                 []User      `json:"left_users"`
	Pending                   bool        `json:"pending"`
	PendingScore              int64       `json:"pending_score"`
	ReshareReceiveCount       int         `json:"reshare_receive_count"`
	ReshareSendCount          int         `json:"reshare_send_count"`
	ViewerID                  int64       `json:"viewer_id"`
	ValuedRequest             bool        `json:"valued_request"`
	LastActivityAt            int64       `json:"last_activity_at"`
	Muted                     bool        `json:"muted"`
	IsPin                     bool        `json:"is_pin"`
	Named                     bool        `json:"named"`
	ThreadType                string      `json:"thread_type"`
	ExpiringMediaSendCount    int         `json:"expiring_media_send_count"`
	ExpiringMediaReceiveCount int         `json:"expiring_media_receive_count"`
	Inviter                   User        `json:"inviter"`
	HasOlder                  bool        `json:"has_older"`
	HasNewer                  bool        `json:"has_newer"`
	LastSeenAt                struct {
		Num7629421016 struct {
			Timestamp string `json:"timestamp"`
			ItemID    string `json:"item_id"`
		} `json:"7629421016"`
	} `json:"last_seen_at"`
	NewestCursor      string `json:"newest_cursor"`
	OldestCursor      string `json:"oldest_cursor"`
	IsSpam            bool   `json:"is_spam"`
	LastPermanentItem Item   `json:"last_permanent_item"`
}

func (c Conversation) Error() error {
	return c.err
}

func (c Conversation) lastItemID() string {
	n := len(c.Items)
	if n == 0 {
		return ""
	}
	return c.Items[n-1].ID
}

// Next loads next set of private messages.
//
// See example: examples/inbox/conversation.go
func (c *Conversation) Next() bool {
	if c.err != nil {
		return false
	}
	if c.firstRun {
		c.firstRun = false
		return true
	}

	insta := c.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlInboxThread, c.ID),
			Query: map[string]string{
				"cursor":            c.lastItemID(),
				"direction":         "older", // go to upper
				"use_unified_inbox": "true",
			},
		},
	)
	if err == nil {
		resp := threadResp{}
		err = json.Unmarshal(body, &resp)
		if err == nil {
			*c = resp.Conversation
			c.inst = insta
			if !c.HasOlder {
				c.err = ErrNoMore
			}
			return true
		}
	}
	c.err = err
	return false
}
<commit_msg>Added Send function<commit_after>package goinsta

import (
	"encoding/json"
	"fmt"
)

// InboxItem is any conversation message.
type InboxItem struct {
	ID            string `json:"item_id"`
	UserID        int64  `json:"user_id"`
	Timestamp     int64  `json:"timestamp"`
	ClientContext string `json:"client_context"`

	// Type there are a few types:
	// text, like, raven_media
	Type string `json:"item_type"`

	// Text is message text.
	Text string `json:"text"`

	// InboxItemLike is the heart that your girlfriend send to you.
	// (or in my case: the heart that my fans sends to me hehe)

	Like string `json:"like"`

	// Media is image or video
	Media struct {
		ID                   string `json:"id"`
		Images               Images `json:"image_versions2"`
		OriginalWidth        int    `json:"original_width"`
		OriginalHeight       int    `json:"original_height"`
		MediaType            int    `json:"media_type"`
		MediaID              int64  `json:"media_id"`
		PlaybackDurationSecs int    `json:"playback_duration_secs"`
		URLExpireAtSecs      int    `json:"url_expire_at_secs"`
		OrganicTrackingToken string `json:"organic_tracking_token"`
	}
}

// Inbox is the direct message inbox.
//
// Inbox contains Conversations. Each conversation has InboxItems.
// InboxItems are the message of the chat.
type Inbox struct {
	inst *Instagram

	Conversations []Conversation `json:"threads"`

	HasNewer            bool  `json:"has_newer"` // TODO
	HasOlder            bool  `json:"has_older"`
	UnseenCount         int   `json:"unseen_count"`
	UnseenCountTs       int64 `json:"unseen_count_ts"`
	BlendedInboxEnabled bool  `json:"blended_inbox_enabled"`
	// this fields are copied from response
	SeqID                int   `json:"seq_id"`
	PendingRequestsTotal int   `json:"pending_requests_total"`
	SnapshotAtMs         int64 `json:"snapshot_at_ms"`
}

type inboxResp struct {
	Inbox                Inbox  `json:"inbox"`
	SeqID                int    `json:"seq_id"`
	PendingRequestsTotal int    `json:"pending_requests_total"`
	SnapshotAtMs         int64  `json:"snapshot_at_ms"`
	Status               string `json:"status"`
}

func newInbox(inst *Instagram) *Inbox {
	return &Inbox{inst: inst}
}

// Sync updates inbox messages.
//
// See example: examples/inbox/sync.go
func (inbox *Inbox) Sync() error {
	// TODO: Next for pagination
	insta := inbox.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: urlInbox,
			Query: map[string]string{
				"persistentBadging": "true",
				"use_unified_inbox": "true",
				"limit":             "0",
			},
		},
	)
	if err == nil {
		resp := inboxResp{}
		err = json.Unmarshal(body, &resp)
		if err == nil {
			*inbox = resp.Inbox
			inbox.inst = insta
			inbox.SeqID = resp.Inbox.SeqID
			inbox.PendingRequestsTotal = resp.Inbox.PendingRequestsTotal
			inbox.SnapshotAtMs = resp.Inbox.SnapshotAtMs
			for i := range inbox.Conversations {
				inbox.Conversations[i].inst = insta
				inbox.Conversations[i].firstRun = true
			}
		}
	}
	return err
}

type Conversation struct {
	inst     *Instagram
	err      error
	firstRun bool

	ID   string `json:"thread_id"`
	V2ID int64  `json:"thread_v2_id"`
	// Items can be of many types.
	Items                     []InboxItem `json:"items"`
	Title                     string      `json:"thread_title"`
	Users                     []User      `json:"users"`
	LeftUsers                 []User      `json:"left_users"`
	Pending                   bool        `json:"pending"`
	PendingScore              int64       `json:"pending_score"`
	ReshareReceiveCount       int         `json:"reshare_receive_count"`
	ReshareSendCount          int         `json:"reshare_send_count"`
	ViewerID                  int64       `json:"viewer_id"`
	ValuedRequest             bool        `json:"valued_request"`
	LastActivityAt            int64       `json:"last_activity_at"`
	Muted                     bool        `json:"muted"`
	IsPin                     bool        `json:"is_pin"`
	Named                     bool        `json:"named"`
	ThreadType                string      `json:"thread_type"`
	ExpiringMediaSendCount    int         `json:"expiring_media_send_count"`
	ExpiringMediaReceiveCount int         `json:"expiring_media_receive_count"`
	Inviter                   User        `json:"inviter"`
	HasOlder                  bool        `json:"has_older"`
	HasNewer                  bool        `json:"has_newer"`
	LastSeenAt                struct {
		Num7629421016 struct {
			Timestamp string `json:"timestamp"`
			ItemID    string `json:"item_id"`
		} `json:"7629421016"`
	} `json:"last_seen_at"`
	NewestCursor      string `json:"newest_cursor"`
	OldestCursor      string `json:"oldest_cursor"`
	IsSpam            bool   `json:"is_spam"`
	LastPermanentItem Item   `json:"last_permanent_item"`
}

func (c Conversation) Error() error {
	return c.err
}

func (c Conversation) lastItemID() string {
	n := len(c.Items)
	if n == 0 {
		return ""
	}
	return c.Items[n-1].ID
}

// Send sends message in conversation
func (c *Conversation) Send(msg string) error {
	insta := c.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"recipient_users": c.recipientUsers(),
			"action":          "send_item",
			"text":            msg,
		},
	)
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: urlSendMessage,
			Query:    generateSignature(data),
			IsPost:   true,
		},
	)
	if err == nil {

	}
	return err
}

// Write is like Send but being compatible with io.Writer.
func (c *Conversation) Write(b []byte) (int, error) {
	n := len(b)
	return n, c.Send(b2s(b))
}

// Next loads next set of private messages.
//
// See example: examples/inbox/conversation.go
func (c *Conversation) Next() bool {
	if c.err != nil {
		return false
	}
	if c.firstRun {
		c.firstRun = false
		return true
	}

	insta := c.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlInboxThread, c.ID),
			Query: map[string]string{
				"cursor":            c.lastItemID(),
				"direction":         "older", // go to upper
				"use_unified_inbox": "true",
			},
		},
	)
	if err == nil {
		resp := threadResp{}
		err = json.Unmarshal(body, &resp)
		if err == nil {
			*c = resp.Conversation
			c.inst = insta
			if !c.HasOlder {
				c.err = ErrNoMore
			}
			return true
		}
	}
	c.err = err
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer"
	"github.com/blevesearch/bleve/analysis/char_filters/html_char_filter"
	"github.com/blevesearch/bleve/analysis/language/fr"
	"github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter"
	"github.com/blevesearch/bleve/analysis/tokenizers/exception"
	bleveuni "github.com/blevesearch/bleve/analysis/tokenizers/unicode"
	"github.com/blevesearch/bleve/index/store/boltdb"
	"github.com/blevesearch/bleve/index/upside_down"
	"github.com/pmezard/apec/jstruct"
	"github.com/pquerna/ffjson/ffjson"
)

type offerResult struct {
	Id    string
	Offer *jstruct.JsonOffer
	Err   error
}

func loadOffers(store *Store) ([]*jstruct.JsonOffer, error) {
	ids, err := store.List()
	if err != nil {
		return nil, err
	}
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := getStoreJsonOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jstruct.JsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string    `json:"id"`
	HTML      string    `json:"html"`
	Title     string    `json:"title"`
	MinSalary int       `json:"min_salary"`
	MaxSalary int       `json:"max_salary"`
	Date      time.Time `json:"date"`
	URL       string
	Location  string `json:"location"`
}

const (
	ApecURL = "https://cadres.apec.fr/home/mes-offres/recherche-des-offres-demploi/" +
		"liste-des-offres-demploi/detail-de-loffre-demploi.html?numIdOffre="
)

func convertOffer(offer *jstruct.JsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func getStoreJsonOffer(store *Store, id string) (*jstruct.JsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	js := &jstruct.JsonOffer{}
	err = ffjson.Unmarshal(data, js)
	return js, err
}

func getStoreOffer(store *Store, id string) (*Offer, error) {
	js, err := getStoreJsonOffer(store, id)
	if err != nil || js == nil {
		return nil, err
	}
	return convertOffer(js)
}

func convertOffers(offers []*jstruct.JsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

var (
	indexExceptions = []string{
		"c++",
		"c#",
	}
)

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	parts := []string{}
	for _, exc := range indexExceptions {
		parts = append(parts, regexp.QuoteMeta(exc))
	}
	pattern := strings.Join(parts, "|")
	pattern = "(?i)(?:" + pattern + ")"

	m := bleve.NewIndexMapping()
	apecName := "apec"
	err = m.AddCustomTokenizer(apecName, map[string]interface{}{
		"type":       exception.Name,
		"exceptions": []string{pattern},
		"tokenizer":  bleveuni.Name,
	})
	if err != nil {
		return nil, err
	}

	frTokens := []string{
		lower_case_filter.Name,
		fr.ElisionName,
		fr.StopName,
		fr.LightStemmerName,
	}
	fr := map[string]interface{}{
		"type":          custom_analyzer.Name,
		"tokenizer":     apecName,
		"token_filters": frTokens,
	}
	frHtml := map[string]interface{}{
		"type": custom_analyzer.Name,
		"char_filters": []string{
			html_char_filter.Name,
		},
		"tokenizer":     apecName,
		"token_filters": frTokens,
	}
	err = m.AddCustomAnalyzer("fr", fr)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
	}
	err = m.AddCustomAnalyzer("fr_html", frHtml)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
	}

	htmlFr := bleve.NewTextFieldMapping()
	htmlFr.Store = false
	htmlFr.IncludeInAll = false
	htmlFr.IncludeTermVectors = false
	htmlFr.Analyzer = "fr_html"

	textFr := bleve.NewTextFieldMapping()
	textFr.Store = false
	textFr.IncludeInAll = false
	textFr.IncludeTermVectors = false
	textFr.Analyzer = "fr"

	textAll := bleve.NewTextFieldMapping()
	textAll.Store = false
	textAll.IncludeInAll = true
	textAll.IncludeTermVectors = false

	date := bleve.NewDateTimeFieldMapping()
	date.Index = false
	date.Store = true
	date.IncludeInAll = false
	date.IncludeTermVectors = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", htmlFr)
	offer.AddFieldMappingsAt("title", textFr)
	offer.AddFieldMappingsAt("date", date)

	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.NewUsing(dir, m, upside_down.Name, boltdb.Name,
		map[string]interface{}{
			"nosync": true,
		})
	if err != nil {
		return nil, err
	}
	return index, nil
}

func OpenOfferIndex(path string) (bleve.Index, error) {
	return bleve.OpenUsing(path, map[string]interface{}{
		"nosync": false,
	})
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	// Work around kingpin messing with boolean flags starting with --no-xxx (#54)
	indexIndex = indexCmd.Flag("index", "enable indexing (use --no-index to disable it)").
			Default("true").Bool()
	indexMinQuota = indexCmd.Flag("min-quota",
		"stop geocoding when call quota moves below supplied value").Default("500").Int()
)

func indexOffers(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	defer store.Close()
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}

	rejected := 0
	geocodingKey := cfg.GeocodingKey()
	if geocodingKey != "" {
		geocoder, err := NewGeocoder(geocodingKey, cfg.Geocoder())
		if err != nil {
			return err
		}
		defer geocoder.Close()
		rejected, err = geocodeOffers(store, geocoder, offers, *indexMinQuota)
		if err != nil {
			return err
		}
		fmt.Printf("%d rejected geocoding\n", rejected)
	}
	if *indexIndex {
		index, err := NewOfferIndex(cfg.Index())
		if err != nil {
			return err
		}
		start := time.Now()
		indexed := 0
		for i, offer := range offers {
			if (i+1)%500 == 0 {
				now := time.Now()
				elapsed := float64(now.Sub(start)) / float64(time.Second)
				fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
			}
			err = index.Index(offer.Id, offer)
			if err != nil {
				return err
			}
			indexed += 1
		}
		err = index.Close()
		if err != nil {
			return err
		}
		end := time.Now()
		fmt.Printf("%d/%d documents indexed in %.2fs\n", indexed, len(offers),
			float64(end.Sub(start))/float64(time.Second))
	}
	return nil
}
<commit_msg>index: add --id to index a single document<commit_after>package main

import (
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer"
	"github.com/blevesearch/bleve/analysis/char_filters/html_char_filter"
	"github.com/blevesearch/bleve/analysis/language/fr"
	"github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter"
	"github.com/blevesearch/bleve/analysis/tokenizers/exception"
	bleveuni "github.com/blevesearch/bleve/analysis/tokenizers/unicode"
	"github.com/blevesearch/bleve/index/store/boltdb"
	"github.com/blevesearch/bleve/index/upside_down"
	"github.com/pmezard/apec/jstruct"
	"github.com/pquerna/ffjson/ffjson"
)

type offerResult struct {
	Id    string
	Offer *jstruct.JsonOffer
	Err   error
}

func loadOffers(store *Store) ([]*jstruct.JsonOffer, error) {
	ids, err := store.List()
	if err != nil {
		return nil, err
	}
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := getStoreJsonOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jstruct.JsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string    `json:"id"`
	HTML      string    `json:"html"`
	Title     string    `json:"title"`
	MinSalary int       `json:"min_salary"`
	MaxSalary int       `json:"max_salary"`
	Date      time.Time `json:"date"`
	URL       string
	Location  string `json:"location"`
}

const (
	ApecURL = "https://cadres.apec.fr/home/mes-offres/recherche-des-offres-demploi/" +
		"liste-des-offres-demploi/detail-de-loffre-demploi.html?numIdOffre="
)

func convertOffer(offer *jstruct.JsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func getStoreJsonOffer(store *Store, id string) (*jstruct.JsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	js := &jstruct.JsonOffer{}
	err = ffjson.Unmarshal(data, js)
	return js, err
}

func getStoreOffer(store *Store, id string) (*Offer, error) {
	js, err := getStoreJsonOffer(store, id)
	if err != nil || js == nil {
		return nil, err
	}
	return convertOffer(js)
}

func convertOffers(offers []*jstruct.JsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

var (
	indexExceptions = []string{
		"c++",
		"c#",
	}
)

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	parts := []string{}
	for _, exc := range indexExceptions {
		parts = append(parts, regexp.QuoteMeta(exc))
	}
	pattern := strings.Join(parts, "|")
	pattern = "(?i)(?:" + pattern + ")"

	m := bleve.NewIndexMapping()
	apecName := "apec"
	err = m.AddCustomTokenizer(apecName, map[string]interface{}{
		"type":       exception.Name,
		"exceptions": []string{pattern},
		"tokenizer":  bleveuni.Name,
	})
	if err != nil {
		return nil, err
	}

	frTokens := []string{
		lower_case_filter.Name,
		fr.ElisionName,
		fr.StopName,
		fr.LightStemmerName,
	}
	fr := map[string]interface{}{
		"type":          custom_analyzer.Name,
		"tokenizer":     apecName,
		"token_filters": frTokens,
	}
	frHtml := map[string]interface{}{
		"type": custom_analyzer.Name,
		"char_filters": []string{
			html_char_filter.Name,
		},
		"tokenizer":     apecName,
		"token_filters": frTokens,
	}
	err = m.AddCustomAnalyzer("fr", fr)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
	}
	err = m.AddCustomAnalyzer("fr_html", frHtml)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
	}

	htmlFr := bleve.NewTextFieldMapping()
	htmlFr.Store = false
	htmlFr.IncludeInAll = false
	htmlFr.IncludeTermVectors = false
	htmlFr.Analyzer = "fr_html"

	textFr := bleve.NewTextFieldMapping()
	textFr.Store = false
	textFr.IncludeInAll = false
	textFr.IncludeTermVectors = false
	textFr.Analyzer = "fr"

	textAll := bleve.NewTextFieldMapping()
	textAll.Store = false
	textAll.IncludeInAll = true
	textAll.IncludeTermVectors = false

	date := bleve.NewDateTimeFieldMapping()
	date.Index = false
	date.Store = true
	date.IncludeInAll = false
	date.IncludeTermVectors = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", htmlFr)
	offer.AddFieldMappingsAt("title", textFr)
	offer.AddFieldMappingsAt("date", date)

	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.NewUsing(dir, m, upside_down.Name, boltdb.Name,
		map[string]interface{}{
			"nosync": true,
		})
	if err != nil {
		return nil, err
	}
	return index, nil
}

func OpenOfferIndex(path string) (bleve.Index, error) {
	return bleve.OpenUsing(path, map[string]interface{}{
		"nosync": false,
	})
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	// Work around kingpin messing with boolean flags starting with --no-xxx (#54)
	indexIndex = indexCmd.Flag("index", "enable indexing (use --no-index to disable it)").
			Default("true").Bool()
	indexMinQuota = indexCmd.Flag("min-quota",
		"stop geocoding when call quota moves below supplied value").Default("500").Int()
	indexDocId = indexCmd.Flag("id", "index only specified document").String()
)

func indexOffers(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	defer store.Close()
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	if *indexDocId != "" {
		kept := []*jstruct.JsonOffer{}
		for _, o := range rawOffers {
			if o.Id == *indexDocId {
				kept = append(kept, o)
			}
		}
		rawOffers = kept
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}

	rejected := 0
	geocodingKey := cfg.GeocodingKey()
	if geocodingKey != "" {
		geocoder, err := NewGeocoder(geocodingKey, cfg.Geocoder())
		if err != nil {
			return err
		}
		defer geocoder.Close()
		rejected, err = geocodeOffers(store, geocoder, offers, *indexMinQuota)
		if err != nil {
			return err
		}
		fmt.Printf("%d rejected geocoding\n", rejected)
	}
	if *indexIndex {
		index, err := NewOfferIndex(cfg.Index())
		if err != nil {
			return err
		}
		start := time.Now()
		indexed := 0
		for i, offer := range offers {
			if (i+1)%500 == 0 {
				now := time.Now()
				elapsed := float64(now.Sub(start)) / float64(time.Second)
				fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
			}
			err = index.Index(offer.Id, offer)
			if err != nil {
				return err
			}
			indexed += 1
		}
		err = index.Close()
		if err != nil {
			return err
		}
		end := time.Now()
		fmt.Printf("%d/%d documents indexed in %.2fs\n", indexed, len(offers),
			float64(end.Sub(start))/float64(time.Second))
	}
	return nil
}
<|endoftext|>


<commit_before>package squircy

import (
	"fmt"
	"html"
	"net/http"
	"strconv"

	"github.com/HouzuoGuo/tiedot/db"
	"github.com/antage/eventsource"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
	"github.com/tyler-sommer/squircy2/squircy/config"
	"github.com/tyler-sommer/squircy2/squircy/irc"
	"github.com/tyler-sommer/squircy2/squircy/script"
	"github.com/tyler-sommer/stick"
)

type stickHandler struct {
	env *stick.Env
	res http.ResponseWriter
}

func (h *stickHandler) HTML(status int, name string, ctx map[string]stick.Value) {
	h.res.WriteHeader(200)
	err := h.env.Execute(name, h.res, ctx)
	if err != nil {
		fmt.Println(err)
	}
}

func newStickHandler() martini.Handler {
	env := stick.New(newTemplateLoader())
	env.Functions["escape"] = func(ctx stick.Context, args ...stick.Value) stick.Value {
		if len(args) < 1 {
			return nil
		}
		return html.EscapeString(stick.CoerceString(args[0]))
	}
	return func(res http.ResponseWriter, req *http.Request, c martini.Context) {
		c.Map(&stickHandler{env, res})
	}
}

func configureWeb(manager *Manager) {
	manager.Handlers(
		newStaticHandler(),
		newStickHandler(),
		render.Renderer(),
	)
	manager.Get("/event", func(es eventsource.EventSource, w http.ResponseWriter, r *http.Request) {
		es.ServeHTTP(w, r)
	})
	manager.Get("/", indexAction)
	manager.Get("/status", statusAction)
	manager.Group("/manage", func(r martini.Router) {
		r.Get("", manageAction)
		r.Post("/update", manageUpdateAction)
	})
	manager.Post("/connect", connectAction)
	manager.Post("/disconnect", disconnectAction)
	manager.Group("/script", func(r martini.Router) {
		r.Get("", scriptAction)
		r.Post("/reinit", scriptReinitAction)
		r.Get("/new", newScriptAction)
		r.Post("/create", createScriptAction)
		r.Get("/:id/edit", editScriptAction)
		r.Post("/:id/update", updateScriptAction)
		r.Post("/:id/remove", removeScriptAction)
		r.Post("/:id/toggle", toggleScriptAction)
	})
	manager.Group("/repl", func(r martini.Router) {
		r.Get("", replAction)
		r.Post("/execute", replExecuteAction)
	})
}

func indexAction(s *stickHandler, t *eventTracer) {
	s.HTML(200, "index.html.twig", map[string]stick.Value{
		"terminal": t.History(OutputEvent),
		"irc":      t.History(irc.IrcEvent),
	})
}

type appStatus struct {
	Status irc.ConnectionStatus
}

func statusAction(r render.Render, mgr *irc.IrcConnectionManager) {
	r.JSON(200, appStatus{mgr.Status()})
}

func scriptAction(s *stickHandler, repo script.ScriptRepository) {
	scripts := repo.FetchAll()

	s.HTML(200, "script/index.html.twig", map[string]stick.Value{"scripts": scripts})
}

func scriptReinitAction(r render.Render, mgr *script.ScriptManager) {
	mgr.ReInit()

	r.JSON(200, nil)
}

func newScriptAction(s *stickHandler) {
	s.HTML(200, "script/new.html.twig", nil)
}

func createScriptAction(r render.Render, repo script.ScriptRepository, request *http.Request) {
	sType := request.FormValue("type")
	title := request.FormValue("title")
	body := request.FormValue("body")

	repo.Save(&script.Script{0, script.ScriptType(sType), title, body, true})

	r.Redirect("/script", 302)
}

func editScriptAction(s *stickHandler, repo script.ScriptRepository, params martini.Params) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)

	script := repo.Fetch(int(id))

	s.HTML(200, "script/edit.html.twig", map[string]stick.Value{"script": script})
}

func updateScriptAction(r render.Render, repo script.ScriptRepository, params martini.Params, request *http.Request) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)
	sType := request.FormValue("type")
	title := request.FormValue("title")
	body := request.FormValue("body")

	repo.Save(&script.Script{int(id), script.ScriptType(sType), title, body, true})

	r.Redirect("/script", 302)
}

func removeScriptAction(r render.Render, repo script.ScriptRepository, params martini.Params) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)

	repo.Delete(int(id))

	r.JSON(200, nil)
}

func toggleScriptAction(r render.Render, repo script.ScriptRepository, params martini.Params) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)

	script := repo.Fetch(int(id))
	script.Enabled = !script.Enabled
	repo.Save(script)

	r.JSON(200, nil)
}

func replAction(s *stickHandler) {
	s.HTML(200, "repl/index.html.twig", nil)
}

func replExecuteAction(r render.Render, manager *script.ScriptManager, request *http.Request) {
	code := request.FormValue("script")
	sType := script.ScriptType(request.FormValue("scriptType"))

	res, err := manager.RunUnsafe(sType, code)
	var errStr interface{}
	if err != nil {
		errStr = err.Error()
	}
	r.JSON(200, map[string]interface{}{
		"res": res,
		"err": errStr,
	})
}

func connectAction(r render.Render, mgr *irc.IrcConnectionManager) {
	mgr.Connect()

	r.JSON(200, nil)
}

func disconnectAction(r render.Render, mgr *irc.IrcConnectionManager) {
	mgr.Quit()

	r.JSON(200, nil)
}

func manageAction(s *stickHandler, config *config.Configuration) {
	s.HTML(200, "manage/edit.html.twig", map[string]stick.Value{
		"config": config,
	})
}

func manageUpdateAction(r render.Render, database *db.DB, conf *config.Configuration, request *http.Request) {
	conf.Network = request.FormValue("network")
	conf.Nick = request.FormValue("nick")
	conf.Username = request.FormValue("username")
	conf.OwnerNick = request.FormValue("owner_nick")
	conf.OwnerHost = request.FormValue("owner_host")
	conf.TLS = (request.FormValue("tls") == "on")

	config.SaveConfig(database, conf)

	r.Redirect("/manage", 302)
}
<commit_msg>Add a webhook martini handle and manage request in webhookAction function<commit_after>package squircy

import (
	"encoding/json"
	"fmt"
	"github.com/HouzuoGuo/tiedot/db"
	"github.com/antage/eventsource"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
	"github.com/tyler-sommer/squircy2/squircy/config"
	"github.com/tyler-sommer/squircy2/squircy/irc"
	"github.com/tyler-sommer/squircy2/squircy/script"
	"github.com/tyler-sommer/squircy2/squircy/webhook"
	"github.com/tyler-sommer/stick"
	"html"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

type stickHandler struct {
	env *stick.Env
	res http.ResponseWriter
}

func (h *stickHandler) HTML(status int, name string, ctx map[string]stick.Value) {
	h.res.WriteHeader(200)
	err := h.env.Execute(name, h.res, ctx)
	if err != nil {
		fmt.Println(err)
	}
}

func newStickHandler() martini.Handler {
	env := stick.New(newTemplateLoader())
	env.Functions["escape"] = func(ctx stick.Context, args ...stick.Value) stick.Value {
		if len(args) < 1 {
			return nil
		}
		return html.EscapeString(stick.CoerceString(args[0]))
	}
	return func(res http.ResponseWriter, req *http.Request, c martini.Context) {
		c.Map(&stickHandler{env, res})
	}
}

type webhookHandler struct {
	env *stick.Env
	res http.ResponseWriter
}

func newWebhookHandler() martini.Handler {
	env := stick.New(newTemplateLoader())
	env.Functions["escape"] = func(ctx stick.Context, args ...stick.Value) stick.Value {
		if len(args) < 1 {
			return nil
		}
		return html.EscapeString(stick.CoerceString(args[0]))
	}
	return func(res http.ResponseWriter, req *http.Request, c martini.Context) {
		c.Map(&webhookHandler{env, res})
	}
}

func configureWeb(manager *Manager) {
	manager.Handlers(
		newStaticHandler(),
		newStickHandler(),
		render.Renderer(),
		newWebhookHandler(),
	)
	manager.Get("/event", func(es eventsource.EventSource, w http.ResponseWriter, r *http.Request) {
		es.ServeHTTP(w, r)
	})
	manager.Get("/", indexAction)
	manager.Get("/status", statusAction)
	manager.Group("/manage", func(r martini.Router) {
		r.Get("", manageAction)
		r.Post("/update", manageUpdateAction)
	})
	manager.Post("/connect", connectAction)
	manager.Post("/disconnect", disconnectAction)
	manager.Group("/script", func(r martini.Router) {
		r.Get("", scriptAction)
		r.Post("/reinit", scriptReinitAction)
		r.Get("/new", newScriptAction)
		r.Post("/create", createScriptAction)
		r.Get("/:id/edit", editScriptAction)
		r.Post("/:id/update", updateScriptAction)
		r.Post("/:id/remove", removeScriptAction)
		r.Post("/:id/toggle", toggleScriptAction)
	})
	manager.Group("/repl", func(r martini.Router) {
		r.Get("", replAction)
		r.Post("/execute", replExecuteAction)
	})
	manager.Post("/webhooks", webhookAction)
}

func indexAction(s *stickHandler, t *eventTracer) {
	s.HTML(200, "index.html.twig", map[string]stick.Value{
		"terminal": t.History(OutputEvent),
		"irc":      t.History(irc.IrcEvent),
	})
}

type appStatus struct {
	Status irc.ConnectionStatus
}

func statusAction(r render.Render, mgr *irc.IrcConnectionManager) {
	r.JSON(200, appStatus{mgr.Status()})
}

func scriptAction(s *stickHandler, repo script.ScriptRepository) {
	scripts := repo.FetchAll()

	s.HTML(200, "script/index.html.twig", map[string]stick.Value{"scripts": scripts})
}

func scriptReinitAction(r render.Render, mgr *script.ScriptManager) {
	mgr.ReInit()

	r.JSON(200, nil)
}

func newScriptAction(s *stickHandler) {
	s.HTML(200, "script/new.html.twig", nil)
}

func createScriptAction(r render.Render, repo script.ScriptRepository, request *http.Request) {
	sType := request.FormValue("type")
	title := request.FormValue("title")
	body := request.FormValue("body")

	repo.Save(&script.Script{0, script.ScriptType(sType), title, body, true})

	r.Redirect("/script", 302)
}

func editScriptAction(s *stickHandler, repo script.ScriptRepository, params martini.Params) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)

	script := repo.Fetch(int(id))

	s.HTML(200, "script/edit.html.twig", map[string]stick.Value{"script": script})
}

func updateScriptAction(r render.Render, repo script.ScriptRepository, params martini.Params, request *http.Request) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)
	sType := request.FormValue("type")
	title := request.FormValue("title")
	body := request.FormValue("body")

	repo.Save(&script.Script{int(id), script.ScriptType(sType), title, body, true})

	r.Redirect("/script", 302)
}

func removeScriptAction(r render.Render, repo script.ScriptRepository, params martini.Params) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)

	repo.Delete(int(id))

	r.JSON(200, nil)
}

func toggleScriptAction(r render.Render, repo script.ScriptRepository, params martini.Params) {
	id, _ := strconv.ParseInt(params["id"], 0, 64)

	script := repo.Fetch(int(id))
	script.Enabled = !script.Enabled
	repo.Save(script)

	r.JSON(200, nil)
}

func replAction(s *stickHandler) {
	s.HTML(200, "repl/index.html.twig", nil)
}

func replExecuteAction(r render.Render, manager *script.ScriptManager, request *http.Request) {
	code := request.FormValue("script")
	sType := script.ScriptType(request.FormValue("scriptType"))

	res, err := manager.RunUnsafe(sType, code)
	var errStr interface{}
	if err != nil {
		errStr = err.Error()
	}
	r.JSON(200, map[string]interface{}{
		"res": res,
		"err": errStr,
	})
}

func connectAction(r render.Render, mgr *irc.IrcConnectionManager) {
	mgr.Connect()

	r.JSON(200, nil)
}

func disconnectAction(r render.Render, mgr *irc.IrcConnectionManager) {
	mgr.Quit()

	r.JSON(200, nil)
}

func manageAction(s *stickHandler, config *config.Configuration) {
	s.HTML(200, "manage/edit.html.twig", map[string]stick.Value{
		"config": config,
	})
}

func manageUpdateAction(r render.Render, database *db.DB, conf *config.Configuration, request *http.Request) {
	conf.Network = request.FormValue("network")
	conf.Nick = request.FormValue("nick")
	conf.Username = request.FormValue("username")
	conf.OwnerNick = request.FormValue("owner_nick")
	conf.OwnerHost = request.FormValue("owner_host")
	conf.TLS = (request.FormValue("tls") == "on")

	config.SaveConfig(database, conf)

	r.Redirect("/manage", 302)
}

// Manage webhook events
func webhookAction(render render.Render, mgr *irc.IrcConnectionManager, request *http.Request) {
	// Parse body
	body, err := ioutil.ReadAll(request.Body)
	if err != nil {
		log.Printf("error reading the request body. %+v\n", err)
		render.JSON(400, "Invalid data")
	}
	// Process json
	contentType := request.Header.Get("Content-Type")

	var payload map[string]interface{}

	if strings.Contains(contentType, "json") {
		decoder := json.NewDecoder(strings.NewReader(string(body)))
		decoder.UseNumber()

		err := decoder.Decode(&payload)

		if err != nil {
			log.Printf("error parsing JSON payload %+v\n", err)
			render.JSON(400, "Invalid json")
		}
	} else {
		render.JSON(400, "Invalid content-type")
	}
	// All is good
	hook := webhook.WebhookEvent{}
	hook.Message = "test"
	err = hook.Process(mgr)
	render.JSON(200, nil)
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"math/rand"
	"testing"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAWSEcsDataSource_ecsCluster(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccCheckAwsEcsClusterDataSourceConfig,
				Check: resource.ComposeTestCheckFunc(
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "status", "ACTIVE"),
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "pending_tasks_count", "0"),
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "running_tasks_count", "0"),
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "registered_container_instances_count", "0"),
				),
			},
		},
	})
}

var testAccCheckAwsEcsClusterDataSourceConfig = fmt.Sprintf(`
resource "aws_ecs_cluster" "default" {
  name = "default-%d"
}

resource "aws_ecs_task_definition" "mongo" {
  family = "mongodb"
  container_definitions = <<DEFINITION
[
  {
    "cpu": 128,
    "essential": true,
    "image": "mongo:latest",
    "memory": 128,
    "memoryReservation": 64,
    "name": "mongodb"
  }
]
DEFINITION
}

resource "aws_ecs_service" "mongo" {
  name = "mongodb"
  cluster = "${aws_ecs_cluster.default.id}"
  task_definition = "${aws_ecs_task_definition.mongo.arn}"
  desired_count = 1
}

data "aws_ecs_cluster" "default" {
  cluster_name = "${aws_ecs_cluster.default.name}"
}
`, rand.New(rand.NewSource(time.Now().UnixNano())).Int())
<commit_msg>provider/aws: code review feedback<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAWSEcsDataSource_ecsCluster(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccCheckAwsEcsClusterDataSourceConfig,
				Check: resource.ComposeTestCheckFunc(
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "status", "ACTIVE"),
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "pending_tasks_count", "0"),
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "running_tasks_count", "0"),
					resource.TestCheckResourceAttr("data.aws_ecs_cluster.default", "registered_container_instances_count", "0"),
				),
			},
		},
	})
}

var testAccCheckAwsEcsClusterDataSourceConfig = fmt.Sprintf(`
resource "aws_ecs_cluster" "default" {
  name = "default-%d"
}

resource "aws_ecs_task_definition" "mongo" {
  family = "mongodb"
  container_definitions = <<DEFINITION
[
  {
    "cpu": 128,
    "essential": true,
    "image": "mongo:latest",
    "memory": 128,
    "memoryReservation": 64,
    "name": "mongodb"
  }
]
DEFINITION
}

resource "aws_ecs_service" "mongo" {
  name = "mongodb"
  cluster = "${aws_ecs_cluster.default.id}"
  task_definition = "${aws_ecs_task_definition.mongo.arn}"
  desired_count = 1
}

data "aws_ecs_cluster" "default" {
  cluster_name = "${aws_ecs_cluster.default.name}"
}
`, acctest.RandInt())
<|endoftext|>


<commit_before>package rancher

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	rancherClient "github.com/rancher/go-rancher/client"
)

func resourceRancherEnvironment() *schema.Resource {
	return &schema.Resource{
		Create: resourceRancherEnvironmentCreate,
		Read:   resourceRancherEnvironmentRead,
		Update: resourceRancherEnvironmentUpdate,
		Delete: resourceRancherEnvironmentDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"orchestration": &schema.Schema{
				Type:         schema.TypeString,
				Default:      "cattle",
				Optional:     true,
				ValidateFunc: validation.StringInSlice([]string{"cattle", "kubernetes", "mesos", "swarm"}, true),
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"member": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"external_id_type": {
							Type:     schema.TypeString,
							Required: true,
						},
						"external_id": {
							Type:     schema.TypeString,
							Required: true,
						},
						"role": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func resourceRancherEnvironmentCreate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] Creating Environment: %s", d.Id())
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	name := d.Get("name").(string)
	description := d.Get("description").(string)
	orchestration := d.Get("orchestration").(string)

	data := map[string]interface{}{
		"name":        &name,
		"description": &description,
	}

	setOrchestrationFields(orchestration, data)

	var newEnv rancherClient.Project
	if err := client.Create("project", data, &newEnv); err != nil {
		return err
	}

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"active", "removed", "removing"},
		Target:     []string{"active"},
		Refresh:    EnvironmentStateRefreshFunc(client, newEnv.Id),
		Timeout:    10 * time.Minute,
		Delay:      1 * time.Second,
		MinTimeout: 3 * time.Second,
	}
	_, waitErr := stateConf.WaitForState()
	if waitErr != nil {
		return fmt.Errorf(
			"Error waiting for environment (%s) to be created: %s", newEnv.Id, waitErr)
	}

	d.SetId(newEnv.Id)
	log.Printf("[INFO] Environment ID: %s", d.Id())

	// Add members
	envClient, err := meta.(*Config).EnvironmentClient(d.Id())
	if err != nil {
		return err
	}
	if v, ok := d.GetOk("members"); ok {
		members := v.([]interface{})
		_, err = envClient.Project.ActionSetmembers(&newEnv, &rancherClient.SetProjectMembersInput{
			Members: members,
		})
		if err != nil {
			return err
		}
	}

	return resourceRancherEnvironmentRead(d, meta)
}

func resourceRancherEnvironmentRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] Refreshing Environment: %s", d.Id())
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	env, err := client.Project.ById(d.Id())
	if err != nil {
		return err
	}

	if env == nil {
		log.Printf("[INFO] Environment %s not found", d.Id())
		d.SetId("")
		return nil
	}

	if removed(env.State) {
		log.Printf("[INFO] Environment %s was removed on %v", d.Id(), env.Removed)
		d.SetId("")
		return nil
	}

	log.Printf("[INFO] Environment Name: %s", env.Name)

	d.Set("description", env.Description)
	d.Set("name", env.Name)
	d.Set("orchestration", getActiveOrchestration(env))

	envClient, err := meta.(*Config).EnvironmentClient(d.Id())
	if err != nil {
		return err
	}

	members, _ := envClient.ProjectMember.List(NewListOpts())

	d.Set("member", normalizeMembers(members.Data))
	return nil
}

func resourceRancherEnvironmentUpdate(d *schema.ResourceData, meta interface{}) error {
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	name := d.Get("name").(string)
	description := d.Get("description").(string)
	orchestration := d.Get("orchestration").(string)

	data := map[string]interface{}{
		"name":        &name,
		"description": &description,
	}

	setOrchestrationFields(orchestration, data)

	var newEnv rancherClient.Project
	env, err := client.Project.ById(d.Id())
	if err != nil {
		return err
	}

	if err := client.Update("project", &env.Resource, data, &newEnv); err != nil {
		return err
	}

	// Update members
	envClient, err := meta.(*Config).EnvironmentClient(d.Id())
	if err != nil {
		return err
	}
	members := d.Get("member").(*schema.Set).List()
	_, err = envClient.Project.ActionSetmembers(&newEnv, &rancherClient.SetProjectMembersInput{
		Members: makeProjectMembers(members),
	})
	if err != nil {
		return err
	}

	return resourceRancherEnvironmentRead(d, meta)
}

func resourceRancherEnvironmentDelete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] Deleting Environment: %s", d.Id())
	id := d.Id()
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	env, err := client.Project.ById(id)
	if err != nil {
		return err
	}

	if err := client.Project.Delete(env); err != nil {
		return fmt.Errorf("Error deleting Environment: %s", err)
	}

	log.Printf("[DEBUG] Waiting for environment (%s) to be removed", id)

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"active", "removed", "removing"},
		Target:     []string{"removed"},
		Refresh:    EnvironmentStateRefreshFunc(client, id),
		Timeout:    10 * time.Minute,
		Delay:      1 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	_, waitErr := stateConf.WaitForState()
	if waitErr != nil {
		return fmt.Errorf(
			"Error waiting for environment (%s) to be removed: %s", id, waitErr)
	}

	d.SetId("")
	return nil
}

func setOrchestrationFields(orchestration string, data map[string]interface{}) {
	orch := strings.ToLower(orchestration)

	data["swarm"] = false
	data["kubernetes"] = false
	data["mesos"] = false

	if orch == "k8s" {
		orch = "kubernetes"
	}

	data[orch] = true
}

func normalizeMembers(in []rancherClient.ProjectMember) (out []interface{}) {
	for _, m := range in {
		mm := map[string]string{
			"external_id_type": m.ExternalIdType,
			"external_id":      m.ExternalId,
			"role":             m.Role,
		}
		out = append(out, mm)
	}
	return
}

func makeProjectMembers(in []interface{}) (out []interface{}) {
	for _, m := range in {
		mMap := m.(map[string]interface{})
		mm := rancherClient.ProjectMember{
			ExternalIdType: mMap["external_id_type"].(string),
			ExternalId:     mMap["external_id"].(string),
			Role:           mMap["role"].(string),
		}
		out = append(out, mm)
	}
	return
}

// EnvironmentStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// a Rancher Environment.
func EnvironmentStateRefreshFunc(client *rancherClient.RancherClient, environmentID string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		env, err := client.Project.ById(environmentID)

		if err != nil {
			return nil, "", err
		}

		return env, env.State, nil
	}
}
<commit_msg>rancher_environment: initialize envClient only when members are set<commit_after>package rancher

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	rancherClient "github.com/rancher/go-rancher/client"
)

func resourceRancherEnvironment() *schema.Resource {
	return &schema.Resource{
		Create: resourceRancherEnvironmentCreate,
		Read:   resourceRancherEnvironmentRead,
		Update: resourceRancherEnvironmentUpdate,
		Delete: resourceRancherEnvironmentDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"orchestration": &schema.Schema{
				Type:         schema.TypeString,
				Default:      "cattle",
				Optional:     true,
				ValidateFunc: validation.StringInSlice([]string{"cattle", "kubernetes", "mesos", "swarm"}, true),
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"member": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"external_id_type": {
							Type:     schema.TypeString,
							Required: true,
						},
						"external_id": {
							Type:     schema.TypeString,
							Required: true,
						},
						"role": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func resourceRancherEnvironmentCreate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] Creating Environment: %s", d.Id())
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	name := d.Get("name").(string)
	description := d.Get("description").(string)
	orchestration := d.Get("orchestration").(string)

	data := map[string]interface{}{
		"name":        &name,
		"description": &description,
	}

	setOrchestrationFields(orchestration, data)

	var newEnv rancherClient.Project
	if err := client.Create("project", data, &newEnv); err != nil {
		return err
	}

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"active", "removed", "removing"},
		Target:     []string{"active"},
		Refresh:    EnvironmentStateRefreshFunc(client, newEnv.Id),
		Timeout:    10 * time.Minute,
		Delay:      1 * time.Second,
		MinTimeout: 3 * time.Second,
	}
	_, waitErr := stateConf.WaitForState()
	if waitErr != nil {
		return fmt.Errorf(
			"Error waiting for environment (%s) to be created: %s", newEnv.Id, waitErr)
	}

	d.SetId(newEnv.Id)
	log.Printf("[INFO] Environment ID: %s", d.Id())

	// Add members
	if v, ok := d.GetOk("members"); ok {
		envClient, err := meta.(*Config).EnvironmentClient(d.Id())
		if err != nil {
			return err
		}
		members := v.([]interface{})
		_, err = envClient.Project.ActionSetmembers(&newEnv, &rancherClient.SetProjectMembersInput{
			Members: members,
		})
		if err != nil {
			return err
		}
	}

	return resourceRancherEnvironmentRead(d, meta)
}

func resourceRancherEnvironmentRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] Refreshing Environment: %s", d.Id())
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	env, err := client.Project.ById(d.Id())
	if err != nil {
		return err
	}

	if env == nil {
		log.Printf("[INFO] Environment %s not found", d.Id())
		d.SetId("")
		return nil
	}

	if removed(env.State) {
		log.Printf("[INFO] Environment %s was removed on %v", d.Id(), env.Removed)
		d.SetId("")
		return nil
	}

	log.Printf("[INFO] Environment Name: %s", env.Name)

	d.Set("description", env.Description)
	d.Set("name", env.Name)
	d.Set("orchestration", getActiveOrchestration(env))

	envClient, err := meta.(*Config).EnvironmentClient(d.Id())
	if err != nil {
		return err
	}

	members, _ := envClient.ProjectMember.List(NewListOpts())

	d.Set("member", normalizeMembers(members.Data))
	return nil
}

func resourceRancherEnvironmentUpdate(d *schema.ResourceData, meta interface{}) error {
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	name := d.Get("name").(string)
	description := d.Get("description").(string)
	orchestration := d.Get("orchestration").(string)

	data := map[string]interface{}{
		"name":        &name,
		"description": &description,
	}

	setOrchestrationFields(orchestration, data)

	var newEnv rancherClient.Project
	env, err := client.Project.ById(d.Id())
	if err != nil {
		return err
	}

	if err := client.Update("project", &env.Resource, data, &newEnv); err != nil {
		return err
	}

	// Update members
	envClient, err := meta.(*Config).EnvironmentClient(d.Id())
	if err != nil {
		return err
	}
	members := d.Get("member").(*schema.Set).List()
	_, err = envClient.Project.ActionSetmembers(&newEnv, &rancherClient.SetProjectMembersInput{
		Members: makeProjectMembers(members),
	})
	if err != nil {
		return err
	}

	return resourceRancherEnvironmentRead(d, meta)
}

func resourceRancherEnvironmentDelete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] Deleting Environment: %s", d.Id())
	id := d.Id()
	client, err := meta.(*Config).GlobalClient()
	if err != nil {
		return err
	}

	env, err := client.Project.ById(id)
	if err != nil {
		return err
	}

	if err := client.Project.Delete(env); err != nil {
		return fmt.Errorf("Error deleting Environment: %s", err)
	}

	log.Printf("[DEBUG] Waiting for environment (%s) to be removed", id)

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"active", "removed", "removing"},
		Target:     []string{"removed"},
		Refresh:    EnvironmentStateRefreshFunc(client, id),
		Timeout:    10 * time.Minute,
		Delay:      1 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	_, waitErr := stateConf.WaitForState()
	if waitErr != nil {
		return fmt.Errorf(
			"Error waiting for environment (%s) to be removed: %s", id, waitErr)
	}

	d.SetId("")
	return nil
}

func setOrchestrationFields(orchestration string, data map[string]interface{}) {
	orch := strings.ToLower(orchestration)

	data["swarm"] = false
	data["kubernetes"] = false
	data["mesos"] = false

	if orch == "k8s" {
		orch = "kubernetes"
	}

	data[orch] = true
}

func normalizeMembers(in []rancherClient.ProjectMember) (out []interface{}) {
	for _, m := range in {
		mm := map[string]string{
			"external_id_type": m.ExternalIdType,
			"external_id":      m.ExternalId,
			"role":             m.Role,
		}
		out = append(out, mm)
	}
	return
}

func makeProjectMembers(in []interface{}) (out []interface{}) {
	for _, m := range in {
		mMap := m.(map[string]interface{})
		mm := rancherClient.ProjectMember{
			ExternalIdType: mMap["external_id_type"].(string),
			ExternalId:     mMap["external_id"].(string),
			Role:           mMap["role"].(string),
		}
		out = append(out, mm)
	}
	return
}

// EnvironmentStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// a Rancher Environment.
func EnvironmentStateRefreshFunc(client *rancherClient.RancherClient, environmentID string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		env, err := client.Project.ById(environmentID)

		if err != nil {
			return nil, "", err
		}

		return env, env.State, nil
	}
}
<|endoftext|>


<commit_before>package store_test

import (
	"appengine/aetest"
	. "github.com/alexandre-normand/glukit/app/store"
	"github.com/alexandre-normand/osin"
	"testing"
	"time"
)

func TestGetClient(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	t.Logf("Request is [%v] from context [%v]", c.Request(), c)

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	_, err = osinStorage.GetClientWithContext("***REMOVED***", c)

	if err != nil {
		t.Fatal(err)
	}
}

func TestAccessDataStorage(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	t.Logf("Request is [%v] from context [%v]", c.Request(), c)

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	client, err := osinStorage.GetClientWithContext("***REMOVED***", c)
	d := osin.AccessData{client, nil, nil, "token", "test", 0, "scope", "uri", time.Now(), TEST_USER}
	err = osinStorage.SaveAccessWithContext(&d, c)
	if err != nil {
		t.Fatal(err)
	}

	_, err = osinStorage.LoadAccessWithContext("token", c)
	if err != nil {
		t.Fatal(err)
	}
}

func TestAuthorizeDataStorage(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	client, err := osinStorage.GetClientWithContext("***REMOVED***", c)
	d := osin.AuthorizeData{client, "code", 0, "scope", "uri", "state", time.Now(), TEST_USER}
	err = osinStorage.SaveAuthorizeWithContext(&d, c)
	if err != nil {
		t.Fatal(err)
	}

	_, err = osinStorage.LoadAuthorizeWithContext("code", c)
	if err != nil {
		t.Fatal(err)
	}
}

func TestFullAccessDataStorage(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	client, err := osinStorage.GetClientWithContext("***REMOVED***", c)
	d := osin.AuthorizeData{client, "code", 0, "scope", "uri", "state", time.Now(), TEST_USER}
	err = osinStorage.SaveAuthorizeWithContext(&d, c)
	if err != nil {
		t.Fatal(err)
	}

	previousAccess := osin.AccessData{client, nil, nil, "token", "test", 0, "scope", "uri", time.Now(), TEST_USER}
	err = osinStorage.SaveAccessWithContext(&previousAccess, c)
	if err != nil {
		t.Fatal(err)
	}

	access := osin.AccessData{client, &d, &previousAccess, "token", "test", 0, "scope", "uri", time.Now(), TEST_USER}
	err = osinStorage.SaveAccessWithContext(&access, c)
	if err != nil {
		t.Fatal(err)
	}

	_, err = osinStorage.LoadAccessWithContext("token", c)
	if err != nil {
		t.Fatal(err)
	}

}
<commit_msg>Fix Osin Storage Tests.<commit_after>package store_test

import (
	"appengine/aetest"
	. "github.com/alexandre-normand/glukit/app/store"
	"github.com/alexandre-normand/osin"
	"testing"
	"time"
)

func TestGetClient(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	t.Logf("Request is [%v] from context [%v]", c.Request(), c)

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	_, err = osinStorage.GetClientWithContext("***REMOVED***", c)

	if err != nil {
		t.Fatal(err)
	}
}

func TestAccessDataStorage(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	t.Logf("Request is [%v] from context [%v]", c.Request(), c)

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	client, err := osinStorage.GetClientWithContext("***REMOVED***", c)
	d := osin.AccessData{client, nil, nil, "token", "test", 0, "scope", "uri", time.Now(), TEST_USER}
	err = osinStorage.SaveAccessWithContext(&d, c)
	if err != nil {
		t.Fatal(err)
	}

	_, err = osinStorage.LoadAccessWithContext("token", c)
	if err != nil {
		t.Fatal(err)
	}
}

func TestAuthorizeDataStorage(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	client, err := osinStorage.GetClientWithContext("***REMOVED***", c)
	d := osin.AuthorizeData{client, "code", 0, "scope", "uri", "state", time.Now(), TEST_USER}
	err = osinStorage.SaveAuthorizeWithContext(&d, c)
	if err != nil {
		t.Fatal(err)
	}

	_, err = osinStorage.LoadAuthorizeWithContext("code", c)
	if err != nil {
		t.Fatal(err)
	}
}

func TestFullAccessDataStorage(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	osinStorage := NewOsinAppEngineStoreWithContext(c)
	client, err := osinStorage.GetClientWithContext("***REMOVED***", c)
	d := osin.AuthorizeData{client, "code", 0, "scope", "uri", "state", time.Now(), TEST_USER}
	err = osinStorage.SaveAuthorizeWithContext(&d, c)
	if err != nil {
		t.Fatal(err)
	}

	previousAccess := osin.AccessData{client, nil, nil, "oldtoken", "oldtest", 0, "scope", "uri", time.Now(), TEST_USER}
	err = osinStorage.SaveAccessWithContext(&previousAccess, c)
	if err != nil {
		t.Fatal(err)
	}

	access := osin.AccessData{client, &d, &previousAccess, "token", "test", 0, "scope", "uri", time.Now(), TEST_USER}
	err = osinStorage.SaveAccessWithContext(&access, c)
	if err != nil {
		t.Fatal(err)
	}

	_, err = osinStorage.LoadAccessWithContext("token", c)
	if err != nil {
		t.Fatal(err)
	}

}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Hiram Jerónimo Pérez worg{at}linuxmail[dot]org

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// Package meth is a thin wrapper around upper.io/db to ease some repettitive tasks
package meth

import (
	"errors"
	"reflect"
	"upper.io/db"
	"upper.io/db/util"
)

var (
	ErrNoId    = errors.New(`No id field found`)
	ErrNoSlice = errors.New(`Slice expected`)
)

// Persistent is the interface that allows us to communicate with upper.io/db collection
type Persistent interface {
	Collection() db.Collection
}

// One fills p with one element based on the id field
func One(p Persistent) error {
	col := p.Collection()

	id, err := getId(p)
	if err != nil {
		return err
	}

	res := col.Find(db.Cond{`id`: id})

	if err := res.One(p); err != nil {
		return err
	}

	return nil
}

// OneBy fills p with one element matching the given conditions
func OneBy(p Persistent, cond ...interface{}) error {
	col := p.Collection()
	res := col.Find(cond...)

	if err := res.One(p); err != nil {
		return err
	}

	return nil
}

// AllBy fills rows with data from the result set matching the given conditions
func AllBy(p Persistent, rows interface{}, cond ...interface{}) error {
	col := p.Collection()
	res := col.Find(cond...)

	if err := res.All(rows); err != nil {
		return err
	}

	return nil
}

// All works as an alias for AllBy
func All(p Persistent, rows interface{}) error {
	return AllBy(p, rows)
}

func getId(t interface{}) (int64, error) {
	valP := getValue(t)
	i := util.GetStructFieldIndex(valP.Type(), `id`)
	if len(i) > 1 || len(i) < 1 {
		return 0, ErrNoId
	}

	id := valP.FieldByIndex(i).Int()

	return id, nil
}

func getValue(t interface{}) (rslt reflect.Value) {
	rslt = reflect.ValueOf(t)
	for {
		if rslt.Kind() != reflect.Ptr {
			break
		}
		rslt = rslt.Elem()
	}

	return
}

func isSlice(t interface{}) bool {
	if k := reflect.TypeOf(t).Kind(); k == reflect.Slice {
		return true
	}

	return false
}
<commit_msg>change license comment<commit_after>// Copyright (c) 2014 Hiram Jerónimo Pérez worg{at}linuxmail[dot]org
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Package meth is a thin wrapper around upper.io/db to ease some repettitive tasks
package meth

import (
	"errors"
	"reflect"
	"upper.io/db"
	"upper.io/db/util"
)

var (
	ErrNoId    = errors.New(`No id field found`)
	ErrNoSlice = errors.New(`Slice expected`)
)

// Persistent is the interface that allows us to communicate with upper.io/db collection
type Persistent interface {
	Collection() db.Collection
}

// One fills p with one element based on the id field
func One(p Persistent) error {
	col := p.Collection()

	id, err := getId(p)
	if err != nil {
		return err
	}

	res := col.Find(db.Cond{`id`: id})

	if err := res.One(p); err != nil {
		return err
	}

	return nil
}

// OneBy fills p with one element matching the given conditions
func OneBy(p Persistent, cond ...interface{}) error {
	col := p.Collection()
	res := col.Find(cond...)

	if err := res.One(p); err != nil {
		return err
	}

	return nil
}

// AllBy fills rows with data from the result set matching the given conditions
func AllBy(p Persistent, rows interface{}, cond ...interface{}) error {
	col := p.Collection()
	res := col.Find(cond...)

	if err := res.All(rows); err != nil {
		return err
	}

	return nil
}

// All works as an alias for AllBy
func All(p Persistent, rows interface{}) error {
	return AllBy(p, rows)
}

func getId(t interface{}) (int64, error) {
	valP := getValue(t)
	i := util.GetStructFieldIndex(valP.Type(), `id`)
	if len(i) > 1 || len(i) < 1 {
		return 0, ErrNoId
	}

	id := valP.FieldByIndex(i).Int()

	return id, nil
}

func getValue(t interface{}) (rslt reflect.Value) {
	rslt = reflect.ValueOf(t)
	for {
		if rslt.Kind() != reflect.Ptr {
			break
		}
		rslt = rslt.Elem()
	}

	return
}

func isSlice(t interface{}) bool {
	if k := reflect.TypeOf(t).Kind(); k == reflect.Slice {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>// +build !seccomp,!windows

package daemon

import (
	"github.com/docker/docker/container"
	"github.com/opencontainers/specs/specs-go"
)

func setSeccomp(daemon *Daemon, rs *specs.Spec, c *container.Container) error {
	return nil
}
<commit_msg>Error out if user tries to specify a custom seccomp profile on system that does not support it<commit_after>// +build !seccomp,!windows

package daemon

import (
	"fmt"

	"github.com/docker/docker/container"
	"github.com/opencontainers/specs/specs-go"
)

func setSeccomp(daemon *Daemon, rs *specs.Spec, c *container.Container) error {
	if c.SeccompProfile != "" && c.SeccompProfile != "unconfined" {
		return fmt.Errorf("seccomp profiles are not supported on this daemon, you cannot specify a custom seccomp profile")
	}
	return nil
}
<|endoftext|>


<commit_before>package midi

import (
	"errors"
)

// Message is a MIDI message
type Message interface {
	// String inspects the MIDI message in an informative way
	String() string

	// Raw returns the raw bytes of the MIDI message
	Raw() []byte
}

// Writer writes MIDI messages
type Writer interface {
	// Write writes the given MIDI message and returns any error
	Write(Message) error
}

// Reader reads MIDI messages
type Reader interface {
	// Read reads a MIDI message
	Read() (Message, error)
}

// WriteCloser is a Writer that must be closed at the end of writing.
type WriteCloser interface {
	Writer
	Close() error
}

// ReadCloser is a Reader that must be closed at the end of reading.
type ReadCloser interface {
	Reader
	Close() error
}

// ErrUnexpectedEOF is returned, when an unexspected end of file is reached.
var ErrUnexpectedEOF = errors.New("Unexpected End of File found.")

/*
   A MIDI message is made up of an eight-bit status byte which is generally followed by one or two data bytes.

   MIDI message (status byte + 1-2 data bytes)
      |
      -------- Channel Message (channel number included in status byte) 1000 | 1001 | 1010 | 1011 | 1100 | 1101 | 1110
      |            |
      |            ------ Channel Voice Message (musical performance)
      |            |
      |            ------ Mode Message  (how to does instr. respond to Channel Voice message)
      |                   (1011nnnn Channel Mode Message)
      |
      ---------System Message (no channel number in status byte), all beginning with 1111
                   |
                   ------ System Common Messages
                   |
                   ------ System Real Time Messages
                   |
                   ------ System Exclusive Messages (F0, F7)

   There are a number of different types of MIDI messages. At the highest level, MIDI messages are classified
   as being either Channel Messages or System Messages.

   Channel messages are those which apply to a specific
   Channel, and the Channel number is included in the status byte for these messages. System messages are not
   Channel specific, and no Channel number is indicated in their status bytes.

   Channel Messages may be further classified as being either Channel Voice Messages, or Mode Messages.
   Channel Voice Messages carry musical performance data, and these messages comprise most of the traffic in
   a typical MIDI data stream. Channel Mode messages affect the way a receiving instrument will respond to the
   Channel Voice messages.

   MIDI System Messages are classified as being System Common Messages, System Real Time Messages, or
   System Exclusive Messages. System Common messages are intended for all receivers in the system. System
   Real Time messages are used for synchronisation between clock-based MIDI components. System Exclusive
   messages include a Manufacturer's Identification (ID) code, and are used to transfer any number of data bytes
   in a format specified by the referenced manufacturer.
*/

/*
   read in the next byte (uint8)

   if it is FF -> meta event
   if it is F0 or F7 -> sysex event
   else ->
      System Common Message F0-FF

      F0 1111 0000 sysex event
      F7 1111 0111 sysex event
      FF 1111 1111 meta event

      channel voice message D7-D0

      D0 1101 0000
      D7 1101 0111

*/
<commit_msg>use fmt instead off errors package to create error<commit_after>package midi

import (
	"fmt"
)

// Message is a MIDI message
type Message interface {
	// String inspects the MIDI message in an informative way
	String() string

	// Raw returns the raw bytes of the MIDI message
	Raw() []byte
}

// Writer writes MIDI messages
type Writer interface {
	// Write writes the given MIDI message and returns any error
	Write(Message) error
}

// Reader reads MIDI messages
type Reader interface {
	// Read reads a MIDI message
	Read() (Message, error)
}

// WriteCloser is a Writer that must be closed at the end of writing.
type WriteCloser interface {
	Writer
	Close() error
}

// ReadCloser is a Reader that must be closed at the end of reading.
type ReadCloser interface {
	Reader
	Close() error
}

// ErrUnexpectedEOF is returned, when an unexspected end of file is reached.
var ErrUnexpectedEOF = fmt.Errorf("Unexpected End of File found.")

/*
   A MIDI message is made up of an eight-bit status byte which is generally followed by one or two data bytes.

   MIDI message (status byte + 1-2 data bytes)
      |
      -------- Channel Message (channel number included in status byte) 1000 | 1001 | 1010 | 1011 | 1100 | 1101 | 1110
      |            |
      |            ------ Channel Voice Message (musical performance)
      |            |
      |            ------ Mode Message  (how to does instr. respond to Channel Voice message)
      |                   (1011nnnn Channel Mode Message)
      |
      ---------System Message (no channel number in status byte), all beginning with 1111
                   |
                   ------ System Common Messages
                   |
                   ------ System Real Time Messages
                   |
                   ------ System Exclusive Messages (F0, F7)

   There are a number of different types of MIDI messages. At the highest level, MIDI messages are classified
   as being either Channel Messages or System Messages.

   Channel messages are those which apply to a specific
   Channel, and the Channel number is included in the status byte for these messages. System messages are not
   Channel specific, and no Channel number is indicated in their status bytes.

   Channel Messages may be further classified as being either Channel Voice Messages, or Mode Messages.
   Channel Voice Messages carry musical performance data, and these messages comprise most of the traffic in
   a typical MIDI data stream. Channel Mode messages affect the way a receiving instrument will respond to the
   Channel Voice messages.

   MIDI System Messages are classified as being System Common Messages, System Real Time Messages, or
   System Exclusive Messages. System Common messages are intended for all receivers in the system. System
   Real Time messages are used for synchronisation between clock-based MIDI components. System Exclusive
   messages include a Manufacturer's Identification (ID) code, and are used to transfer any number of data bytes
   in a format specified by the referenced manufacturer.
*/

/*
   read in the next byte (uint8)

   if it is FF -> meta event
   if it is F0 or F7 -> sysex event
   else ->
      System Common Message F0-FF

      F0 1111 0000 sysex event
      F7 1111 0111 sysex event
      FF 1111 1111 meta event

      channel voice message D7-D0

      D0 1101 0000
      D7 1101 0111

*/
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"math"
	"net/http"
	_ "net/http/pprof"
	"strconv"
	"strings"
	"time"
)

type Point struct {
	Val float64
	Ts  uint32
}

type Series struct {
	Target     string
	Datapoints []Point
	Interval   uint32
}

func (s *Series) MarshalJSON() ([]byte, error) {
	var buf bytes.Buffer
	buf.WriteString(`{"Target":"`)
	buf.WriteString(s.Target)
	buf.WriteString(`","Datapoints":[`)
	for _, p := range s.Datapoints {
		buf.WriteString(`[`)
		if math.IsNaN(p.Val) {
			buf.WriteString(`null,`)
		} else {
			buf.WriteString(strconv.FormatFloat(p.Val, 'f', 3, 64))
			buf.WriteString(`,`)
		}
		buf.WriteString(strconv.FormatUint(uint64(p.Ts), 10))
		buf.WriteString(`],`)
	}
	buf.Truncate(buf.Len() - 1) // cut last comma
	buf.WriteString(`],"Interval":`)
	buf.WriteString(strconv.FormatInt(int64(s.Interval), 10))
	buf.WriteString(`}`)
	return buf.Bytes(), nil
}

func get(store Store, defCache *DefCache, aggSettings []aggSetting) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		Get(w, req, store, defCache, aggSettings)
	}
}

// note: we don't normalize/quantize/fill-unknowns
// we just serve what we know
func Get(w http.ResponseWriter, req *http.Request, store Store, defCache *DefCache, aggSettings []aggSetting) {
	pre := time.Now()
	req.ParseForm()
	log.Debug(fmt.Sprintf("http.Get(): INCOMING REQ. targets: %q, maxDataPoints: %q", req.Form.Get("target"), req.Form.Get("maxDataPoints")))

	maxDataPoints := uint32(800)
	maxDataPointsStr := req.Form.Get("maxDataPoints")
	var err error
	if maxDataPointsStr != "" {
		tmp, err := strconv.Atoi(maxDataPointsStr)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		maxDataPoints = uint32(tmp)
	}

	targets, ok := req.Form["target"]
	if !ok {
		http.Error(w, "missing target arg", http.StatusBadRequest)
		return
	}
	now := time.Now()
	fromUnix := uint32(now.Add(-time.Duration(24) * time.Hour).Unix())
	toUnix := uint32(now.Add(time.Duration(1) * time.Second).Unix())
	from := req.Form.Get("from")
	if from != "" {
		fromUnixInt, err := strconv.Atoi(from)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		fromUnix = uint32(fromUnixInt)
	}
	to := req.Form.Get("to")
	if to != "" {
		toUnixInt, err := strconv.Atoi(to)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		toUnix = uint32(toUnixInt)
	}
	if fromUnix >= toUnix {
		http.Error(w, "to must be higher than from", http.StatusBadRequest)
		return
	}

	reqs := make([]Req, len(targets))
	for i, target := range targets {
		var consolidateBy string
		id := target
		// yes, i am aware of the arguably grossness of the below.
		// however, it is solid based on the documented allowed input format.
		// once we need to support several functions, we can implement
		// a proper expression parser
		if strings.HasPrefix(target, "consolidateBy(") {
			if target[len(target)-2:] != "')" || !strings.Contains(target, ",'") || strings.Count(target, "'") != 2 || strings.Count(target, ",") != 1 {
				http.Error(w, "target parse error", http.StatusBadRequest)
				return
			}
			consolidateBy = target[strings.Index(target, "'")+1 : strings.LastIndex(target, "'")]
			id = target[strings.Index(target, "(")+1 : strings.Index(target, ",")]
		}

		if consolidateBy == "" {
			def, ok := defCache.Get(id)
			consolidateBy = "avg"
			if ok && def.TargetType == "counter" {
				consolidateBy = "last"
			}
		}
		var consolidator consolidation.Consolidator
		switch consolidateBy {
		case "avg", "average":
			consolidator = consolidation.Avg
		case "last":
			consolidator = consolidation.Last
		case "min":
			consolidator = consolidation.Min
		case "max":
			consolidator = consolidation.Max
		case "sum":
			consolidator = consolidation.Sum
		default:
			http.Error(w, "unrecognized consolidation function", http.StatusBadRequest)
			return
		}
		req := NewReq(id, fromUnix, toUnix, maxDataPoints, consolidator)
		reqs[i] = req
	}
	err = findMetricsForRequests(reqs, defCache)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	reqs, err = alignRequests(reqs, aggSettings)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	out := make([]Series, len(reqs))
	for i, req := range reqs {
		log.Debug("===================================")
		log.Debug("HTTP Get()          %s", req)
		points, interval, err := getTarget(store, req)
		if err != nil {
			log.Error(0, err.Error())
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		out[i] = Series{
			Target:     targets[i],
			Datapoints: points,
			Interval:   interval,
		}
	}
	js, err := json.Marshal(out)
	if err != nil {
		log.Error(0, err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	reqHandleDuration.Value(time.Now().Sub(pre))
	w.Write(js)
}

// report ApplicationStatus for use by loadBalancer healthChecks.
// We only want requests to be sent to this node if it is the primary
// node or if it has been online for at *warmUpPeriod
func appStatus(w http.ResponseWriter, req *http.Request) {
	if clusterStatus.IsPrimary() {
		w.Write([]byte("OK"))
		return
	}
	if time.Since(startupTime) < (time.Duration(*warmUpPeriod) * time.Second) {
		http.Error(w, "Service not ready", http.StatusServiceUnavailable)
		return
	}

	w.Write([]byte("OK"))
	return
}
<commit_msg>work with []byte directly, so that we can avoid string allocations<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"math"
	"net/http"
	_ "net/http/pprof"
	"strconv"
	"strings"
	"time"
)

type Point struct {
	Val float64
	Ts  uint32
}

type Series struct {
	Target     string
	Datapoints []Point
	Interval   uint32
}

func (s *Series) MarshalJSON() ([]byte, error) {
	var b []byte
	b = append(b, `{"Target":"`...)
	b = append(b, s.Target...)
	b = append(b, `","Datapoints":[`...)
	for _, p := range s.Datapoints {
		b = append(b, '[')
		if math.IsNaN(p.Val) {
			b = append(b, `null,`...)
		} else {
			b = strconv.AppendFloat(b, p.Val, 'f', 3, 64)
			b = append(b, ',')
		}
		b = strconv.AppendUint(b, uint64(p.Ts), 10)
		b = append(b, `],`...)
	}
	b = b[:len(b)-1] // cut last comma
	b = append(b, `],"Interval":`...)
	b = strconv.AppendInt(b, int64(s.Interval), 10)
	b = append(b, '}')
	return b, nil
}

func get(store Store, defCache *DefCache, aggSettings []aggSetting) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		Get(w, req, store, defCache, aggSettings)
	}
}

// note: we don't normalize/quantize/fill-unknowns
// we just serve what we know
func Get(w http.ResponseWriter, req *http.Request, store Store, defCache *DefCache, aggSettings []aggSetting) {
	pre := time.Now()
	req.ParseForm()
	log.Debug(fmt.Sprintf("http.Get(): INCOMING REQ. targets: %q, maxDataPoints: %q", req.Form.Get("target"), req.Form.Get("maxDataPoints")))

	maxDataPoints := uint32(800)
	maxDataPointsStr := req.Form.Get("maxDataPoints")
	var err error
	if maxDataPointsStr != "" {
		tmp, err := strconv.Atoi(maxDataPointsStr)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		maxDataPoints = uint32(tmp)
	}

	targets, ok := req.Form["target"]
	if !ok {
		http.Error(w, "missing target arg", http.StatusBadRequest)
		return
	}
	now := time.Now()
	fromUnix := uint32(now.Add(-time.Duration(24) * time.Hour).Unix())
	toUnix := uint32(now.Add(time.Duration(1) * time.Second).Unix())
	from := req.Form.Get("from")
	if from != "" {
		fromUnixInt, err := strconv.Atoi(from)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		fromUnix = uint32(fromUnixInt)
	}
	to := req.Form.Get("to")
	if to != "" {
		toUnixInt, err := strconv.Atoi(to)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		toUnix = uint32(toUnixInt)
	}
	if fromUnix >= toUnix {
		http.Error(w, "to must be higher than from", http.StatusBadRequest)
		return
	}

	reqs := make([]Req, len(targets))
	for i, target := range targets {
		var consolidateBy string
		id := target
		// yes, i am aware of the arguably grossness of the below.
		// however, it is solid based on the documented allowed input format.
		// once we need to support several functions, we can implement
		// a proper expression parser
		if strings.HasPrefix(target, "consolidateBy(") {
			if target[len(target)-2:] != "')" || !strings.Contains(target, ",'") || strings.Count(target, "'") != 2 || strings.Count(target, ",") != 1 {
				http.Error(w, "target parse error", http.StatusBadRequest)
				return
			}
			consolidateBy = target[strings.Index(target, "'")+1 : strings.LastIndex(target, "'")]
			id = target[strings.Index(target, "(")+1 : strings.Index(target, ",")]
		}

		if consolidateBy == "" {
			def, ok := defCache.Get(id)
			consolidateBy = "avg"
			if ok && def.TargetType == "counter" {
				consolidateBy = "last"
			}
		}
		var consolidator consolidation.Consolidator
		switch consolidateBy {
		case "avg", "average":
			consolidator = consolidation.Avg
		case "last":
			consolidator = consolidation.Last
		case "min":
			consolidator = consolidation.Min
		case "max":
			consolidator = consolidation.Max
		case "sum":
			consolidator = consolidation.Sum
		default:
			http.Error(w, "unrecognized consolidation function", http.StatusBadRequest)
			return
		}
		req := NewReq(id, fromUnix, toUnix, maxDataPoints, consolidator)
		reqs[i] = req
	}
	err = findMetricsForRequests(reqs, defCache)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	reqs, err = alignRequests(reqs, aggSettings)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	out := make([]Series, len(reqs))
	for i, req := range reqs {
		log.Debug("===================================")
		log.Debug("HTTP Get()          %s", req)
		points, interval, err := getTarget(store, req)
		if err != nil {
			log.Error(0, err.Error())
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		out[i] = Series{
			Target:     targets[i],
			Datapoints: points,
			Interval:   interval,
		}
	}
	js, err := json.Marshal(out)
	if err != nil {
		log.Error(0, err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	reqHandleDuration.Value(time.Now().Sub(pre))
	w.Write(js)
}

// report ApplicationStatus for use by loadBalancer healthChecks.
// We only want requests to be sent to this node if it is the primary
// node or if it has been online for at *warmUpPeriod
func appStatus(w http.ResponseWriter, req *http.Request) {
	if clusterStatus.IsPrimary() {
		w.Write([]byte("OK"))
		return
	}
	if time.Since(startupTime) < (time.Duration(*warmUpPeriod) * time.Second) {
		http.Error(w, "Service not ready", http.StatusServiceUnavailable)
		return
	}

	w.Write([]byte("OK"))
	return
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	expire        uint = 600
	RedisKeyIndex      = make(map[string]RedisKey)
	RedisKeys          = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: false, expire: false},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "imageboards", fieldcount: 1, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key, ok := RedisKeyIndex[params[0]]
		if !ok {
			c.Next()
			return
		}

		// set the key minus the base
		key.SetKey(params[1:]...)

		result, err = key.Get()
		if err == redis.ErrCacheMiss {
			// go to the controller
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		} else if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		return redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		if r.expire {
			return redis.RedisCache.SetEx(r.key, expire, data)
		} else {
			return redis.RedisCache.Set(r.key, data)
		}
	}

	return
}
<commit_msg>use explicit keys<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: false, expire: false},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "imageboards", fieldcount: 1, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key, ok := RedisKeyIndex[params[0]]
		if !ok {
			c.Next()
			return
		}

		// set the key minus the base
		key.SetKey(params[1:]...)

		result, err = key.Get()
		if err == redis.ErrCacheMiss {
			// go to the controller
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		} else if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		return redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		if r.expire {
			return redis.RedisCache.SetEx(r.key, 600, data)
		} else {
			return redis.RedisCache.Set(r.key, data)
		}
	}

	return
}
<|endoftext|>


<commit_before>package abalone

import "encoding/json"

type HexSet map[Hex]struct{}

func (s HexSet) MarshalJSON() ([]byte, error) {
	var out []Hex
	for h, _ := range s {
		out = append(out, h)
	}
	return json.Marshal(out)
}

func (s *HexSet) UnmarshalJSON(data []byte) error {
	*s = make(HexSet)
	var in []Hex
	if err := json.Unmarshal(data, &in); err != nil {
		return err
	}
	for _, h := range in {
		(*s)[h] = struct{}{}
	}
	return nil
}

func (hm *HexSet) hexSet2Slice() []Hex {
	res := make([]Hex, len(*hm))
	i := 0
	for position, _ := range *hm {
		res[i] = position
		i++
	}
	return res
}

func (m1 HexSet) eq(m2 HexSet) bool {
	if len(m1) != len(m2) {
		return false
	}
	for x, _ := range m1 {
		if !m2.has(x) {
			return false
		}
	}
	return true
}

func (s HexSet) has(h Hex) bool {
	_, found := s[h]
	return found
}

func slice2HexSet(hexes []Hex) HexSet {
	res := make(HexSet)
	for _, h := range hexes {
		res[h] = struct{}{}
	}
	return res
}
<commit_msg>Minor - remove dup code<commit_after>package abalone

import "encoding/json"

type HexSet map[Hex]struct{}

func (s HexSet) MarshalJSON() ([]byte, error) {
	return json.Marshal(s.toSlice())
}

func (s *HexSet) UnmarshalJSON(data []byte) error {
	*s = make(HexSet)
	var in []Hex
	if err := json.Unmarshal(data, &in); err != nil {
		return err
	}
	for _, h := range in {
		(*s)[h] = struct{}{}
	}
	return nil
}

func (set HexSet) toSlice() []Hex {
	res := make([]Hex, len(set))
	i := 0
	for position, _ := range set {
		res[i] = position
		i++
	}
	return res
}

func (m1 HexSet) eq(m2 HexSet) bool {
	if len(m1) != len(m2) {
		return false
	}
	for x, _ := range m1 {
		if !m2.has(x) {
			return false
		}
	}
	return true
}

func (s HexSet) has(h Hex) bool {
	_, found := s[h]
	return found
}

func slice2HexSet(hexes []Hex) HexSet {
	res := make(HexSet)
	for _, h := range hexes {
		res[h] = struct{}{}
	}
	return res
}
<|endoftext|>


<commit_before>package externals

import (
	"fmt"
	"strings"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

func normalizeIconKey(key string) string {
	switch key {
	case "bitcoin":
		return "btc"
	case "zcash.t", "zcash.z", "zcash.s":
		return "zcash"
	case "http", "https", "dns":
		return "web"
	default:
		return key
	}
}

func MakeIcons(mctx libkb.MetaContext, serviceKey, imgName string, size int) (res []keybase1.SizedImage) {
	for _, factor := range []int{1, 2} {
		factorix := ""
		if factor > 1 {
			factorix = fmt.Sprintf("@%vx", factor)
		}

		res = append(res, keybase1.SizedImage{
			Path: strings.Join([]string{
				libkb.SiteURILookup[mctx.G().Env.GetRunMode()],
				"images/paramproofs/services",
				normalizeIconKey(serviceKey),
				fmt.Sprintf("%v_%v%v.png", imgName, size, factorix),
			}, "/"),
			Width: size * factor,
		})
	}
	return res
}
<commit_msg>Serve dev logos from 127 (#17106)<commit_after>package externals

import (
	"fmt"
	"strings"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

func normalizeIconKey(key string) string {
	switch key {
	case "bitcoin":
		return "btc"
	case "zcash.t", "zcash.z", "zcash.s":
		return "zcash"
	case "http", "https", "dns":
		return "web"
	default:
		return key
	}
}

func MakeIcons(mctx libkb.MetaContext, serviceKey, imgName string, size int) (res []keybase1.SizedImage) {
	for _, factor := range []int{1, 2} {
		factorix := ""
		if factor > 1 {
			factorix = fmt.Sprintf("@%vx", factor)
		}

		site := libkb.SiteURILookup[mctx.G().Env.GetRunMode()]
		if mctx.G().Env.GetRunMode() == libkb.DevelRunMode {
			site = strings.Replace(site, "localhost", "127.0.0.1", 1)
		}

		res = append(res, keybase1.SizedImage{
			Path: strings.Join([]string{site,
				"images/paramproofs/services",
				normalizeIconKey(serviceKey),
				fmt.Sprintf("%v_%v%v.png", imgName, size, factorix),
			}, "/"),
			Width: size * factor,
		})
	}
	return res
}
<|endoftext|>