<commit_before>package endpoints

import (
	"../config"
	"../models"

	"bytes"
	"crypto/rand"
	"database/sql"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"image/jpeg"
	"image/png"
	"net/http"
	"os"
	"path"
	"strconv"

	"github.com/gorilla/mux"
)

// randomFilename produces a random 16-character (8-byte) hex string which, when formatted like
// {rand}.{ext} is a filename for an image with a given extension that is not already taken.
// Since the space of acceptable file names is so large despite requiring so few random bytes,
// this function should only execute the code within each loop once.
func randomFilename(cfg *config.Config, ext string) string {
	fname := ""
	randBytes := make([]byte, 8)
	generatedFName := false
	// Keep trying to generate names until we find one that isn't taken.
	for !generatedFName {
		generatedBytes := false
		// Keep trying to read random bytes until we definitely fill the buffer.
		for !generatedBytes {
			_, genErr := rand.Read(randBytes)
			generatedBytes = genErr == nil
			fmt.Printf("[+++] Generated bytes %v | Error: %v\n", randBytes, genErr)
		}
		fname = path.Join(cfg.ImageDirectory, hex.EncodeToString(randBytes)+"."+ext)
		fmt.Println("[+++] Generated file name", fname)
		_, findErr := os.Stat(fname)
		generatedFName = findErr != nil
	}
	return fname
}

// RegisterPageHandlers attaches the closures generated by each function defined below
// to handle incoming requests to the appropriate endpoint using a subrouter with an
// appropriate prefix, specified in main.
func RegisterPageHandlers(r *mux.Router, db *sql.DB, cfg *config.Config) {
	r.HandleFunc("/", listPages(db, cfg)).Methods("GET")
	r.HandleFunc("/", createPage(db, cfg)).Methods("POST")
	r.HandleFunc("/{pageId}", deletePage(db, cfg)).Methods("DELETE")
}

// GET /projects/{projectId}/releases/{releaseId}/pages

type listPagesRequest struct {
	ProjectID int
	ReleaseID int
}

type listPagesResponse struct {
	Error *string       `json:"error"`
	Pages []models.Page `json:"pages"`
}

// listPages lists descriptive information about
func listPages(db *sql.DB, cfg *config.Config) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		request := listPagesRequest{}
		vars := mux.Vars(r)
		pid := vars["projectId"]
		rid := vars["releaseId"]
		projectId, parseErr1 := strconv.Atoi(pid)
		releaseId, parseErr2 := strconv.Atoi(rid)

		encoder := json.NewEncoder(w)
		if parseErr1 != nil || parseErr2 != nil {
			fmt.Printf("[---] Parse error: %v || %v\n", parseErr1, parseErr2)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "projectId and releaseId must be integer IDs."
			encoder.Encode(listPagesResponse{&errMsg, []models.Page{}})
			return
		}
		request.ProjectID = projectId
		request.ReleaseID = releaseId
		pages, listErr := models.ListPages(request.ReleaseID, db)
		if listErr != nil {
			fmt.Println("[---] List error:", listErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Could not list pages. Please check that the projectId is correct or try again later."
			encoder.Encode(listPagesResponse{&errMsg, []models.Page{}})
			return
		}
		encoder.Encode(listPagesResponse{nil, pages})
	}
}

// POST /projects/{projectId}/releases/{releaseId}/pages

type createPageRequest struct {
	ProjectID int    // Pulled from the URL parameters
	ReleaseID int    // Pulled from the URL parameters
	Number    string `json:"page"`
	ImageData string `json:"data"`
}

type createPageResponse struct {
	Error   *string `json:"error"`
	Success bool    `json:"success"`
	ID      int     `json:"id"`
}

// createPage inserts a new page into the DB and saves page data to a file.
func createPage(db *sql.DB, cfg *config.Config) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		request := createPageRequest{}
		vars := mux.Vars(r)
		pid := vars["projectId"]
		rid := vars["releaseId"]
		projectId, parseErr1 := strconv.Atoi(pid)
		releaseId, parseErr2 := strconv.Atoi(rid)
		decoder := json.NewDecoder(r.Body)
		defer r.Body.Close()
		decodeErr := decoder.Decode(&request)
		fmt.Printf("[+++] Project ID = %d, Release ID = %d\n", projectId, releaseId)

		encoder := json.NewEncoder(w)
		if parseErr1 != nil || parseErr2 != nil {
			fmt.Printf("[---] Parse error: %v || %v\n", parseErr1, parseErr2)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "projectId and releaseId must be integer IDs."
			encoder.Encode(createPageResponse{&errMsg, false, 0})
			return
		}
		request.ProjectID = projectId
		request.ReleaseID = releaseId
		if decodeErr != nil {
			fmt.Println("[---] Decode error:", decodeErr)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "JSON format error or missing field detected."
			encoder.Encode(createPageResponse{&errMsg, false, 0})
			return
		}
		imageData, decodeErr := base64.StdEncoding.DecodeString(request.ImageData)
		if decodeErr != nil {
			fmt.Println("[---] Image decode error:", decodeErr)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "The supplied image data is not base64 encoded."
			encoder.Encode(createPageResponse{&errMsg, false, 0})
			return
		}
		fmt.Println("[+++] Successfully decoded image data")
		var filePath string // The path that the image ends up being saved to.
		_, jpgParseErr := jpeg.Decode(bytes.NewReader(imageData))
		if jpgParseErr != nil {
			_, pngParseErr := png.Decode(bytes.NewReader(imageData))
			if pngParseErr != nil {
				// The image is neither a valid JPG/JPEG nor a valid PNG image.
				fmt.Printf("[---] Uploaded error: %v && %v\n", jpgParseErr, pngParseErr)
				w.WriteHeader(http.StatusBadRequest)
				errMsg := "The uploaded image is neither a valid JPG/JPEG or PNG image."
				encoder.Encode(createPageResponse{&errMsg, false, -1})
				return
			} else {
				// The image is a valid PNG image.
				filePath = randomFilename(cfg, "png")
			}
		} else {
			// The image is a valid JPG/JPEG image.
			filePath = randomFilename(cfg, "jpg")
		}
		fmt.Printf("[+++] Computed filename %s\n", filePath)
		f, saveErr := os.Create(filePath)
		if saveErr == nil {
			_, saveErr = f.Write(imageData)
		}
		defer f.Close()
		if saveErr != nil {
			fmt.Println("[---] Save error:", saveErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Failed to save image file. Please try again later."
			encoder.Encode(createPageResponse{&errMsg, false, -1})
			return
		}
		fmt.Println("[+++] Successfully saved image to disk")
		page := models.NewPage(request.Number, filePath, request.ReleaseID)
		saveErr = page.Save(db)
		if saveErr != nil {
			fmt.Println("[---] Insert error:", saveErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Failed to save page. Please try again later."
			encoder.Encode(createPageResponse{&errMsg, false, -1})
			return
		}
		encoder.Encode(createPageResponse{nil, true, page.Id})
	}
}

// DELETE /projects/{projectId}/releases/{releaseId}/pages/{pageId}

type deletePageRequest struct {
	ProjectID int
	ReleaseID int
	PageID    int
}

type deletePageResponse struct {
	Error   *string `json:"error"`
	Success bool    `json:"success"`
}

// deletePage removes a page from the DB and deletes the file containing the image.
func deletePage(db *sql.DB, cfg *config.Config) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		request := deletePageRequest{}
		vars := mux.Vars(r)
		prid := vars["projectId"]
		reid := vars["releaseId"]
		paid := vars["pageId"]
		projectId, parseErr1 := strconv.Atoi(prid)
		releaseId, parseErr2 := strconv.Atoi(reid)
		pageId, parseErr3 := strconv.Atoi(paid)

		encoder := json.NewEncoder(w)
		if parseErr1 != nil || parseErr2 != nil || parseErr3 != nil {
			fmt.Println("[---] Parse error: %v || %v || %v\n", parseErr1, parseErr2, parseErr3)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "projectId, releaseId, and pageId must all be integer IDs."
			encoder.Encode(deletePageResponse{&errMsg, false})
			return
		}
		request.ProjectID = projectId
		request.ReleaseID = releaseId
		request.PageID = pageId
		// Unlike with projects and releases, we need to actually load the page we want to
		// delete from DB, because it needs the correct location of the page image on disk
		// to be able to delete that as well.
		page, findErr := models.FindPage(request.PageID, db)
		if findErr != nil {
			fmt.Println("[---] Find error:", findErr)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "Could not find the requested page. Please check that the page id is correct or try again later."
			encoder.Encode(deletePageResponse{&errMsg, false})
			return
		}
		fmt.Println("[+++] Attempting to delete page", page)
		deleteErr := page.Delete(db)
		if deleteErr != nil {
			fmt.Println("[---] Delete error:", findErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Could not delete the requested page. Please try again later."
			encoder.Encode(deletePageResponse{&errMsg, false})
			return
		}
		encoder.Encode(deletePageResponse{nil, true})
	}
}
<commit_msg>Implemented the endpoint for downloading individual images<commit_after>package endpoints

import (
	"../config"
	"../models"

	"bytes"
	"crypto/rand"
	"database/sql"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"image/jpeg"
	"image/png"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/gorilla/mux"
)

// randomFilename produces a random 16-character (8-byte) hex string which, when formatted like
// {rand}.{ext} is a filename for an image with a given extension that is not already taken.
// Since the space of acceptable file names is so large despite requiring so few random bytes,
// this function should only execute the code within each loop once.
func randomFilename(cfg *config.Config, ext string) string {
	fname := ""
	randBytes := make([]byte, 8)
	generatedFName := false
	// Keep trying to generate names until we find one that isn't taken.
	for !generatedFName {
		generatedBytes := false
		// Keep trying to read random bytes until we definitely fill the buffer.
		for !generatedBytes {
			_, genErr := rand.Read(randBytes)
			generatedBytes = genErr == nil
			fmt.Printf("[+++] Generated bytes %v | Error: %v\n", randBytes, genErr)
		}
		fname = path.Join(cfg.ImageDirectory, hex.EncodeToString(randBytes)+"."+ext)
		fmt.Println("[+++] Generated file name", fname)
		_, findErr := os.Stat(fname)
		generatedFName = findErr != nil
	}
	return fname
}

// RegisterPageHandlers attaches the closures generated by each function defined below
// to handle incoming requests to the appropriate endpoint using a subrouter with an
// appropriate prefix, specified in main.
func RegisterPageHandlers(r *mux.Router, db *sql.DB, cfg *config.Config) {
	r.HandleFunc("/", listPages(db, cfg)).Methods("GET")
	r.HandleFunc("/", createPage(db, cfg)).Methods("POST")
	r.HandleFunc("/{pageId}", getPage(db, cfg)).Methods("GET")
	r.HandleFunc("/{pageId}", deletePage(db, cfg)).Methods("DELETE")
}

// GET /projects/{projectId}/releases/{releaseId}/pages

type listPagesRequest struct {
	ProjectID int
	ReleaseID int
}

type listPagesResponse struct {
	Error *string       `json:"error"`
	Pages []models.Page `json:"pages"`
}

// listPages lists descriptive information about
func listPages(db *sql.DB, cfg *config.Config) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		request := listPagesRequest{}
		vars := mux.Vars(r)
		pid := vars["projectId"]
		rid := vars["releaseId"]
		projectId, parseErr1 := strconv.Atoi(pid)
		releaseId, parseErr2 := strconv.Atoi(rid)

		encoder := json.NewEncoder(w)
		if parseErr1 != nil || parseErr2 != nil {
			fmt.Printf("[---] Parse error: %v || %v\n", parseErr1, parseErr2)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "projectId and releaseId must be integer IDs."
			encoder.Encode(listPagesResponse{&errMsg, []models.Page{}})
			return
		}
		request.ProjectID = projectId
		request.ReleaseID = releaseId
		pages, listErr := models.ListPages(request.ReleaseID, db)
		if listErr != nil {
			fmt.Println("[---] List error:", listErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Could not list pages. Please check that the projectId is correct or try again later."
			encoder.Encode(listPagesResponse{&errMsg, []models.Page{}})
			return
		}
		encoder.Encode(listPagesResponse{nil, pages})
	}
}

// POST /projects/{projectId}/releases/{releaseId}/pages

type createPageRequest struct {
	ProjectID int    // Pulled from the URL parameters
	ReleaseID int    // Pulled from the URL parameters
	Number    string `json:"page"`
	ImageData string `json:"data"`
}

type createPageResponse struct {
	Error   *string `json:"error"`
	Success bool    `json:"success"`
	ID      int     `json:"id"`
}

// createPage inserts a new page into the DB and saves page data to a file.
func createPage(db *sql.DB, cfg *config.Config) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		request := createPageRequest{}
		vars := mux.Vars(r)
		pid := vars["projectId"]
		rid := vars["releaseId"]
		projectId, parseErr1 := strconv.Atoi(pid)
		releaseId, parseErr2 := strconv.Atoi(rid)
		decoder := json.NewDecoder(r.Body)
		defer r.Body.Close()
		decodeErr := decoder.Decode(&request)
		fmt.Printf("[+++] Project ID = %d, Release ID = %d\n", projectId, releaseId)

		encoder := json.NewEncoder(w)
		if parseErr1 != nil || parseErr2 != nil {
			fmt.Printf("[---] Parse error: %v || %v\n", parseErr1, parseErr2)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "projectId and releaseId must be integer IDs."
			encoder.Encode(createPageResponse{&errMsg, false, 0})
			return
		}
		request.ProjectID = projectId
		request.ReleaseID = releaseId
		if decodeErr != nil {
			fmt.Println("[---] Decode error:", decodeErr)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "JSON format error or missing field detected."
			encoder.Encode(createPageResponse{&errMsg, false, 0})
			return
		}
		imageData, decodeErr := base64.StdEncoding.DecodeString(request.ImageData)
		if decodeErr != nil {
			fmt.Println("[---] Image decode error:", decodeErr)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "The supplied image data is not base64 encoded."
			encoder.Encode(createPageResponse{&errMsg, false, 0})
			return
		}
		fmt.Println("[+++] Successfully decoded image data")
		var filePath string // The path that the image ends up being saved to.
		_, jpgParseErr := jpeg.Decode(bytes.NewReader(imageData))
		if jpgParseErr != nil {
			_, pngParseErr := png.Decode(bytes.NewReader(imageData))
			if pngParseErr != nil {
				// The image is neither a valid JPG/JPEG nor a valid PNG image.
				fmt.Printf("[---] Uploaded error: %v && %v\n", jpgParseErr, pngParseErr)
				w.WriteHeader(http.StatusBadRequest)
				errMsg := "The uploaded image is neither a valid JPG/JPEG or PNG image."
				encoder.Encode(createPageResponse{&errMsg, false, -1})
				return
			} else {
				// The image is a valid PNG image.
				filePath = randomFilename(cfg, "png")
			}
		} else {
			// The image is a valid JPG/JPEG image.
			filePath = randomFilename(cfg, "jpg")
		}
		fmt.Printf("[+++] Computed filename %s\n", filePath)
		f, saveErr := os.Create(filePath)
		if saveErr == nil {
			_, saveErr = f.Write(imageData)
		}
		defer f.Close()
		if saveErr != nil {
			fmt.Println("[---] Save error:", saveErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Failed to save image file. Please try again later."
			encoder.Encode(createPageResponse{&errMsg, false, -1})
			return
		}
		fmt.Println("[+++] Successfully saved image to disk")
		page := models.NewPage(request.Number, filePath, request.ReleaseID)
		saveErr = page.Save(db)
		if saveErr != nil {
			fmt.Println("[---] Insert error:", saveErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Failed to save page. Please try again later."
			encoder.Encode(createPageResponse{&errMsg, false, -1})
			return
		}
		encoder.Encode(createPageResponse{nil, true, page.Id})
	}
}

// GET /projects/{projectId}/releases/{releaseId}/pages/{pageId}

type getPageRequest struct {
	ProjectID int
	ReleaseID int
	PageID    int
}

// getPage retrieves the contents of a page from disk.
func getPage(db *sql.DB, cfg *config.Config) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		request := getPageRequest{}
		vars := mux.Vars(r)
		prid := vars["projectId"]
		reid := vars["releaseId"]
		paid := vars["pageId"]
		projectId, parseErr1 := strconv.Atoi(prid)
		releaseId, parseErr2 := strconv.Atoi(reid)
		pageId, parseErr3 := strconv.Atoi(paid)

		if parseErr1 != nil || parseErr2 != nil || parseErr3 != nil {
			fmt.Println("[---] Parse error: %v || %v || %v\n", parseErr1, parseErr2, parseErr3)
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte("Please ensure that each of the projectId, releaseId, and pageId parameters are valid integers."))
			return
		}
		request.ProjectID = projectId
		request.ReleaseID = releaseId
		request.PageID = pageId
		page, findErr := models.FindPage(request.PageID, db)
		if findErr != nil {
			fmt.Println("[---] Find error:", findErr)
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte("Could not find the requested page. Please ensure that the pageId is correct."))
			return
		}
		f, openErr := os.Open(page.Location)
		if openErr != nil {
			fmt.Println("[---] Open error:", openErr)
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte("Could not read the page file. Please try again later."))
			return
		}
		imageBytes, readErr := ioutil.ReadAll(f)
		defer f.Close()
		if readErr != nil {
			fmt.Println("[---] Open error:", openErr)
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte("Could not read the page file. Please try again later."))
			return
		}
		w.WriteHeader(http.StatusOK)
		if strings.HasSuffix(page.Location, "png") {
			w.Header().Set("Content-Type", "image/png")
		} else {
			w.Header().Set("Content-Type", "image/jpeg")
		}
		w.Write(imageBytes)
	}
}

// DELETE /projects/{projectId}/releases/{releaseId}/pages/{pageId}

type deletePageRequest struct {
	ProjectID int
	ReleaseID int
	PageID    int
}

type deletePageResponse struct {
	Error   *string `json:"error"`
	Success bool    `json:"success"`
}

// deletePage removes a page from the DB and deletes the file containing the image.
func deletePage(db *sql.DB, cfg *config.Config) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		request := deletePageRequest{}
		vars := mux.Vars(r)
		prid := vars["projectId"]
		reid := vars["releaseId"]
		paid := vars["pageId"]
		projectId, parseErr1 := strconv.Atoi(prid)
		releaseId, parseErr2 := strconv.Atoi(reid)
		pageId, parseErr3 := strconv.Atoi(paid)

		encoder := json.NewEncoder(w)
		if parseErr1 != nil || parseErr2 != nil || parseErr3 != nil {
			fmt.Println("[---] Parse error: %v || %v || %v\n", parseErr1, parseErr2, parseErr3)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "projectId, releaseId, and pageId must all be integer IDs."
			encoder.Encode(deletePageResponse{&errMsg, false})
			return
		}
		request.ProjectID = projectId
		request.ReleaseID = releaseId
		request.PageID = pageId
		// Unlike with projects and releases, we need to actually load the page we want to
		// delete from DB, because it needs the correct location of the page image on disk
		// to be able to delete that as well.
		page, findErr := models.FindPage(request.PageID, db)
		if findErr != nil {
			fmt.Println("[---] Find error:", findErr)
			w.WriteHeader(http.StatusBadRequest)
			errMsg := "Could not find the requested page. Please check that the page id is correct or try again later."
			encoder.Encode(deletePageResponse{&errMsg, false})
			return
		}
		fmt.Println("[+++] Attempting to delete page", page)
		deleteErr := page.Delete(db)
		if deleteErr != nil {
			fmt.Println("[---] Delete error:", findErr)
			w.WriteHeader(http.StatusInternalServerError)
			errMsg := "Could not delete the requested page. Please try again later."
			encoder.Encode(deletePageResponse{&errMsg, false})
			return
		}
		encoder.Encode(deletePageResponse{nil, true})
	}
}
<|endoftext|>


<commit_before>package goinsta

import (
	"encoding/json"
	"fmt"
	"strconv"
)

// Item represents media items
//
// All Item has
type Item struct {
	TakenAt          int     `json:"taken_at"`
	ID               int64   `json:"pk"`
	IDStr            string  `json:"id"`
	DeviceTimestamp  int64   `json:"device_timestamp"`
	MediaType        int     `json:"media_type"`
	Code             string  `json:"code"`
	ClientCacheKey   string  `json:"client_cache_key"`
	FilterType       int     `json:"filter_type"`
	CarouselParentID string  `json:"carousel_parent_id"`
	CarouselMedia    []Item  `json:"carousel_media,omitempty"`
	User             User    `json:"user"`
	CanViewerReshare bool    `json:"can_viewer_reshare"`
	Caption          Caption `json:"caption"`
	CaptionIsEdited  bool    `json:"caption_is_edited"`
	Likes            int     `json:"like_count"`
	HasLiked         bool    `json:"has_liked"`
	// _TopLikers can be `string` or `[]string`.
	// Use TopLikers function instead of getting it directly.
	_TopLikers                   interface{} `json:"top_likers"`
	CommentLikesEnabled          bool        `json:"comment_likes_enabled"`
	CommentThreadingEnabled      bool        `json:"comment_threading_enabled"`
	HasMoreComments              bool        `json:"has_more_comments"`
	MaxNumVisiblePreviewComments int         `json:"max_num_visible_preview_comments"`
	// _PreviewComments can be `string` or `[]string`.
	// Use PreviewComments function instead of getting it directly.
	_PreviewComments     interface{} `json:"preview_comments,omitempty"`
	CommentCount         int         `json:"comment_count"`
	PhotoOfYou           bool        `json:"photo_of_you"`
	Usertags             Tag         `json:"usertags,omitempty"`
	FbUserTags           Tag         `json:"fb_user_tags"`
	CanViewerSave        bool        `json:"can_viewer_save"`
	OrganicTrackingToken string      `json:"organic_tracking_token"`
	// Images contains URL images in different versions.
	// Version = quality.
	Images          Images `json:"image_versions2,omitempty"`
	OriginalWidth   int    `json:"original_width,omitempty"`
	OriginalHeight  int    `json:"original_height,omitempty"`
	ImportedTakenAt int    `json:"imported_taken_at,omitempty"`

	// Only for stories
	StoryEvents              []interface{} `json:"story_events"`
	StoryHashtags            []interface{} `json:"story_hashtags"`
	StoryPolls               []interface{} `json:"story_polls"`
	StoryFeedMedia           []interface{} `json:"story_feed_media"`
	StorySoundOn             []interface{} `json:"story_sound_on"`
	CreativeConfig           interface{}   `json:"creative_config"`
	StoryLocations           []interface{} `json:"story_locations"`
	StorySliders             []interface{} `json:"story_sliders"`
	StoryQuestions           []interface{} `json:"story_questions"`
	StoryProductItems        []interface{} `json:"story_product_items"`
	SupportsReelReactions    bool          `json:"supports_reel_reactions"`
	ShowOneTapFbShareTooltip bool          `json:"show_one_tap_fb_share_tooltip"`
	HasSharedToFb            int           `json:"has_shared_to_fb"`
	Mentions                 []Mentions
	Videos                   []Videos `json:"video_versions,omitempty"`
	HasAudio                 bool     `json:"has_audio,omitempty"`
	VideoDuration            float64  `json:"video_duration,omitempty"`
	IsDashEligible           int      `json:"is_dash_eligible,omitempty"`
	VideoDashManifest        string   `json:"video_dash_manifest,omitempty"`
	NumberOfQualities        int      `json:"number_of_qualities,omitempty"`
}

// TopLikers returns string slice or single string (inside string slice)
// Depending on TopLikers parameter.
func (item *Item) TopLikers() []string {
	switch s := item._TopLikers.(type) {
	case string:
		return []string{s}
	case []string:
		return s
	}
	return nil
}

// PreviewComments returns string slice or single string (inside string slice)
// Depending on PreviewComments parameter.
func (item *Item) PreviewComments() []string {
	switch s := item._PreviewComments.(type) {
	case string:
		return []string{s}
	case []string:
		return s
	}
	return nil
}

type Media interface {
	Next() error
}

type StoryMedia struct {
	inst     *Instagram
	endpoint string
	uid      int64

	ID              int      `json:"id"`
	LatestReelMedia int      `json:"latest_reel_media"`
	ExpiringAt      int      `json:"expiring_at"`
	Seen            float64  `json:"seen"`
	CanReply        bool     `json:"can_reply"`
	CanReshare      bool     `json:"can_reshare"`
	ReelType        string   `json:"reel_type"`
	User            User     `json:"user"`
	Items           []Item   `json:"items"`
	ReelMentions    []string `json:"reel_mentions"`
	PrefetchCount   int      `json:"prefetch_count"`
	HasBestiesMedia bool     `json:"has_besties_media"`
	Status          string   `json:"status"`
}

// Next allows to paginate after calling:
// User.Stories
func (media *StoryMedia) Next() (err error) {
	var body []byte
	insta := media.inst
	endpoint := media.endpoint

	if media.uid != 0 {
		endpoint = fmt.Sprintf(endpoint, media.uid)
	}
	body, err = insta.sendSimpleRequest(endpoint)
	if err == nil {
		m := StoryMedia{}
		err = json.Unmarshal(body, &m)
		if err == nil {
			err = ErrNoMore
			*media = m
			media.inst = insta
			media.endpoint = endpoint
			// TODO check NextID media
		}
	}
	return err
}

// Media represent a set of media items
type FeedMedia struct {
	inst *Instagram

	uid      int64
	endpoint string

	Items               []Item `json:"items"`
	NumResults          int    `json:"num_results"`
	MoreAvailable       bool   `json:"more_available"`
	AutoLoadMoreEnabled bool   `json:"auto_load_more_enabled"`
	Status              string `json:"status"`
	// Can be int64 and string
	// this is why recomend Next() usage :')
	NextID interface{} `json:"next_max_id"`
}

// Next allows to paginate after calling:
// User.Feed
//
// returns ErrNoMore when list reach the end.
func (media *FeedMedia) Next() (err error) {
	var body []byte
	insta := media.inst
	endpoint := media.endpoint
	next := ""

	switch s := media.NextID.(type) {
	case string:
		next = s
	case int64:
		next = strconv.FormatInt(s, 10)
	}

	if media.uid != 0 {
		endpoint = fmt.Sprintf(endpoint, media.uid)
	}

	body, err = insta.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":         next,
				"rank_token":     insta.rankToken,
				"min_timestamp":  "",
				"ranked_content": "true",
			},
		},
	)
	if err == nil {
		m := FeedMedia{}
		err = json.Unmarshal(body, &m)
		if err == nil {
			*media = m
			media.inst = insta
			media.endpoint = endpoint
			if m.NextID == 0 || !m.MoreAvailable {
				err = ErrNoMore
			}
		}
	}
	return err
}
<commit_msg>Changed pagination function mode<commit_after>package goinsta

import (
	"encoding/json"
	"fmt"
	"strconv"
)

// Item represents media items
//
// All Item has
type Item struct {
	TakenAt          int     `json:"taken_at"`
	ID               int64   `json:"pk"`
	IDStr            string  `json:"id"`
	DeviceTimestamp  int64   `json:"device_timestamp"`
	MediaType        int     `json:"media_type"`
	Code             string  `json:"code"`
	ClientCacheKey   string  `json:"client_cache_key"`
	FilterType       int     `json:"filter_type"`
	CarouselParentID string  `json:"carousel_parent_id"`
	CarouselMedia    []Item  `json:"carousel_media,omitempty"`
	User             User    `json:"user"`
	CanViewerReshare bool    `json:"can_viewer_reshare"`
	Caption          Caption `json:"caption"`
	CaptionIsEdited  bool    `json:"caption_is_edited"`
	Likes            int     `json:"like_count"`
	HasLiked         bool    `json:"has_liked"`
	// _TopLikers can be `string` or `[]string`.
	// Use TopLikers function instead of getting it directly.
	_TopLikers                   interface{} `json:"top_likers"`
	CommentLikesEnabled          bool        `json:"comment_likes_enabled"`
	CommentThreadingEnabled      bool        `json:"comment_threading_enabled"`
	HasMoreComments              bool        `json:"has_more_comments"`
	MaxNumVisiblePreviewComments int         `json:"max_num_visible_preview_comments"`
	// _PreviewComments can be `string` or `[]string`.
	// Use PreviewComments function instead of getting it directly.
	_PreviewComments     interface{} `json:"preview_comments,omitempty"`
	CommentCount         int         `json:"comment_count"`
	PhotoOfYou           bool        `json:"photo_of_you"`
	Usertags             Tag         `json:"usertags,omitempty"`
	FbUserTags           Tag         `json:"fb_user_tags"`
	CanViewerSave        bool        `json:"can_viewer_save"`
	OrganicTrackingToken string      `json:"organic_tracking_token"`
	// Images contains URL images in different versions.
	// Version = quality.
	Images          Images `json:"image_versions2,omitempty"`
	OriginalWidth   int    `json:"original_width,omitempty"`
	OriginalHeight  int    `json:"original_height,omitempty"`
	ImportedTakenAt int    `json:"imported_taken_at,omitempty"`

	// Only for stories
	StoryEvents              []interface{} `json:"story_events"`
	StoryHashtags            []interface{} `json:"story_hashtags"`
	StoryPolls               []interface{} `json:"story_polls"`
	StoryFeedMedia           []interface{} `json:"story_feed_media"`
	StorySoundOn             []interface{} `json:"story_sound_on"`
	CreativeConfig           interface{}   `json:"creative_config"`
	StoryLocations           []interface{} `json:"story_locations"`
	StorySliders             []interface{} `json:"story_sliders"`
	StoryQuestions           []interface{} `json:"story_questions"`
	StoryProductItems        []interface{} `json:"story_product_items"`
	SupportsReelReactions    bool          `json:"supports_reel_reactions"`
	ShowOneTapFbShareTooltip bool          `json:"show_one_tap_fb_share_tooltip"`
	HasSharedToFb            int           `json:"has_shared_to_fb"`
	Mentions                 []Mentions
	Videos                   []Videos `json:"video_versions,omitempty"`
	HasAudio                 bool     `json:"has_audio,omitempty"`
	VideoDuration            float64  `json:"video_duration,omitempty"`
	IsDashEligible           int      `json:"is_dash_eligible,omitempty"`
	VideoDashManifest        string   `json:"video_dash_manifest,omitempty"`
	NumberOfQualities        int      `json:"number_of_qualities,omitempty"`
}

// TopLikers returns string slice or single string (inside string slice)
// Depending on TopLikers parameter.
func (item *Item) TopLikers() []string {
	switch s := item._TopLikers.(type) {
	case string:
		return []string{s}
	case []string:
		return s
	}
	return nil
}

// PreviewComments returns string slice or single string (inside string slice)
// Depending on PreviewComments parameter.
func (item *Item) PreviewComments() []string {
	switch s := item._PreviewComments.(type) {
	case string:
		return []string{s}
	case []string:
		return s
	}
	return nil
}

type Media interface {
	Next() error
}

type StoryMedia struct {
	inst     *Instagram
	endpoint string
	uid      int64

	ID              int      `json:"id"`
	LatestReelMedia int      `json:"latest_reel_media"`
	ExpiringAt      int      `json:"expiring_at"`
	Seen            float64  `json:"seen"`
	CanReply        bool     `json:"can_reply"`
	CanReshare      bool     `json:"can_reshare"`
	ReelType        string   `json:"reel_type"`
	User            User     `json:"user"`
	Items           []Item   `json:"items"`
	ReelMentions    []string `json:"reel_mentions"`
	PrefetchCount   int      `json:"prefetch_count"`
	HasBestiesMedia bool     `json:"has_besties_media"`
	Status          string   `json:"status"`
}

// Next allows to paginate after calling:
// User.Stories
func (media *StoryMedia) Next() (err error) {
	var body []byte
	insta := media.inst
	endpoint := media.endpoint
	if media.uid != 0 {
		endpoint = fmt.Sprintf(endpoint, media.uid)
	}

	body, err = insta.sendSimpleRequest(endpoint)
	if err == nil {
		m := StoryMedia{}
		err = json.Unmarshal(body, &m)
		if err == nil {
			// TODO check NextID media
			err = ErrNoMore
			*media = m
			media.inst = insta
			media.endpoint = endpoint
		}
	}
	return err
}

// Media represent a set of media items
type FeedMedia struct {
	inst *Instagram

	uid       int64
	endpoint  string
	timestamp string

	Items               []Item `json:"items"`
	NumResults          int    `json:"num_results"`
	MoreAvailable       bool   `json:"more_available"`
	AutoLoadMoreEnabled bool   `json:"auto_load_more_enabled"`
	Status              string `json:"status"`
	// Can be int64 and string
	// this is why recomend Next() usage :')
	NextID interface{} `json:"next_max_id"`
}

// Next allows to paginate after calling:
// User.Feed
//
// returns ErrNoMore when list reach the end.
func (media *FeedMedia) Next() (err error) {
	var body []byte
	insta := media.inst
	endpoint := media.endpoint
	next := ""

	switch s := media.NextID.(type) {
	case string:
		next = s
	case int64:
		next = strconv.FormatInt(s, 10)
	}

	if media.uid != 0 {
		endpoint = fmt.Sprintf(endpoint, media.uid)
	}

	body, err = insta.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":         next,
				"rank_token":     insta.rankToken,
				"min_timestamp":  media.timestamp,
				"ranked_content": "true",
			},
		},
	)
	if err == nil {
		m := FeedMedia{}
		err = json.Unmarshal(body, &m)
		if err == nil {
			*media = m
			media.inst = insta
			media.endpoint = endpoint
			if m.NextID == 0 || !m.MoreAvailable {
				err = ErrNoMore
			}
		}
	}
	return err
}
<|endoftext|>


<commit_before>package experiments

import (
	"time"
	"sort"
	"github.com/yaricom/goNEAT/neat/genetics"
	"io"
	"encoding/gob"
	"fmt"
)

// An Experiment is a collection of trials for one experiment. It's useful for statistical analysis of a series of
// experiments
type Experiment struct {
	Id   int
	Name string
	Trials
}

func (e Experiment) LastExecuted() time.Time {
	var u time.Time
	for _, e := range e.Trials {
		ut := e.LastExecuted()
		if u.Before(ut) {
			u = ut
		}
	}
	return u
}

// Finds the most fit organism among all epochs in this trial. It's also possible to get the best organism only among the ones
// which was able to solve the experiment's problem. Returns the best fit organism in this experiment among with ID of trial
// where it was found and boolean value to indicate if search was successful.
func (e Experiment) BestOrganism(onlySolvers bool) (*genetics.Organism, int, bool) {
	var orgs = make(genetics.Organisms, 0, len(e.Trials))
	for i, t := range e.Trials {
		org, found := t.BestOrganism(onlySolvers)
		if found {
			orgs = append(orgs, org)
			org.Flag = i
		}

	}
	if len(orgs) > 0 {
		sort.Sort(sort.Reverse(orgs))
		return orgs[0], orgs[0].Flag, true
	} else {
		return nil, -1, false
	}

}

func (e Experiment) Solved() bool {
	for _, t := range e.Trials {
		if t.Solved() {
			return true
		}
	}
	return false
}

// The fitness values of the best organisms for each trial
func (e Experiment) BestFitness() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		if org, ok := t.BestOrganism(false); ok {
			x[i] = org.Fitness
		}
	}
	return x
}

// The age values of the organisms for each trial
func (e Experiment) BestAge() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		if org, ok := t.BestOrganism(false); ok {
			x[i] = float64(org.Species.Age)
		}
	}
	return x
}

// The complexity values of the best organisms for each trial
func (e Experiment) BestComplexity() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		if org, ok := t.BestOrganism(false); ok {
			x[i] = float64(org.Phenotype.Complexity())
		}
	}
	return x
}

// Diversity returns the average number of species in each trial
func (e Experiment) Diversity() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		x[i] = t.Diversity().Mean()
	}
	return x
}

// Trials returns the number of epochs in each trial
func (e Experiment) Epochs() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		x[i] = float64(len(t.Generations))
	}
	return x
}

// Returns average number of nodes, genes, organisms evaluations, and species diversity of winner genomes among all
// trials, i.e. for all trials where winning solution was found
func (e Experiment) AvgWinner() (avg_nodes, avg_genes, avg_evals, avg_diversity float64) {
	total_nodes, total_genes, total_evals, total_diversity := 0, 0, 0, 0
	count := 0
	for _, t := range e.Trials {
		if t.Solved() {
			nodes, genes, evals, diversity := t.Winner()
			total_nodes += nodes
			total_genes += genes
			total_evals += evals
			total_diversity += diversity

			count++
		}
	}
	avg_nodes = float64(total_nodes) / float64(count)
	avg_genes = float64(total_genes) / float64(count)
	avg_evals = float64(total_evals) / float64(count)
	avg_diversity = float64(total_diversity) / float64(count)
	return avg_nodes, avg_genes, avg_evals, avg_diversity
}

// Prints experiment statistics
func (ex Experiment) PrintStatistics() {
	// Print absolute champion statistics
	if org, trid, found := ex.BestOrganism(true); found {
		nodes, genes, evals, divers := ex.Trials[trid].Winner()
		fmt.Printf("\nChampion found in %d trial\n\tWinner Nodes:\t%.1f\n\tWinner Genes:\t%.1f\n\tWinner Evals:\t%.1f\n\tDiversity:\t%.1f",
			trid, nodes, genes, evals, divers)
		fmt.Printf("\n\tComplexity:\t%d\n\tAge:\t\t%d\n\tFitness:\t%.1f\n",
			org.Phenotype.Complexity(), org.Species.Age, org.Fitness)
	} else {
		fmt.Println("\nNo winner found in the experiment!!!")
	}

	// Print average winner statistics
	if len(ex.Trials) > 1 {
		avg_nodes, avg_genes, avg_evals, avg_divers := ex.AvgWinner()

		fmt.Printf("\nAverage\n\tWinner Nodes:\t%.1f\n\tWinner Genes:\t%.1f\n\tWinner Evals:\t%.1f\n\tDiversity:\t%.1f\n",
			avg_nodes, avg_genes, avg_evals, avg_divers)
	}

	// Print best organisms statistics per epoch per trial, i.e. every the best found
	mean_complexity, mean_diversity, mean_age, mean_fitness := 0.0, 0.0, 0.0, 0.0
	for _, t := range ex.Trials {
		mean_complexity += t.BestComplexity().Mean()
		mean_diversity += t.Diversity().Mean()
		mean_age += t.BestAge().Mean()
		mean_fitness += t.BestFitness().Mean()
	}
	count := float64(len(ex.Trials))
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	mean_fitness /=count
	fmt.Printf("Mean for the all most fit/best organisms found\n\tComplexity:\t%.1f\n\tDiversity:\t%.1f\n\tAge:\t\t%.1f\n\tFitness:\t%.1f\n",
		mean_complexity, mean_diversity, mean_age, mean_fitness)

	// Print the average values for each population of organisms evaluated
	for _, t := range ex.Trials {
		fitness, age, complexity := t.Average()

		mean_complexity += complexity
		mean_diversity += t.Diversity().Mean()
		mean_age += age
		mean_fitness += fitness
	}
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	mean_fitness /=count
	fmt.Printf("\nAverages for each population of organisms evaluated\n\tComplexity:\t%.1f\n\tDiversity:\t%.1f\n\tAge:\t\t%.1f\n\tFitness:\t%.1f\n",
		mean_complexity, mean_diversity, mean_age, mean_fitness)

}

// Encodes experiment and writes to provided writer
func (ex Experiment) Write(w io.Writer) error {
	enc := gob.NewEncoder(w)
	return ex.Encode(enc)
}

// Encodes experiment with GOB encoding
func (ex Experiment) Encode(enc *gob.Encoder) error {
	err := enc.Encode(ex.Id)
	err = enc.Encode(ex.Name)

	// encode trials
	err = enc.Encode(len(ex.Trials))
	for _, t := range ex.Trials {
		err = t.Encode(enc)
		if err != nil {
			return err
		}
	}
	return err
}

// Reads experiment data from provided reader and decodes it
func (ex *Experiment) Read(r io.Reader) error {
	dec := gob.NewDecoder(r)
	return ex.Decode(dec)
}

// Decodes experiment data
func (ex *Experiment) Decode(dec *gob.Decoder) error {
	err := dec.Decode(&ex.Id)
	err = dec.Decode(&ex.Name)

	// decode Trials
	var t_num int
	err = dec.Decode(&t_num)
	if err != nil {
		return err
	}

	ex.Trials = make([]Trial, t_num)
	for i := 0; i < t_num; i++ {
		trial := Trial{}
		err = trial.Decode(dec)
		ex.Trials[i] = trial
	}
	return err
}

// Experiments is a sortable list of experiments by execution time and Id
type Experiments []Experiment

func (es Experiments) Len() int {
	return len(es)
}
func (es Experiments) Swap(i, j int) {
	es[i], es[j] = es[j], es[i]
}
func (es Experiments) Less(i, j int) bool {
	ui := es[i].LastExecuted()
	uj := es[j].LastExecuted()
	if ui.Equal(uj) {
		return es[i].Id < es[j].Id
	}
	return ui.Before(uj)
}<commit_msg>Fixed print statistics formatting<commit_after>package experiments

import (
	"time"
	"sort"
	"github.com/yaricom/goNEAT/neat/genetics"
	"io"
	"encoding/gob"
	"fmt"
)

// An Experiment is a collection of trials for one experiment. It's useful for statistical analysis of a series of
// experiments
type Experiment struct {
	Id   int
	Name string
	Trials
}

func (e Experiment) LastExecuted() time.Time {
	var u time.Time
	for _, e := range e.Trials {
		ut := e.LastExecuted()
		if u.Before(ut) {
			u = ut
		}
	}
	return u
}

// Finds the most fit organism among all epochs in this trial. It's also possible to get the best organism only among the ones
// which was able to solve the experiment's problem. Returns the best fit organism in this experiment among with ID of trial
// where it was found and boolean value to indicate if search was successful.
func (e Experiment) BestOrganism(onlySolvers bool) (*genetics.Organism, int, bool) {
	var orgs = make(genetics.Organisms, 0, len(e.Trials))
	for i, t := range e.Trials {
		org, found := t.BestOrganism(onlySolvers)
		if found {
			orgs = append(orgs, org)
			org.Flag = i
		}

	}
	if len(orgs) > 0 {
		sort.Sort(sort.Reverse(orgs))
		return orgs[0], orgs[0].Flag, true
	} else {
		return nil, -1, false
	}

}

func (e Experiment) Solved() bool {
	for _, t := range e.Trials {
		if t.Solved() {
			return true
		}
	}
	return false
}

// The fitness values of the best organisms for each trial
func (e Experiment) BestFitness() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		if org, ok := t.BestOrganism(false); ok {
			x[i] = org.Fitness
		}
	}
	return x
}

// The age values of the organisms for each trial
func (e Experiment) BestAge() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		if org, ok := t.BestOrganism(false); ok {
			x[i] = float64(org.Species.Age)
		}
	}
	return x
}

// The complexity values of the best organisms for each trial
func (e Experiment) BestComplexity() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		if org, ok := t.BestOrganism(false); ok {
			x[i] = float64(org.Phenotype.Complexity())
		}
	}
	return x
}

// Diversity returns the average number of species in each trial
func (e Experiment) Diversity() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		x[i] = t.Diversity().Mean()
	}
	return x
}

// Trials returns the number of epochs in each trial
func (e Experiment) Epochs() Floats {
	var x Floats = make([]float64, len(e.Trials))
	for i, t := range e.Trials {
		x[i] = float64(len(t.Generations))
	}
	return x
}

// Returns average number of nodes, genes, organisms evaluations, and species diversity of winner genomes among all
// trials, i.e. for all trials where winning solution was found
func (e Experiment) AvgWinner() (avg_nodes, avg_genes, avg_evals, avg_diversity float64) {
	total_nodes, total_genes, total_evals, total_diversity := 0, 0, 0, 0
	count := 0
	for _, t := range e.Trials {
		if t.Solved() {
			nodes, genes, evals, diversity := t.Winner()
			total_nodes += nodes
			total_genes += genes
			total_evals += evals
			total_diversity += diversity

			count++
		}
	}
	avg_nodes = float64(total_nodes) / float64(count)
	avg_genes = float64(total_genes) / float64(count)
	avg_evals = float64(total_evals) / float64(count)
	avg_diversity = float64(total_diversity) / float64(count)
	return avg_nodes, avg_genes, avg_evals, avg_diversity
}

// Prints experiment statistics
func (ex Experiment) PrintStatistics() {
	// Print absolute champion statistics
	if org, trid, found := ex.BestOrganism(true); found {
		nodes, genes, evals, divers := ex.Trials[trid].Winner()
		fmt.Printf("\nChampion found in %d trial\n\tWinner Nodes:\t%d\n\tWinner Genes:\t%d\n\tWinner Evals:\t%d\n\n\tDiversity:\t%d",
			trid, nodes, genes, evals, divers)
		fmt.Printf("\n\tComplexity:\t%d\n\tAge:\t\t%d\n\tFitness:\t%.1f\n",
			org.Phenotype.Complexity(), org.Species.Age, org.Fitness)
	} else {
		fmt.Println("\nNo winner found in the experiment!!!")
	}

	// Print average winner statistics
	if len(ex.Trials) > 1 {
		avg_nodes, avg_genes, avg_evals, avg_divers := ex.AvgWinner()

		fmt.Printf("\nAverage among winners\n\tWinner Nodes:\t%.1f\n\tWinner Genes:\t%.1f\n\tWinner Evals:\t%.1f\n\tDiversity:\t%.1f\n",
			avg_nodes, avg_genes, avg_evals, avg_divers)
	}

	// Print best organisms statistics per epoch per trial, i.e. every the best found
	mean_complexity, mean_diversity, mean_age, mean_fitness := 0.0, 0.0, 0.0, 0.0
	for _, t := range ex.Trials {
		mean_complexity += t.BestComplexity().Mean()
		mean_diversity += t.Diversity().Mean()
		mean_age += t.BestAge().Mean()
		mean_fitness += t.BestFitness().Mean()
	}
	count := float64(len(ex.Trials))
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	mean_fitness /=count
	fmt.Printf("\nMean for the most fit/best organisms found\n\tComplexity:\t%.1f\n\tDiversity:\t%.1f\n\tAge:\t\t%.1f\n\tFitness:\t%.1f\n",
		mean_complexity, mean_diversity, mean_age, mean_fitness)

	// Print the average values for each population of organisms evaluated
	for _, t := range ex.Trials {
		fitness, age, complexity := t.Average()

		mean_complexity += complexity.Mean()
		mean_diversity += t.Diversity().Mean()
		mean_age += age.Mean()
		mean_fitness += fitness.Mean()
	}
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	mean_fitness /=count
	fmt.Printf("\nAverages for each population of organisms evaluated\n\tComplexity:\t%.1f\n\tDiversity:\t%.1f\n\tAge:\t\t%.1f\n\tFitness:\t%.1f\n\n",
		mean_complexity, mean_diversity, mean_age, mean_fitness)

}

// Encodes experiment and writes to provided writer
func (ex Experiment) Write(w io.Writer) error {
	enc := gob.NewEncoder(w)
	return ex.Encode(enc)
}

// Encodes experiment with GOB encoding
func (ex Experiment) Encode(enc *gob.Encoder) error {
	err := enc.Encode(ex.Id)
	err = enc.Encode(ex.Name)

	// encode trials
	err = enc.Encode(len(ex.Trials))
	for _, t := range ex.Trials {
		err = t.Encode(enc)
		if err != nil {
			return err
		}
	}
	return err
}

// Reads experiment data from provided reader and decodes it
func (ex *Experiment) Read(r io.Reader) error {
	dec := gob.NewDecoder(r)
	return ex.Decode(dec)
}

// Decodes experiment data
func (ex *Experiment) Decode(dec *gob.Decoder) error {
	err := dec.Decode(&ex.Id)
	err = dec.Decode(&ex.Name)

	// decode Trials
	var t_num int
	err = dec.Decode(&t_num)
	if err != nil {
		return err
	}

	ex.Trials = make([]Trial, t_num)
	for i := 0; i < t_num; i++ {
		trial := Trial{}
		err = trial.Decode(dec)
		ex.Trials[i] = trial
	}
	return err
}

// Experiments is a sortable list of experiments by execution time and Id
type Experiments []Experiment

func (es Experiments) Len() int {
	return len(es)
}
func (es Experiments) Swap(i, j int) {
	es[i], es[j] = es[j], es[i]
}
func (es Experiments) Less(i, j int) bool {
	ui := es[i].LastExecuted()
	uj := es[j].LastExecuted()
	if ui.Equal(uj) {
		return es[i].Id < es[j].Id
	}
	return ui.Before(uj)
}<|endoftext|>


<commit_before>package memdb

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/t3rm1n4l/memdb/skiplist"
	"io"
	"math/rand"
	"os"
	"path"
	"runtime"
	"sync"
	"sync/atomic"
)

type KeyCompare func([]byte, []byte) int
type ItemCallback func(*Item)

type FileType int

const (
	encodeBufSize = 4
	readerBufSize = 10000
)

const (
	ForestdbFile FileType = iota
	RawdbFile
)

func DefaultConfig() Config {
	var cfg Config
	cfg.SetKeyComparator(defaultKeyCmp)
	cfg.SetFileType(ForestdbFile)
	return cfg
}

type Item struct {
	bornSn, deadSn uint32
	data           []byte
}

func (itm *Item) Encode(buf []byte, w io.Writer) error {
	l := 2
	if len(buf) < l {
		return ErrNotEnoughSpace
	}

	binary.BigEndian.PutUint16(buf[0:2], uint16(len(itm.data)))
	if _, err := w.Write(buf[0:2]); err != nil {
		return err
	}
	if _, err := w.Write(itm.data); err != nil {
		return err
	}

	return nil
}

func (itm *Item) Decode(buf []byte, r io.Reader) error {
	if _, err := io.ReadFull(r, buf[0:2]); err != nil {
		return err
	}
	l := binary.BigEndian.Uint16(buf[0:2])
	itm.data = make([]byte, int(l))
	_, err := io.ReadFull(r, itm.data)

	return err
}

func (itm *Item) Bytes() []byte {
	return itm.data
}

func NewItem(data []byte) *Item {
	return &Item{
		data: data,
	}
}

func newInsertCompare(keyCmp KeyCompare) skiplist.CompareFn {
	return func(this skiplist.Item, that skiplist.Item) int {
		var v int
		thisItem := this.(*Item)
		thatItem := that.(*Item)
		if v = keyCmp(thisItem.data, thatItem.data); v == 0 {
			v = int(thisItem.bornSn) - int(thatItem.bornSn)
		}

		return v
	}
}

func newIterCompare(keyCmp KeyCompare) skiplist.CompareFn {
	return func(this skiplist.Item, that skiplist.Item) int {
		thisItem := this.(*Item)
		thatItem := that.(*Item)
		return keyCmp(thisItem.data, thatItem.data)
	}
}

func defaultKeyCmp(this []byte, that []byte) int {
	var l int

	l1 := len(this)
	l2 := len(that)
	if l1 < l2 {
		l = l1
	} else {
		l = l2
	}

	return bytes.Compare(this[:l], that[:l])
}

//
//compare item,sn
type Writer struct {
	rand *rand.Rand
	buf  *skiplist.ActionBuffer
	iter *skiplist.Iterator
	*MemDB
}

func (w *Writer) Put(x *Item) {
	x.bornSn = w.getCurrSn()
	w.store.Insert2(x, w.insCmp, w.buf, w.rand.Float32)
	atomic.AddInt64(&w.count, 1)
}

// Find first item, seek until dead=0, mark dead=sn
func (w *Writer) Delete(x *Item) (success bool) {
	defer func() {
		if success {
			atomic.AddInt64(&w.count, -1)
		}
	}()

	gotItem := w.Get(x)
	if gotItem != nil {
		sn := w.getCurrSn()
		if gotItem.bornSn == sn {
			success = w.store.Delete(gotItem, w.insCmp, w.buf)
			return
		}

		success = atomic.CompareAndSwapUint32(&gotItem.deadSn, 0, sn)
		return
	}

	return
}

func (w *Writer) Get(x *Item) *Item {
	var curr *Item
	found := w.iter.Seek(x)
	if !found {
		return nil
	}

	// Seek until most recent item for key is found
	curr = w.iter.Get().(*Item)
	for {
		w.iter.Next()
		if !w.iter.Valid() {
			break
		}
		next := w.iter.Get().(*Item)
		if w.iterCmp(next, curr) != 0 {
			break
		}

		curr = next
	}

	if curr.deadSn != 0 {
		return nil
	}

	return curr
}

type Config struct {
	keyCmp  KeyCompare
	insCmp  skiplist.CompareFn
	iterCmp skiplist.CompareFn

	fileType FileType
}

func (cfg *Config) SetKeyComparator(cmp KeyCompare) {
	cfg.keyCmp = cmp
	cfg.insCmp = newInsertCompare(cmp)
	cfg.iterCmp = newIterCompare(cmp)
}

func (cfg *Config) SetFileType(t FileType) error {
	switch t {
	case ForestdbFile, RawdbFile:
	default:
		return errors.New("Invalid format")
	}

	cfg.fileType = t
	return nil
}

type MemDB struct {
	store        *skiplist.Skiplist
	currSn       uint32
	snapshots    *skiplist.Skiplist
	isGCRunning  int32
	lastGCSn     uint32
	leastUnrefSn uint32
	count        int64

	Config
}

func NewWithConfig(cfg Config) *MemDB {
	m := &MemDB{
		store:     skiplist.New(),
		snapshots: skiplist.New(),
		currSn:    1,
		Config:    cfg,
	}

	return m

}

func New() *MemDB {
	return NewWithConfig(DefaultConfig())
}

func (m *MemDB) Reset() {
	m.store = skiplist.New()
	m.snapshots = skiplist.New()
	m.currSn = 1
}

func (m *MemDB) getCurrSn() uint32 {
	return atomic.LoadUint32(&m.currSn)
}

func (m *MemDB) setLeastUnrefSn() {
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)
	iter := m.snapshots.NewIterator(CompareSnapshot, buf)
	iter.SeekFirst()
	if iter.Valid() {
		snap := iter.Get().(*Snapshot)
		atomic.StoreUint32(&m.leastUnrefSn, snap.sn-1)
	}
}

func (m *MemDB) getLeastUnrefSn() uint32 {
	return atomic.LoadUint32(&m.leastUnrefSn)
}

func (m *MemDB) NewWriter() *Writer {
	buf := m.store.MakeBuf()

	return &Writer{
		rand:  rand.New(rand.NewSource(int64(rand.Int()))),
		buf:   buf,
		iter:  m.store.NewIterator(m.iterCmp, buf),
		MemDB: m,
	}
}

type Snapshot struct {
	sn       uint32
	refCount int32
	db       *MemDB
	count    int64
}

func (s Snapshot) Count() int64 {
	return s.count
}

func (s *Snapshot) Encode(buf []byte, w io.Writer) error {
	l := 4
	if len(buf) < l {
		return ErrNotEnoughSpace
	}

	binary.BigEndian.PutUint32(buf[0:4], s.sn)
	if _, err := w.Write(buf[0:4]); err != nil {
		return err
	}

	return nil

}

func (s *Snapshot) Decode(buf []byte, r io.Reader) error {
	if _, err := io.ReadFull(r, buf[0:4]); err != nil {
		return err
	}
	s.sn = binary.BigEndian.Uint32(buf[0:4])
	return nil
}

func (s *Snapshot) Open() bool {
	if atomic.LoadInt32(&s.refCount) == 0 {
		return false
	}
	atomic.AddInt32(&s.refCount, 1)
	return true
}

func (s *Snapshot) Close() {
	newRefcount := atomic.AddInt32(&s.refCount, -1)
	if newRefcount == 0 {
		buf := s.db.snapshots.MakeBuf()
		defer s.db.snapshots.FreeBuf(buf)
		s.db.snapshots.Delete(s, CompareSnapshot, buf)
		s.db.setLeastUnrefSn()
		if atomic.CompareAndSwapInt32(&s.db.isGCRunning, 0, 1) {
			go s.db.GC()
		}
	}
}

func (s *Snapshot) NewIterator() *Iterator {
	return s.db.NewIterator(s)
}

func CompareSnapshot(this skiplist.Item, that skiplist.Item) int {
	thisItem := this.(*Snapshot)
	thatItem := that.(*Snapshot)

	return int(thisItem.sn) - int(thatItem.sn)
}

func (m *MemDB) NewSnapshot() *Snapshot {
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)

	snap := &Snapshot{db: m, sn: m.getCurrSn(), refCount: 1, count: m.ItemsCount()}
	m.snapshots.Insert(snap, CompareSnapshot, buf)
	atomic.AddUint32(&m.currSn, 1)
	return snap
}

type Iterator struct {
	snap *Snapshot
	iter *skiplist.Iterator
	buf  *skiplist.ActionBuffer
}

func (it *Iterator) skipUnwanted() {
loop:
	if !it.iter.Valid() {
		return
	}
	itm := it.iter.Get().(*Item)
	if itm.bornSn > it.snap.sn || (itm.deadSn > 0 && itm.deadSn <= it.snap.sn) {
		it.iter.Next()
		goto loop
	}
}

func (it *Iterator) SeekFirst() {
	it.iter.SeekFirst()
	it.skipUnwanted()
}

func (it *Iterator) Seek(itm *Item) {
	it.iter.Seek(itm)
	it.skipUnwanted()
}

func (it *Iterator) Valid() bool {
	return it.iter.Valid()
}

func (it *Iterator) Get() *Item {
	return it.iter.Get().(*Item)
}

func (it *Iterator) MarkDeletion() {
	it.iter.MarkDeletion()
}

func (it *Iterator) Next() {
	it.iter.Next()
	it.skipUnwanted()
}

func (it *Iterator) Close() {
	it.snap.Close()
	it.snap.db.store.FreeBuf(it.buf)
}

func (m *MemDB) NewIterator(snap *Snapshot) *Iterator {
	if !snap.Open() {
		return nil
	}
	buf := snap.db.store.MakeBuf()
	return &Iterator{
		snap: snap,
		iter: m.store.NewIterator(m.iterCmp, buf),
		buf:  buf,
	}
}

func (m *MemDB) ItemsCount() int64 {
	return atomic.LoadInt64(&m.count)
}

func (m *MemDB) collectDead(sn uint32) {
	// Mark and sweep GC
	buf1 := m.snapshots.MakeBuf()
	buf2 := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf1)
	defer m.snapshots.FreeBuf(buf2)
	iter := m.store.NewIterator(m.iterCmp, buf1)

	// Mark phase
	for iter.SeekFirst(); iter.Valid(); iter.Next() {
		itm := iter.Get().(*Item)
		if itm.deadSn > 0 && itm.deadSn <= m.getLeastUnrefSn() {
			iter.MarkDeletion()
		}
	}

	// Sweep phase
	for iter.SeekFirst(); iter.Valid(); iter.Next() {
	}
}

func (m *MemDB) GC() {
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)

	iter := m.snapshots.NewIterator(CompareSnapshot, buf)
	iter.SeekFirst()
	if iter.Valid() {
		snap := iter.Get().(*Snapshot)
		if snap.sn != m.lastGCSn && snap.sn > 1 {
			m.lastGCSn = snap.sn - 1
			m.collectDead(m.lastGCSn)
		}
	}

	atomic.CompareAndSwapInt32(&m.isGCRunning, 1, 0)
}

func (m *MemDB) GetSnapshots() []*Snapshot {
	var snaps []*Snapshot
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)
	iter := m.snapshots.NewIterator(CompareSnapshot, buf)
	iter.SeekFirst()
	for ; iter.Valid(); iter.Next() {
		snaps = append(snaps, iter.Get().(*Snapshot))
	}

	return snaps
}

func (m *MemDB) StoreToDisk(dir string, snap *Snapshot, callb ItemCallback) error {
	os.MkdirAll(dir, 0755)
	datafile := path.Join(dir, "records.data")
	w := newFileWriter(m.fileType)
	if err := w.Open(datafile); err != nil {
		return err
	}
	defer w.Close()

	itr := m.NewIterator(snap)
	if itr == nil {
		return errors.New("Invalid snapshot")
	}
	defer itr.Close()

	for itr.SeekFirst(); itr.Valid(); itr.Next() {
		itm := itr.Get()
		if err := w.WriteItem(itm); err != nil {
			return err
		}

		if callb != nil {
			callb(itm)
		}
	}

	return nil
}

func (m *MemDB) LoadFromDisk(dir string, callb ItemCallback) (*Snapshot, error) {
	var wg sync.WaitGroup
	datafile := path.Join(dir, "records.data")
	r := newFileReader(m.fileType)
	if err := r.Open(datafile); err != nil {
		return nil, err
	}

	defer r.Close()

	ch := make(chan *Item, readerBufSize)
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go func(wg *sync.WaitGroup) {
			defer wg.Done()
			w := m.NewWriter()
			for itm := range ch {
				w.Put(itm)
				if callb != nil {
					callb(itm)
				}
			}
		}(&wg)
	}

loop:
	for {
		itm, err := r.ReadItem()
		if err != nil {
			return nil, err
		}

		if itm == nil {
			break loop
		}
		ch <- itm
	}

	close(ch)
	wg.Wait()

	snap := m.NewSnapshot()
	return snap, nil
}

func (m *MemDB) DumpStats() string {
	return m.store.GetStats().String()
}
<commit_msg>memdb: Use rawdb for snapshot file as default<commit_after>package memdb

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/t3rm1n4l/memdb/skiplist"
	"io"
	"math/rand"
	"os"
	"path"
	"runtime"
	"sync"
	"sync/atomic"
)

type KeyCompare func([]byte, []byte) int
type ItemCallback func(*Item)

type FileType int

const (
	encodeBufSize = 4
	readerBufSize = 10000
)

const (
	ForestdbFile FileType = iota
	RawdbFile
)

func DefaultConfig() Config {
	var cfg Config
	cfg.SetKeyComparator(defaultKeyCmp)
	cfg.SetFileType(RawdbFile)
	return cfg
}

type Item struct {
	bornSn, deadSn uint32
	data           []byte
}

func (itm *Item) Encode(buf []byte, w io.Writer) error {
	l := 2
	if len(buf) < l {
		return ErrNotEnoughSpace
	}

	binary.BigEndian.PutUint16(buf[0:2], uint16(len(itm.data)))
	if _, err := w.Write(buf[0:2]); err != nil {
		return err
	}
	if _, err := w.Write(itm.data); err != nil {
		return err
	}

	return nil
}

func (itm *Item) Decode(buf []byte, r io.Reader) error {
	if _, err := io.ReadFull(r, buf[0:2]); err != nil {
		return err
	}
	l := binary.BigEndian.Uint16(buf[0:2])
	itm.data = make([]byte, int(l))
	_, err := io.ReadFull(r, itm.data)

	return err
}

func (itm *Item) Bytes() []byte {
	return itm.data
}

func NewItem(data []byte) *Item {
	return &Item{
		data: data,
	}
}

func newInsertCompare(keyCmp KeyCompare) skiplist.CompareFn {
	return func(this skiplist.Item, that skiplist.Item) int {
		var v int
		thisItem := this.(*Item)
		thatItem := that.(*Item)
		if v = keyCmp(thisItem.data, thatItem.data); v == 0 {
			v = int(thisItem.bornSn) - int(thatItem.bornSn)
		}

		return v
	}
}

func newIterCompare(keyCmp KeyCompare) skiplist.CompareFn {
	return func(this skiplist.Item, that skiplist.Item) int {
		thisItem := this.(*Item)
		thatItem := that.(*Item)
		return keyCmp(thisItem.data, thatItem.data)
	}
}

func defaultKeyCmp(this []byte, that []byte) int {
	var l int

	l1 := len(this)
	l2 := len(that)
	if l1 < l2 {
		l = l1
	} else {
		l = l2
	}

	return bytes.Compare(this[:l], that[:l])
}

//
//compare item,sn
type Writer struct {
	rand *rand.Rand
	buf  *skiplist.ActionBuffer
	iter *skiplist.Iterator
	*MemDB
}

func (w *Writer) Put(x *Item) {
	x.bornSn = w.getCurrSn()
	w.store.Insert2(x, w.insCmp, w.buf, w.rand.Float32)
	atomic.AddInt64(&w.count, 1)
}

// Find first item, seek until dead=0, mark dead=sn
func (w *Writer) Delete(x *Item) (success bool) {
	defer func() {
		if success {
			atomic.AddInt64(&w.count, -1)
		}
	}()

	gotItem := w.Get(x)
	if gotItem != nil {
		sn := w.getCurrSn()
		if gotItem.bornSn == sn {
			success = w.store.Delete(gotItem, w.insCmp, w.buf)
			return
		}

		success = atomic.CompareAndSwapUint32(&gotItem.deadSn, 0, sn)
		return
	}

	return
}

func (w *Writer) Get(x *Item) *Item {
	var curr *Item
	found := w.iter.Seek(x)
	if !found {
		return nil
	}

	// Seek until most recent item for key is found
	curr = w.iter.Get().(*Item)
	for {
		w.iter.Next()
		if !w.iter.Valid() {
			break
		}
		next := w.iter.Get().(*Item)
		if w.iterCmp(next, curr) != 0 {
			break
		}

		curr = next
	}

	if curr.deadSn != 0 {
		return nil
	}

	return curr
}

type Config struct {
	keyCmp  KeyCompare
	insCmp  skiplist.CompareFn
	iterCmp skiplist.CompareFn

	fileType FileType
}

func (cfg *Config) SetKeyComparator(cmp KeyCompare) {
	cfg.keyCmp = cmp
	cfg.insCmp = newInsertCompare(cmp)
	cfg.iterCmp = newIterCompare(cmp)
}

func (cfg *Config) SetFileType(t FileType) error {
	switch t {
	case ForestdbFile, RawdbFile:
	default:
		return errors.New("Invalid format")
	}

	cfg.fileType = t
	return nil
}

type MemDB struct {
	store        *skiplist.Skiplist
	currSn       uint32
	snapshots    *skiplist.Skiplist
	isGCRunning  int32
	lastGCSn     uint32
	leastUnrefSn uint32
	count        int64

	Config
}

func NewWithConfig(cfg Config) *MemDB {
	m := &MemDB{
		store:     skiplist.New(),
		snapshots: skiplist.New(),
		currSn:    1,
		Config:    cfg,
	}

	return m

}

func New() *MemDB {
	return NewWithConfig(DefaultConfig())
}

func (m *MemDB) Reset() {
	m.store = skiplist.New()
	m.snapshots = skiplist.New()
	m.currSn = 1
}

func (m *MemDB) getCurrSn() uint32 {
	return atomic.LoadUint32(&m.currSn)
}

func (m *MemDB) setLeastUnrefSn() {
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)
	iter := m.snapshots.NewIterator(CompareSnapshot, buf)
	iter.SeekFirst()
	if iter.Valid() {
		snap := iter.Get().(*Snapshot)
		atomic.StoreUint32(&m.leastUnrefSn, snap.sn-1)
	}
}

func (m *MemDB) getLeastUnrefSn() uint32 {
	return atomic.LoadUint32(&m.leastUnrefSn)
}

func (m *MemDB) NewWriter() *Writer {
	buf := m.store.MakeBuf()

	return &Writer{
		rand:  rand.New(rand.NewSource(int64(rand.Int()))),
		buf:   buf,
		iter:  m.store.NewIterator(m.iterCmp, buf),
		MemDB: m,
	}
}

type Snapshot struct {
	sn       uint32
	refCount int32
	db       *MemDB
	count    int64
}

func (s Snapshot) Count() int64 {
	return s.count
}

func (s *Snapshot) Encode(buf []byte, w io.Writer) error {
	l := 4
	if len(buf) < l {
		return ErrNotEnoughSpace
	}

	binary.BigEndian.PutUint32(buf[0:4], s.sn)
	if _, err := w.Write(buf[0:4]); err != nil {
		return err
	}

	return nil

}

func (s *Snapshot) Decode(buf []byte, r io.Reader) error {
	if _, err := io.ReadFull(r, buf[0:4]); err != nil {
		return err
	}
	s.sn = binary.BigEndian.Uint32(buf[0:4])
	return nil
}

func (s *Snapshot) Open() bool {
	if atomic.LoadInt32(&s.refCount) == 0 {
		return false
	}
	atomic.AddInt32(&s.refCount, 1)
	return true
}

func (s *Snapshot) Close() {
	newRefcount := atomic.AddInt32(&s.refCount, -1)
	if newRefcount == 0 {
		buf := s.db.snapshots.MakeBuf()
		defer s.db.snapshots.FreeBuf(buf)
		s.db.snapshots.Delete(s, CompareSnapshot, buf)
		s.db.setLeastUnrefSn()
		if atomic.CompareAndSwapInt32(&s.db.isGCRunning, 0, 1) {
			go s.db.GC()
		}
	}
}

func (s *Snapshot) NewIterator() *Iterator {
	return s.db.NewIterator(s)
}

func CompareSnapshot(this skiplist.Item, that skiplist.Item) int {
	thisItem := this.(*Snapshot)
	thatItem := that.(*Snapshot)

	return int(thisItem.sn) - int(thatItem.sn)
}

func (m *MemDB) NewSnapshot() *Snapshot {
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)

	snap := &Snapshot{db: m, sn: m.getCurrSn(), refCount: 1, count: m.ItemsCount()}
	m.snapshots.Insert(snap, CompareSnapshot, buf)
	atomic.AddUint32(&m.currSn, 1)
	return snap
}

type Iterator struct {
	snap *Snapshot
	iter *skiplist.Iterator
	buf  *skiplist.ActionBuffer
}

func (it *Iterator) skipUnwanted() {
loop:
	if !it.iter.Valid() {
		return
	}
	itm := it.iter.Get().(*Item)
	if itm.bornSn > it.snap.sn || (itm.deadSn > 0 && itm.deadSn <= it.snap.sn) {
		it.iter.Next()
		goto loop
	}
}

func (it *Iterator) SeekFirst() {
	it.iter.SeekFirst()
	it.skipUnwanted()
}

func (it *Iterator) Seek(itm *Item) {
	it.iter.Seek(itm)
	it.skipUnwanted()
}

func (it *Iterator) Valid() bool {
	return it.iter.Valid()
}

func (it *Iterator) Get() *Item {
	return it.iter.Get().(*Item)
}

func (it *Iterator) MarkDeletion() {
	it.iter.MarkDeletion()
}

func (it *Iterator) Next() {
	it.iter.Next()
	it.skipUnwanted()
}

func (it *Iterator) Close() {
	it.snap.Close()
	it.snap.db.store.FreeBuf(it.buf)
}

func (m *MemDB) NewIterator(snap *Snapshot) *Iterator {
	if !snap.Open() {
		return nil
	}
	buf := snap.db.store.MakeBuf()
	return &Iterator{
		snap: snap,
		iter: m.store.NewIterator(m.iterCmp, buf),
		buf:  buf,
	}
}

func (m *MemDB) ItemsCount() int64 {
	return atomic.LoadInt64(&m.count)
}

func (m *MemDB) collectDead(sn uint32) {
	// Mark and sweep GC
	buf1 := m.snapshots.MakeBuf()
	buf2 := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf1)
	defer m.snapshots.FreeBuf(buf2)
	iter := m.store.NewIterator(m.iterCmp, buf1)

	// Mark phase
	for iter.SeekFirst(); iter.Valid(); iter.Next() {
		itm := iter.Get().(*Item)
		if itm.deadSn > 0 && itm.deadSn <= m.getLeastUnrefSn() {
			iter.MarkDeletion()
		}
	}

	// Sweep phase
	for iter.SeekFirst(); iter.Valid(); iter.Next() {
	}
}

func (m *MemDB) GC() {
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)

	iter := m.snapshots.NewIterator(CompareSnapshot, buf)
	iter.SeekFirst()
	if iter.Valid() {
		snap := iter.Get().(*Snapshot)
		if snap.sn != m.lastGCSn && snap.sn > 1 {
			m.lastGCSn = snap.sn - 1
			m.collectDead(m.lastGCSn)
		}
	}

	atomic.CompareAndSwapInt32(&m.isGCRunning, 1, 0)
}

func (m *MemDB) GetSnapshots() []*Snapshot {
	var snaps []*Snapshot
	buf := m.snapshots.MakeBuf()
	defer m.snapshots.FreeBuf(buf)
	iter := m.snapshots.NewIterator(CompareSnapshot, buf)
	iter.SeekFirst()
	for ; iter.Valid(); iter.Next() {
		snaps = append(snaps, iter.Get().(*Snapshot))
	}

	return snaps
}

func (m *MemDB) StoreToDisk(dir string, snap *Snapshot, callb ItemCallback) error {
	os.MkdirAll(dir, 0755)
	datafile := path.Join(dir, "records.data")
	w := newFileWriter(m.fileType)
	if err := w.Open(datafile); err != nil {
		return err
	}
	defer w.Close()

	itr := m.NewIterator(snap)
	if itr == nil {
		return errors.New("Invalid snapshot")
	}
	defer itr.Close()

	for itr.SeekFirst(); itr.Valid(); itr.Next() {
		itm := itr.Get()
		if err := w.WriteItem(itm); err != nil {
			return err
		}

		if callb != nil {
			callb(itm)
		}
	}

	return nil
}

func (m *MemDB) LoadFromDisk(dir string, callb ItemCallback) (*Snapshot, error) {
	var wg sync.WaitGroup
	datafile := path.Join(dir, "records.data")
	r := newFileReader(m.fileType)
	if err := r.Open(datafile); err != nil {
		return nil, err
	}

	defer r.Close()

	ch := make(chan *Item, readerBufSize)
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go func(wg *sync.WaitGroup) {
			defer wg.Done()
			w := m.NewWriter()
			for itm := range ch {
				w.Put(itm)
				if callb != nil {
					callb(itm)
				}
			}
		}(&wg)
	}

loop:
	for {
		itm, err := r.ReadItem()
		if err != nil {
			return nil, err
		}

		if itm == nil {
			break loop
		}
		ch <- itm
	}

	close(ch)
	wg.Wait()

	snap := m.NewSnapshot()
	return snap, nil
}

func (m *MemDB) DumpStats() string {
	return m.store.GetStats().String()
}
<|endoftext|>


<commit_before>package commands

import (
	"cf/api"
	"cf/requirements"
	term "cf/terminal"
	"github.com/codegangsta/cli"
)

type CreateOrganization struct {
	ui      term.UI
	orgRepo api.OrganizationRepository
}

func NewCreateOrganization(ui term.UI, orgRepo api.OrganizationRepository) (cmd CreateOrganization) {
	cmd.ui = ui
	cmd.orgRepo = orgRepo
	return
}

func (cmd CreateOrganization) GetRequirements(reqFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {
	return
}

func (cmd CreateOrganization) Run(c *cli.Context) {
	name := c.String("name")
	cmd.createOrganization(name)
}

func (cmd CreateOrganization) createOrganization(name string) {
	cmd.ui.Say("Creating organization %s", term.Cyan(name))
	_, apiErr := cmd.orgRepo.Create(name)
	if apiErr != nil {
		cmd.ui.Failed("Error creating organization", apiErr)
		return
	}

	cmd.ui.Ok()
	return
}
<commit_msg>Inline helper into Run<commit_after>package commands

import (
	"cf/api"
	"cf/requirements"
	term "cf/terminal"
	"github.com/codegangsta/cli"
)

type CreateOrganization struct {
	ui      term.UI
	orgRepo api.OrganizationRepository
}

func NewCreateOrganization(ui term.UI, orgRepo api.OrganizationRepository) (cmd CreateOrganization) {
	cmd.ui = ui
	cmd.orgRepo = orgRepo
	return
}

func (cmd CreateOrganization) GetRequirements(reqFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {
	return
}

func (cmd CreateOrganization) Run(c *cli.Context) {
	name := c.String("name")
	cmd.ui.Say("Creating organization %s", term.Cyan(name))
	_, apiErr := cmd.orgRepo.Create(name)
	if apiErr != nil {
		cmd.ui.Failed("Error creating organization", apiErr)
		return
	}

	cmd.ui.Ok()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitstore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
)

// This server watches a list of git repos for changes and syncs the meta data of all commits
// to a BigTable backed datastore.

func main() {
	// Define the flags and parse them.
	var (
		btInstanceID    = flag.String("bt_instance", "production", "Big Table instance")
		btTableID       = flag.String("bt_table", "git-repos", "BigTable table ID")
		workDir         = flag.String("workdir", "", "Working directory where repos are cached. Use the same directory between calls to speed up checkout time.")
		httpPort        = flag.String("http_port", ":9091", "The http port where ready-ness endpoints are served.")
		projectID       = flag.String("project", "skia-public", "ID of the GCP project")
		repoURLs        = common.NewMultiStringFlag("repo_url", []string{}, "Repo url")
		runInit         = flag.Bool("init", false, "Initialize the BigTable instance and quit. This should be run with a different different user who has admin rights.")
		refreshInterval = flag.Duration("refresh", 10*time.Minute, "Interval in which to poll git and refresh the GitStore.")
	)
	common.Init()

	// Make sure we have a data directory and it exists or can be created.
	if *workDir == "" {
		sklog.Fatal("No workdir specified.")
	}
	useWorkDir := fileutil.Must(fileutil.EnsureDirExists(*workDir))

	// Make sure we have at least one repo configured.
	if len(*repoURLs) == 0 {
		sklog.Fatalf("At least one repository URL must be configured.")
	}

	// Configure the bigtable instance.
	config := &gitstore.BTConfig{
		ProjectID:  *projectID,
		InstanceID: *btInstanceID,
		TableID:    *btTableID,
	}

	// Initialize bigtable if invoked with --init and quit.
	// This should be invoked with a user that has admin privileges, so that the production user that
	// wants to write to the instance does not need admin privileges.
	if *runInit {
		if err := gitstore.InitBT(config); err != nil {
			sklog.Fatalf("Error initializing BT: %s", err)
		}
		sklog.Infof("BigTable instance %s and table %s in project %s initialized.", *btInstanceID, *btTableID, *projectID)
		return
	}

	// Start all repo watchers in the background.
	ctx := context.Background()
	for _, repoURL := range *repoURLs {
		go func(repoURL string) {
			repoDir, err := git.NormalizeURL(repoURL)
			if err != nil {
				sklog.Fatalf("Error getting normalized URL for %q:  %s", repoURL, err)
			}
			repoDir = strings.Replace(repoDir, "/", "_", -1)
			repoDir = filepath.Join(useWorkDir, repoDir)
			sklog.Infof("Checking out %s into %s", repoURL, repoDir)

			watcher, err := NewRepoWatcher(ctx, config, repoURL, repoDir)
			if err != nil {
				sklog.Fatalf("Error initializing repo watcher: %s", err)
			}
			watcher.Start(ctx, *refreshInterval)
		}(repoURL)
	}

	// Set up the http handler to indicate ready-ness and start serving.
	http.HandleFunc("/healthz", httputils.ReadyHandleFunc)
	log.Fatal(http.ListenAndServe(*httpPort, nil))
}
<commit_msg>[gitsync] Add git authentication to run in K8s<commit_after>package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/gitstore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
)

// This server watches a list of git repos for changes and syncs the meta data of all commits
// to a BigTable backed datastore.

func main() {
	// Define the flags and parse them.
	var (
		btInstanceID    = flag.String("bt_instance", "production", "Big Table instance")
		btTableID       = flag.String("bt_table", "git-repos", "BigTable table ID")
		httpPort        = flag.String("http_port", ":9091", "The http port where ready-ness endpoints are served.")
		local           = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
		projectID       = flag.String("project", "skia-public", "ID of the GCP project")
		repoURLs        = common.NewMultiStringFlag("repo_url", []string{}, "Repo url")
		runInit         = flag.Bool("init", false, "Initialize the BigTable instance and quit. This should be run with a different different user who has admin rights.")
		refreshInterval = flag.Duration("refresh", 10*time.Minute, "Interval in which to poll git and refresh the GitStore.")
		workDir         = flag.String("workdir", "", "Working directory where repos are cached. Use the same directory between calls to speed up checkout time.")
	)
	common.Init()

	// Make sure we have a data directory and it exists or can be created.
	if *workDir == "" {
		sklog.Fatal("No workdir specified.")
	}
	useWorkDir := fileutil.Must(fileutil.EnsureDirExists(*workDir))

	// Make sure we have at least one repo configured.
	if len(*repoURLs) == 0 {
		sklog.Fatalf("At least one repository URL must be configured.")
	}

	// TODO(stephana): Pass the token source explicitly to the BigTable related functions below.

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(false, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Set up Git authentication if a service account email was set.
	if !*local {
		user, err := user.Current()
		if err != nil {
			sklog.Fatal(err)
		}

		// Use the gitcookie created by gitauth package if .gitcookies does not already exist.
		gitcookiesPath := filepath.Join(user.HomeDir, ".gitcookies")
		if _, err := gitauth.New(ts, gitcookiesPath, true, ""); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
		sklog.Infof("Git authentication set up successfully.")
	}

	// Configure the bigtable instance.
	config := &gitstore.BTConfig{
		ProjectID:  *projectID,
		InstanceID: *btInstanceID,
		TableID:    *btTableID,
	}

	// Initialize bigtable if invoked with --init and quit.
	// This should be invoked with a user that has admin privileges, so that the production user that
	// wants to write to the instance does not need admin privileges.
	if *runInit {
		if err := gitstore.InitBT(config); err != nil {
			sklog.Fatalf("Error initializing BT: %s", err)
		}
		sklog.Infof("BigTable instance %s and table %s in project %s initialized.", *btInstanceID, *btTableID, *projectID)
		return
	}

	// Start all repo watchers in the background.
	ctx := context.Background()
	for _, repoURL := range *repoURLs {
		go func(repoURL string) {
			repoDir, err := git.NormalizeURL(repoURL)
			if err != nil {
				sklog.Fatalf("Error getting normalized URL for %q:  %s", repoURL, err)
			}
			repoDir = strings.Replace(repoDir, "/", "_", -1)
			repoDir = filepath.Join(useWorkDir, repoDir)
			sklog.Infof("Checking out %s into %s", repoURL, repoDir)

			watcher, err := NewRepoWatcher(ctx, config, repoURL, repoDir)
			if err != nil {
				sklog.Fatalf("Error initializing repo watcher: %s", err)
			}
			watcher.Start(ctx, *refreshInterval)
		}(repoURL)
	}

	// Set up the http handler to indicate ready-ness and start serving.
	http.HandleFunc("/healthz", httputils.ReadyHandleFunc)
	log.Fatal(http.ListenAndServe(*httpPort, nil))
}
<|endoftext|>


<commit_before>package u

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/kjk/atomicfile"
	"github.com/minio/minio-go"
)

// MinioClient represents s3/spaces etc. client
type MinioClient struct {
	StorageKey    string
	StorageSecret string
	Bucket        string
	Endpoint      string // e.g. "nyc3.digitaloceanspaces.com"
	client        *minio.Client
}

// EnsureCondfigured will panic if client not configured
func (c *MinioClient) EnsureConfigured() {
	PanicIf(c.StorageKey == "")
	PanicIf(c.StorageSecret == "")
	PanicIf(c.Bucket == "")
	PanicIf(c.Endpoint == "")
}

// URLBase returns base of url under which files are accesible
// (if public it's a publicly available file)
func (c *MinioClient) URLBase() string {
	return fmt.Sprintf("https://%s.%s/", c.Bucket, c.Endpoint)
}

// GetClient returns a (cached) minio client
func (c *MinioClient) GetClient() (*minio.Client, error) {
	if c.client != nil {
		return c.client, nil
	}

	var err error
	ssl := true
	c.client, err = minio.New(c.Endpoint, c.StorageKey, c.StorageSecret, ssl)
	return c.client, err
}

// ListRemoveFiles returns a list of files under a given prefix
func (c *MinioClient) ListRemoteFiles(prefix string) ([]*minio.ObjectInfo, error) {
	var res []*minio.ObjectInfo
	client, err := c.GetClient()
	if err != nil {
		return nil, err
	}
	doneCh := make(chan struct{})
	defer close(doneCh)

	files := client.ListObjectsV2(c.Bucket, prefix, true, doneCh)
	for oi := range files {
		oic := oi
		res = append(res, &oic)
	}
	return res, nil
}

// IsMinioNotExistsError returns true if an error indicates that a key
// doesn't exist in storage
func IsMinioNotExistsError(err error) bool {
	if err == nil {
		return false
	}
	return err.Error() == "The specified key does not exist."
}

// SetPublicObjectMetadata sets options that mark object as public
// for doing put operation
func SetPublicObjectMetadata(opts *minio.PutObjectOptions) {
	if opts.UserMetadata == nil {
		opts.UserMetadata = map[string]string{}
	}
	opts.UserMetadata["x-amz-acl"] = "public-read"
}

func (c *MinioClient) UploadReaderPublic(remotePath string, r io.Reader, size int64, contentType string) error {
	return c.UploadReader(remotePath, r, size, true, contentType)
}

func (c *MinioClient) UploadReaderPrivate(remotePath string, r io.Reader, size int64, contentType string) error {
	return c.UploadReader(remotePath, r, size, false, contentType)
}

func (c *MinioClient) UploadData(remotePath string, d []byte, opts minio.PutObjectOptions) error {
	client, err := c.GetClient()
	if err != nil {
		return err
	}
	r := bytes.NewReader(d)
	size := int64(len(d))
	_, err = client.PutObject(c.Bucket, remotePath, r, size, opts)
	return err
}

func (c *MinioClient) UploadReader(remotePath string, r io.Reader, size int64, public bool, contentType string) error {
	PanicIf(remotePath[0] == '/', "name '%s' shouldn't start with '/'", remotePath)

	if contentType == "" {
		contentType = MimeTypeFromFileName(remotePath)
	}
	//timeStart := time.Now()
	//sizeStr := humanize.Bytes(uint64(size))
	//fmt.Printf("Uploading '%s' of size %s and type %s as public.", remotePath, sizeStr, contentType)
	client, err := c.GetClient()
	if err != nil {
		return err
	}
	opts := minio.PutObjectOptions{
		ContentType: contentType,
	}
	if public {
		SetPublicObjectMetadata(&opts)
	}
	opts.ContentType = contentType
	_, err = client.PutObject(c.Bucket, remotePath, r, size, opts)
	if err != nil {
		return err
	}
	//fmt.Printf(" Took %s.\n", time.Since(timeStart))
	return nil
}

func (c *MinioClient) DownloadFileAsData(remotePath string) ([]byte, error) {
	client, err := c.GetClient()
	if err != nil {
		return nil, err
	}
	opts := minio.GetObjectOptions{}
	obj, err := client.GetObject(c.Bucket, remotePath, opts)
	if err != nil {
		return nil, err
	}
	defer obj.Close()
	var buf bytes.Buffer
	_, err = io.Copy(&buf, obj)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (c *MinioClient) DownloadFileAtomically(dstPath string, remotePath string) error {
	client, err := c.GetClient()
	if err != nil {
		return err
	}
	opts := minio.GetObjectOptions{}
	obj, err := client.GetObject(c.Bucket, remotePath, opts)
	if err != nil {
		return err
	}
	defer obj.Close()

	// ensure there's a dir for destination file
	dir := filepath.Dir(dstPath)
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	f, err := atomicfile.New(dstPath)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(f, obj)
	if err != nil {
		return err
	}
	return f.Close()
}

func (c *MinioClient) UploadFilePublic(remotePath string, filePath string) error {
	return c.UploadFile(remotePath, filePath, true)
}

func (c *MinioClient) UploadFilePrivate(remotePath string, filePath string) error {
	return c.UploadFile(remotePath, filePath, false)
}

func (c *MinioClient) UploadFile(remotePath string, filePath string, public bool) error {
	stat, err := os.Stat(filePath)
	if err != nil {
		return err
	}
	size := stat.Size()
	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer func() {
		f.Close()
	}()
	return c.UploadReaderPublic(remotePath, f, size, "")
}

func (c *MinioClient) UploadDataPublic(remotePath string, d []byte) error {
	r := bytes.NewBuffer(d)
	return c.UploadReaderPublic(remotePath, r, int64(len(d)), "")
}

func (c *MinioClient) UploadStringPublic(remotePath string, s string) error {
	r := bytes.NewBufferString(s)
	return c.UploadReaderPublic(remotePath, r, int64(len(s)), "")
}

func (c *MinioClient) UploadStringPrivate(remotePath string, s string) error {
	r := bytes.NewBufferString(s)
	return c.UploadReaderPrivate(remotePath, r, int64(len(s)), "")
}

func (c *MinioClient) StatObject(remotePath string) (minio.ObjectInfo, error) {
	client, err := c.GetClient()
	if err != nil {
		return minio.ObjectInfo{}, err
	}
	var opts minio.StatObjectOptions
	return client.StatObject(c.Bucket, remotePath, opts)
}
<commit_msg>better error for minio<commit_after>package u

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/kjk/atomicfile"
	"github.com/minio/minio-go"
)

// MinioClient represents s3/spaces etc. client
type MinioClient struct {
	StorageKey    string
	StorageSecret string
	Bucket        string
	Endpoint      string // e.g. "nyc3.digitaloceanspaces.com"
	client        *minio.Client
}

// EnsureCondfigured will panic if client not configured
func (c *MinioClient) EnsureConfigured() {
	PanicIf(c.StorageKey == "", "minio storage key not set")
	PanicIf(c.StorageSecret == "", "minio storage secret not set")
	PanicIf(c.Bucket == "", "minio bucket not set")
	PanicIf(c.Endpoint == "", "minio endpoint not set")
}

// URLBase returns base of url under which files are accesible
// (if public it's a publicly available file)
func (c *MinioClient) URLBase() string {
	return fmt.Sprintf("https://%s.%s/", c.Bucket, c.Endpoint)
}

// GetClient returns a (cached) minio client
func (c *MinioClient) GetClient() (*minio.Client, error) {
	if c.client != nil {
		return c.client, nil
	}

	var err error
	ssl := true
	c.client, err = minio.New(c.Endpoint, c.StorageKey, c.StorageSecret, ssl)
	return c.client, err
}

// ListRemoveFiles returns a list of files under a given prefix
func (c *MinioClient) ListRemoteFiles(prefix string) ([]*minio.ObjectInfo, error) {
	var res []*minio.ObjectInfo
	client, err := c.GetClient()
	if err != nil {
		return nil, err
	}
	doneCh := make(chan struct{})
	defer close(doneCh)

	files := client.ListObjectsV2(c.Bucket, prefix, true, doneCh)
	for oi := range files {
		oic := oi
		res = append(res, &oic)
	}
	return res, nil
}

// IsMinioNotExistsError returns true if an error indicates that a key
// doesn't exist in storage
func IsMinioNotExistsError(err error) bool {
	if err == nil {
		return false
	}
	return err.Error() == "The specified key does not exist."
}

// SetPublicObjectMetadata sets options that mark object as public
// for doing put operation
func SetPublicObjectMetadata(opts *minio.PutObjectOptions) {
	if opts.UserMetadata == nil {
		opts.UserMetadata = map[string]string{}
	}
	opts.UserMetadata["x-amz-acl"] = "public-read"
}

func (c *MinioClient) UploadReaderPublic(remotePath string, r io.Reader, size int64, contentType string) error {
	return c.UploadReader(remotePath, r, size, true, contentType)
}

func (c *MinioClient) UploadReaderPrivate(remotePath string, r io.Reader, size int64, contentType string) error {
	return c.UploadReader(remotePath, r, size, false, contentType)
}

func (c *MinioClient) UploadData(remotePath string, d []byte, opts minio.PutObjectOptions) error {
	client, err := c.GetClient()
	if err != nil {
		return err
	}
	r := bytes.NewReader(d)
	size := int64(len(d))
	_, err = client.PutObject(c.Bucket, remotePath, r, size, opts)
	return err
}

func (c *MinioClient) UploadReader(remotePath string, r io.Reader, size int64, public bool, contentType string) error {
	PanicIf(remotePath[0] == '/', "name '%s' shouldn't start with '/'", remotePath)

	if contentType == "" {
		contentType = MimeTypeFromFileName(remotePath)
	}
	//timeStart := time.Now()
	//sizeStr := humanize.Bytes(uint64(size))
	//fmt.Printf("Uploading '%s' of size %s and type %s as public.", remotePath, sizeStr, contentType)
	client, err := c.GetClient()
	if err != nil {
		return err
	}
	opts := minio.PutObjectOptions{
		ContentType: contentType,
	}
	if public {
		SetPublicObjectMetadata(&opts)
	}
	opts.ContentType = contentType
	_, err = client.PutObject(c.Bucket, remotePath, r, size, opts)
	if err != nil {
		return err
	}
	//fmt.Printf(" Took %s.\n", time.Since(timeStart))
	return nil
}

func (c *MinioClient) DownloadFileAsData(remotePath string) ([]byte, error) {
	client, err := c.GetClient()
	if err != nil {
		return nil, err
	}
	opts := minio.GetObjectOptions{}
	obj, err := client.GetObject(c.Bucket, remotePath, opts)
	if err != nil {
		return nil, err
	}
	defer obj.Close()
	var buf bytes.Buffer
	_, err = io.Copy(&buf, obj)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (c *MinioClient) DownloadFileAtomically(dstPath string, remotePath string) error {
	client, err := c.GetClient()
	if err != nil {
		return err
	}
	opts := minio.GetObjectOptions{}
	obj, err := client.GetObject(c.Bucket, remotePath, opts)
	if err != nil {
		return err
	}
	defer obj.Close()

	// ensure there's a dir for destination file
	dir := filepath.Dir(dstPath)
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	f, err := atomicfile.New(dstPath)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(f, obj)
	if err != nil {
		return err
	}
	return f.Close()
}

func (c *MinioClient) UploadFilePublic(remotePath string, filePath string) error {
	return c.UploadFile(remotePath, filePath, true)
}

func (c *MinioClient) UploadFilePrivate(remotePath string, filePath string) error {
	return c.UploadFile(remotePath, filePath, false)
}

func (c *MinioClient) UploadFile(remotePath string, filePath string, public bool) error {
	stat, err := os.Stat(filePath)
	if err != nil {
		return err
	}
	size := stat.Size()
	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer func() {
		f.Close()
	}()
	return c.UploadReaderPublic(remotePath, f, size, "")
}

func (c *MinioClient) UploadDataPublic(remotePath string, d []byte) error {
	r := bytes.NewBuffer(d)
	return c.UploadReaderPublic(remotePath, r, int64(len(d)), "")
}

func (c *MinioClient) UploadStringPublic(remotePath string, s string) error {
	r := bytes.NewBufferString(s)
	return c.UploadReaderPublic(remotePath, r, int64(len(s)), "")
}

func (c *MinioClient) UploadStringPrivate(remotePath string, s string) error {
	r := bytes.NewBufferString(s)
	return c.UploadReaderPrivate(remotePath, r, int64(len(s)), "")
}

func (c *MinioClient) StatObject(remotePath string) (minio.ObjectInfo, error) {
	client, err := c.GetClient()
	if err != nil {
		return minio.ObjectInfo{}, err
	}
	var opts minio.StatObjectOptions
	return client.StatObject(c.Bucket, remotePath, opts)
}
<|endoftext|>


<commit_before>package farm

import (
	"fmt"
	"reflect"
	"runtime"
	"testing"

	"github.com/soundcloud/roshi/common"
	"github.com/soundcloud/roshi/instrumentation"
)

func TestAllRepairs(t *testing.T) {
	// Build farm around mock clusters.
	n := 5
	clusters := newMockClusters(n)
	farm := New(clusters, (n/2)+1, SendAllReadAll, NoRepairs, nil)

	// Make inserts, no repair.
	first := common.KeyScoreMember{Key: "foo", Score: 1., Member: "bar"}
	second := common.KeyScoreMember{Key: "foo", Score: 2.34, Member: "bar"}
	farm.Insert([]common.KeyScoreMember{first})         // perfect insert
	clusters[0].Insert([]common.KeyScoreMember{second}) // imperfect insert

	// Pre-repair, we should have divergent view.
	for i := 0; i < n; i++ {
		//t.Logf("pre-repair: cluster %d: %+v", i, clusters[i].(*mockCluster).m)
		expected := first
		if i == 0 {
			expected = second
		}
		if got := <-clusters[i].SelectOffset([]string{"foo"}, 0, 10); !reflect.DeepEqual(expected, got.KeyScoreMembers[0]) {
			t.Errorf("pre-repair: cluster %d: expected %+v, got %+v", i, expected, got.KeyScoreMembers[0])
		}
	}

	// Issue repair.
	AllRepairs(clusters, instrumentation.NopInstrumentation{})([]common.KeyMember{common.KeyMember{Key: "foo", Member: "bar"}})

	// Post-repair, we should have perfect agreement on the correct value.
	expected := second
	for i := 0; i < n; i++ {
		//t.Logf("post-repair: cluster %d: %+v", i, clusters[i].(*mockCluster).m)
		if got := <-clusters[i].SelectOffset([]string{"foo"}, 0, 10); !reflect.DeepEqual(expected, got.KeyScoreMembers[0]) {
			t.Errorf("post-repair: cluster %d: expected %+v, got %+v", i, expected, got.KeyScoreMembers[0])
		}
	}
}

func TestRateLimitedRepairs(t *testing.T) {
	// Build farm around mock clusters.
	n := 5
	clusters := newMockClusters(n)
	farm := New(clusters, (n/2)+1, SendAllReadAll, NoRepairs, nil)

	// Make inserts, no repair.
	a := common.KeyScoreMember{Key: "foo", Score: 1.1, Member: "alpha"}
	b := common.KeyScoreMember{Key: "foo", Score: 1.2, Member: "beta"}
	c := common.KeyScoreMember{Key: "foo", Score: 1.3, Member: "delta"}
	d := common.KeyScoreMember{Key: "foo", Score: 2.1, Member: "alpha"}
	e := common.KeyScoreMember{Key: "foo", Score: 2.2, Member: "beta"}
	f := common.KeyScoreMember{Key: "foo", Score: 2.3, Member: "delta"}
	farm.Insert([]common.KeyScoreMember{a})
	farm.Insert([]common.KeyScoreMember{b})
	farm.Insert([]common.KeyScoreMember{c})
	clusters[0].Insert([]common.KeyScoreMember{d})
	clusters[0].Insert([]common.KeyScoreMember{e})
	clusters[0].Insert([]common.KeyScoreMember{f})

	// Perform all repairs as fast as possible.
	maxRepairsPerSecond := 2
	repairFunc := RateLimited(maxRepairsPerSecond, AllRepairs)(clusters, instrumentation.NopInstrumentation{})
	repairFunc([]common.KeyMember{common.KeyMember{Key: "foo", Member: "alpha"}}) // should succeed
	repairFunc([]common.KeyMember{common.KeyMember{Key: "foo", Member: "beta"}})  // should succeed
	repairFunc([]common.KeyMember{common.KeyMember{Key: "foo", Member: "delta"}}) // should fail

	// Make post-repair checks. We only care about clusters 1 and above.
	expected := []common.KeyScoreMember{e, d, c}
	for i := 0; i < n; i++ {
		got := <-clusters[i].SelectOffset([]string{"foo"}, 0, 10)
		t.Logf("post-repair: cluster %d: has %+v", i, got.KeyScoreMembers)
		if i == 0 {
			continue // assume clusters[0] has everything correctly
		}
		if !reflect.DeepEqual(expected, got.KeyScoreMembers) {
			t.Errorf("post-repair: cluster %d: expected %+v, got %+v", i, expected, got.KeyScoreMembers)
		}
	}
}

func TestExplodingGoroutines(t *testing.T) {
	// Make a farm.
	n := 5
	clusters := newMockClusters(n)
	farm := New(clusters, (n/2)+1, SendAllReadAll, AllRepairs, nil)

	// Insert a big key into every cluster except the first.
	key := "foo"
	maxSize := 2000
	tuples := makeBigKey(key, maxSize)
	for i := 1; i < n; i++ {
		clusters[i].Insert(tuples)
	}

	// Issue repair by making a Select.
	before := runtime.NumGoroutine()
	farm.SelectOffset([]string{key}, 0, maxSize)
	runtime.Gosched()
	after := runtime.NumGoroutine()

	if delta := after - before; delta > 5 {
		t.Errorf("The repair-enabled Select created %d goroutines. That's too many.", delta)
	}
}

func makeBigKey(key string, n int) []common.KeyScoreMember {
	tuples := make([]common.KeyScoreMember, n)
	for i := 0; i < n; i++ {
		tuples[i] = common.KeyScoreMember{
			Key:    key,
			Score:  float64(i),
			Member: fmt.Sprint(i),
		}
	}
	return tuples
}
<commit_msg>tip needs this, I don't know why<commit_after>package farm

import (
	"fmt"
	"reflect"
	"runtime"
	"testing"

	"github.com/soundcloud/roshi/common"
	"github.com/soundcloud/roshi/instrumentation"
)

func TestAllRepairs(t *testing.T) {
	// Build farm around mock clusters.
	n := 5
	clusters := newMockClusters(n)
	farm := New(clusters, (n/2)+1, SendAllReadAll, NoRepairs, nil)

	// Make inserts, no repair.
	first := common.KeyScoreMember{Key: "foo", Score: 1., Member: "bar"}
	second := common.KeyScoreMember{Key: "foo", Score: 2.34, Member: "bar"}
	farm.Insert([]common.KeyScoreMember{first})         // perfect insert
	clusters[0].Insert([]common.KeyScoreMember{second}) // imperfect insert

	// Pre-repair, we should have divergent view.
	for i := 0; i < n; i++ {
		//t.Logf("pre-repair: cluster %d: %+v", i, clusters[i].(*mockCluster).m)
		expected := first
		if i == 0 {
			expected = second
		}
		got := <-clusters[i].SelectOffset([]string{"foo"}, 0, 10)
		if len(got.KeyScoreMembers) <= 0 {
			t.Errorf("pre-repair: cluster %d: only got %d responses", i, len(got.KeyScoreMembers))
			continue
		}
		if !reflect.DeepEqual(expected, got.KeyScoreMembers[0]) {
			t.Errorf("pre-repair: cluster %d: expected %+v, got %+v", i, expected, got.KeyScoreMembers[0])
			continue
		}
	}

	// Issue repair.
	AllRepairs(clusters, instrumentation.NopInstrumentation{})([]common.KeyMember{common.KeyMember{Key: "foo", Member: "bar"}})

	// Post-repair, we should have perfect agreement on the correct value.
	expected := second
	for i := 0; i < n; i++ {
		//t.Logf("post-repair: cluster %d: %+v", i, clusters[i].(*mockCluster).m)
		if got := <-clusters[i].SelectOffset([]string{"foo"}, 0, 10); !reflect.DeepEqual(expected, got.KeyScoreMembers[0]) {
			t.Errorf("post-repair: cluster %d: expected %+v, got %+v", i, expected, got.KeyScoreMembers[0])
		}
	}
}

func TestRateLimitedRepairs(t *testing.T) {
	// Build farm around mock clusters.
	n := 5
	clusters := newMockClusters(n)
	farm := New(clusters, (n/2)+1, SendAllReadAll, NoRepairs, nil)

	// Make inserts, no repair.
	a := common.KeyScoreMember{Key: "foo", Score: 1.1, Member: "alpha"}
	b := common.KeyScoreMember{Key: "foo", Score: 1.2, Member: "beta"}
	c := common.KeyScoreMember{Key: "foo", Score: 1.3, Member: "delta"}
	d := common.KeyScoreMember{Key: "foo", Score: 2.1, Member: "alpha"}
	e := common.KeyScoreMember{Key: "foo", Score: 2.2, Member: "beta"}
	f := common.KeyScoreMember{Key: "foo", Score: 2.3, Member: "delta"}
	farm.Insert([]common.KeyScoreMember{a})
	farm.Insert([]common.KeyScoreMember{b})
	farm.Insert([]common.KeyScoreMember{c})
	clusters[0].Insert([]common.KeyScoreMember{d})
	clusters[0].Insert([]common.KeyScoreMember{e})
	clusters[0].Insert([]common.KeyScoreMember{f})

	// Perform all repairs as fast as possible.
	maxRepairsPerSecond := 2
	repairFunc := RateLimited(maxRepairsPerSecond, AllRepairs)(clusters, instrumentation.NopInstrumentation{})
	repairFunc([]common.KeyMember{common.KeyMember{Key: "foo", Member: "alpha"}}) // should succeed
	repairFunc([]common.KeyMember{common.KeyMember{Key: "foo", Member: "beta"}})  // should succeed
	repairFunc([]common.KeyMember{common.KeyMember{Key: "foo", Member: "delta"}}) // should fail

	// Make post-repair checks. We only care about clusters 1 and above.
	expected := []common.KeyScoreMember{e, d, c}
	for i := 0; i < n; i++ {
		got := <-clusters[i].SelectOffset([]string{"foo"}, 0, 10)
		t.Logf("post-repair: cluster %d: has %+v", i, got.KeyScoreMembers)
		if i == 0 {
			continue // assume clusters[0] has everything correctly
		}
		if !reflect.DeepEqual(expected, got.KeyScoreMembers) {
			t.Errorf("post-repair: cluster %d: expected %+v, got %+v", i, expected, got.KeyScoreMembers)
		}
	}
}

func TestExplodingGoroutines(t *testing.T) {
	// Make a farm.
	n := 5
	clusters := newMockClusters(n)
	farm := New(clusters, (n/2)+1, SendAllReadAll, AllRepairs, nil)

	// Insert a big key into every cluster except the first.
	key := "foo"
	maxSize := 2000
	tuples := makeBigKey(key, maxSize)
	for i := 1; i < n; i++ {
		clusters[i].Insert(tuples)
	}

	// Issue repair by making a Select.
	before := runtime.NumGoroutine()
	farm.SelectOffset([]string{key}, 0, maxSize)
	runtime.Gosched()
	after := runtime.NumGoroutine()

	if delta := after - before; delta > 5 {
		t.Errorf("The repair-enabled Select created %d goroutines. That's too many.", delta)
	}
}

func makeBigKey(key string, n int) []common.KeyScoreMember {
	tuples := make([]common.KeyScoreMember, n)
	for i := 0; i < n; i++ {
		tuples[i] = common.KeyScoreMember{
			Key:    key,
			Score:  float64(i),
			Member: fmt.Sprint(i),
		}
	}
	return tuples
}
<|endoftext|>