<commit_before>package azure

import (
	"io"
	"strings"
	"time"

	az "github.com/Azure/azure-sdk-for-go/storage"
	"github.com/graymeta/stow"
	"github.com/pkg/errors"
)

// The maximum size of an object that can be Put in a single request
const maxPutSize = 256 * 1024 * 1024

// timeFormat is the time format for azure.
var timeFormat = "Mon, 2 Jan 2006 15:04:05 MST"

type container struct {
	id         string
	properties az.ContainerProperties
	client     *az.BlobStorageClient
}

var _ stow.Container = (*container)(nil)

func (c *container) ID() string {
	return c.id
}

func (c *container) Name() string {
	return c.id
}

func (c *container) Item(id string) (stow.Item, error) {
	id = strings.Replace(id, " ", "+", -1)

	blob := c.client.GetContainerReference(c.id).GetBlobReference(id)
	err := blob.GetProperties(nil)
	if err != nil {
		if strings.Contains(err.Error(), "404") {
			return nil, stow.ErrNotFound
		}
		return nil, err
	}
	item := &item{
		id:         id,
		container:  c,
		client:     c.client,
		properties: blob.Properties,
	}

	etag := cleanEtag(item.properties.Etag) // Etags returned from this method include quotes. Strip them.
	item.properties.Etag = etag             // Assign the corrected string value to the field.

	return item, nil
}

func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) {
	params := az.ListBlobsParameters{
		Prefix:     prefix,
		MaxResults: uint(count),
	}
	if cursor != "" {
		params.Marker = cursor
	}
	listblobs, err := c.client.GetContainerReference(c.id).ListBlobs(params)
	if err != nil {
		return nil, "", err
	}
	items := make([]stow.Item, len(listblobs.Blobs))
	for i, blob := range listblobs.Blobs {

		// Clean Etag just in case.
		blob.Properties.Etag = cleanEtag(blob.Properties.Etag)

		items[i] = &item{
			id:         blob.Name,
			container:  c,
			client:     c.client,
			properties: blob.Properties,
		}
	}
	return items, listblobs.NextMarker, nil
}

func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) {
	mdParsed, err := prepMetadata(metadata)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create or update Item, preparing metadata")
	}

	name = strings.Replace(name, " ", "+", -1)

	if size > maxPutSize {
		// Do a multipart upload
		err := c.multipartUpload(name, r, size)
		if err != nil {
			return nil, errors.Wrap(err, "multipart upload")
		}
	} else {
		err = c.client.GetContainerReference(c.id).GetBlobReference(name).CreateBlockBlobFromReader(r, nil)
		if err != nil {
			return nil, errors.Wrap(err, "unable to create or update Item")
		}
	}

	err = c.SetItemMetadata(name, mdParsed)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create or update item, setting Item metadata")
	}

	item := &item{
		id:        name,
		container: c,
		client:    c.client,
		properties: az.BlobProperties{
			LastModified:  az.TimeRFC1123(time.Now()),
			Etag:          "",
			ContentLength: size,
		},
	}
	return item, nil
}

func (c *container) SetItemMetadata(itemName string, md map[string]string) error {
	blob := c.client.GetContainerReference(c.id).GetBlobReference(itemName)
	blob.Metadata = md
	return blob.SetMetadata(nil)
}

func parseMetadata(md map[string]string) (map[string]interface{}, error) {
	rtnMap := make(map[string]interface{}, len(md))
	for key, value := range md {
		rtnMap[key] = value
	}
	return rtnMap, nil
}

func prepMetadata(md map[string]interface{}) (map[string]string, error) {
	rtnMap := make(map[string]string, len(md))
	for key, value := range md {
		str, ok := value.(string)
		if !ok {
			return nil, errors.Errorf(`value of key '%s' in metadata must be of type string`, key)
		}
		rtnMap[key] = str
	}
	return rtnMap, nil
}

func (c *container) RemoveItem(id string) error {
	return c.client.GetContainerReference(c.id).GetBlobReference(id).Delete(nil)
}

// Remove quotation marks from beginning and end. This includes quotations that
// are escaped. Also removes leading `W/` from prefix for weak Etags.
//
// Based on the Etag spec, the full etag value (<FULL ETAG VALUE>) can include:
// - W/"<ETAG VALUE>"
// - "<ETAG VALUE>"
// - ""
// Source: https://tools.ietf.org/html/rfc7232#section-2.3
//
// Based on HTTP spec, forward slash is a separator and must be enclosed in
// quotes to be used as a valid value. Hence, the returned value may include:
// - "<FULL ETAG VALUE>"
// - \"<FULL ETAG VALUE>\"
// Source: https://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2
//
// This function contains a loop to check for the presence of the three possible
// filler characters and strips them, resulting in only the Etag value.
func cleanEtag(etag string) string {
	for {
		// Check if the filler characters are present
		if strings.HasPrefix(etag, `\"`) {
			etag = strings.Trim(etag, `\"`)

		} else if strings.HasPrefix(etag, `"`) {
			etag = strings.Trim(etag, `"`)

		} else if strings.HasPrefix(etag, `W/`) {
			etag = strings.Replace(etag, `W/`, "", 1)

		} else {

			break
		}
	}

	return etag
}
<commit_msg>Remove Replace space in Azure item (#193)<commit_after>package azure

import (
	"io"
	"strings"
	"time"

	az "github.com/Azure/azure-sdk-for-go/storage"
	"github.com/graymeta/stow"
	"github.com/pkg/errors"
)

// The maximum size of an object that can be Put in a single request
const maxPutSize = 256 * 1024 * 1024

// timeFormat is the time format for azure.
var timeFormat = "Mon, 2 Jan 2006 15:04:05 MST"

type container struct {
	id         string
	properties az.ContainerProperties
	client     *az.BlobStorageClient
}

var _ stow.Container = (*container)(nil)

func (c *container) ID() string {
	return c.id
}

func (c *container) Name() string {
	return c.id
}

func (c *container) Item(id string) (stow.Item, error) {
	blob := c.client.GetContainerReference(c.id).GetBlobReference(id)
	err := blob.GetProperties(nil)
	if err != nil {
		if strings.Contains(err.Error(), "404") {
			return nil, stow.ErrNotFound
		}
		return nil, err
	}
	item := &item{
		id:         id,
		container:  c,
		client:     c.client,
		properties: blob.Properties,
	}

	etag := cleanEtag(item.properties.Etag) // Etags returned from this method include quotes. Strip them.
	item.properties.Etag = etag             // Assign the corrected string value to the field.

	return item, nil
}

func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) {
	params := az.ListBlobsParameters{
		Prefix:     prefix,
		MaxResults: uint(count),
	}
	if cursor != "" {
		params.Marker = cursor
	}
	listblobs, err := c.client.GetContainerReference(c.id).ListBlobs(params)
	if err != nil {
		return nil, "", err
	}
	items := make([]stow.Item, len(listblobs.Blobs))
	for i, blob := range listblobs.Blobs {

		// Clean Etag just in case.
		blob.Properties.Etag = cleanEtag(blob.Properties.Etag)

		items[i] = &item{
			id:         blob.Name,
			container:  c,
			client:     c.client,
			properties: blob.Properties,
		}
	}
	return items, listblobs.NextMarker, nil
}

func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) {
	mdParsed, err := prepMetadata(metadata)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create or update Item, preparing metadata")
	}

	name = strings.Replace(name, " ", "+", -1)

	if size > maxPutSize {
		// Do a multipart upload
		err := c.multipartUpload(name, r, size)
		if err != nil {
			return nil, errors.Wrap(err, "multipart upload")
		}
	} else {
		err = c.client.GetContainerReference(c.id).GetBlobReference(name).CreateBlockBlobFromReader(r, nil)
		if err != nil {
			return nil, errors.Wrap(err, "unable to create or update Item")
		}
	}

	err = c.SetItemMetadata(name, mdParsed)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create or update item, setting Item metadata")
	}

	item := &item{
		id:        name,
		container: c,
		client:    c.client,
		properties: az.BlobProperties{
			LastModified:  az.TimeRFC1123(time.Now()),
			Etag:          "",
			ContentLength: size,
		},
	}
	return item, nil
}

func (c *container) SetItemMetadata(itemName string, md map[string]string) error {
	blob := c.client.GetContainerReference(c.id).GetBlobReference(itemName)
	blob.Metadata = md
	return blob.SetMetadata(nil)
}

func parseMetadata(md map[string]string) (map[string]interface{}, error) {
	rtnMap := make(map[string]interface{}, len(md))
	for key, value := range md {
		rtnMap[key] = value
	}
	return rtnMap, nil
}

func prepMetadata(md map[string]interface{}) (map[string]string, error) {
	rtnMap := make(map[string]string, len(md))
	for key, value := range md {
		str, ok := value.(string)
		if !ok {
			return nil, errors.Errorf(`value of key '%s' in metadata must be of type string`, key)
		}
		rtnMap[key] = str
	}
	return rtnMap, nil
}

func (c *container) RemoveItem(id string) error {
	return c.client.GetContainerReference(c.id).GetBlobReference(id).Delete(nil)
}

// Remove quotation marks from beginning and end. This includes quotations that
// are escaped. Also removes leading `W/` from prefix for weak Etags.
//
// Based on the Etag spec, the full etag value (<FULL ETAG VALUE>) can include:
// - W/"<ETAG VALUE>"
// - "<ETAG VALUE>"
// - ""
// Source: https://tools.ietf.org/html/rfc7232#section-2.3
//
// Based on HTTP spec, forward slash is a separator and must be enclosed in
// quotes to be used as a valid value. Hence, the returned value may include:
// - "<FULL ETAG VALUE>"
// - \"<FULL ETAG VALUE>\"
// Source: https://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2
//
// This function contains a loop to check for the presence of the three possible
// filler characters and strips them, resulting in only the Etag value.
func cleanEtag(etag string) string {
	for {
		// Check if the filler characters are present
		if strings.HasPrefix(etag, `\"`) {
			etag = strings.Trim(etag, `\"`)

		} else if strings.HasPrefix(etag, `"`) {
			etag = strings.Trim(etag, `"`)

		} else if strings.HasPrefix(etag, `W/`) {
			etag = strings.Replace(etag, `W/`, "", 1)

		} else {

			break
		}
	}

	return etag
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/gorilla/mux"
	"net/http"
	"github.com/Zenika/MARCEL/backend/medias"
	"github.com/rs/cors"
	"log"
	"os"
	//"github.com/Zenika/MARCEL/backend/plugins"
	"github.com/Zenika/MARCEL/backend/apidoc"
)

//current version of the API
const MARCEL_API_VERSION = "1"
var logFile string = os.Getenv("MARCEL_LOG_FILE")

type App struct {
	Router http.Handler
}

func (a *App) Initialize() {

	a.initializeLog()

	a.initializeData()

	a.initializeRoutes()
}

func (a *App) Run(addr string) {
	log.Fatal(http.ListenAndServe(addr, a.Router))
	log.Printf("Server is started and listening on port %v", addr)
}

func (a *App) initializeRoutes() {

	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		// AllowedOrigins:   []string{"http://localhost:*"},
		AllowedMethods:   []string{"GET", "POST", "DELETE", "OPTION", "PUT"},
		AllowCredentials: true,
	})

	r := mux.NewRouter()
	s := r.PathPrefix("/api/v" + MARCEL_API_VERSION).Subrouter()
	s.HandleFunc("/medias", medias.GetAllHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia}", medias.GetHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia}", medias.PostHandler).Methods("POST")
	s.HandleFunc("/medias/create", medias.CreateHandler).Methods("GET")
	r.HandleFunc("/swagger.json", apidoc.GetConfigHandler).Methods("GET")

	a.Router = c.Handler(r)
}

func (a* App) initializeLog() {
	if len(logFile) == 0 {
		logFile = "marcel.log"
	}
	f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	log.SetOutput(f)
}

func (a* App) initializeData() {

	//Load plugins list from DB
	//plugins.LoadPluginsCatalog()

	//Load Medias configuration from DB
	medias.LoadMedias()
}

<commit_msg>[Improve] Defer close of log file to the end of application<commit_after>package app

import (
	"github.com/gorilla/mux"
	"net/http"
	"github.com/Zenika/MARCEL/backend/medias"
	"github.com/rs/cors"
	"log"
	"os"
	//"github.com/Zenika/MARCEL/backend/plugins"
	"github.com/Zenika/MARCEL/backend/apidoc"
)

//current version of the API
const MARCEL_API_VERSION = "1"
var logFileName string = os.Getenv("MARCEL_LOG_FILE")
var logFile *os.File

type App struct {
	Router http.Handler
}

func (a *App) Initialize() {

	a.initializeLog()

	a.initializeData()

	a.initializeRoutes()
}

func (a *App) Run(addr string) {
	log.Fatal(http.ListenAndServe(addr, a.Router))
	log.Printf("Server is started and listening on port %v", addr)

	defer logFile.Close()

	select {}
}

func (a *App) initializeRoutes() {

	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		// AllowedOrigins:   []string{"http://localhost:*"},
		AllowedMethods:   []string{"GET", "POST", "DELETE", "OPTION", "PUT"},
		AllowCredentials: true,
	})

	r := mux.NewRouter()
	s := r.PathPrefix("/api/v" + MARCEL_API_VERSION).Subrouter()
	s.HandleFunc("/medias", medias.GetAllHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia}", medias.GetHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia}", medias.PostHandler).Methods("POST")
	s.HandleFunc("/medias/create", medias.CreateHandler).Methods("GET")
	r.HandleFunc("/swagger.json", apidoc.GetConfigHandler).Methods("GET")

	a.Router = c.Handler(r)
}

func (a* App) initializeLog() {
	if len(logFileName) == 0 {
		logFileName = "marcel.log"
	}
	var err error = nil
	logFile, err = os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	log.SetOutput(logFile)
}

func (a* App) initializeData() {

	//Load plugins list from DB
	//plugins.LoadPluginsCatalog()

	//Load Medias configuration from DB
	medias.LoadMedias()
}

<|endoftext|>


<commit_before>package model

import (
	"database/sql"
	"github.com/dinever/dingo/app/utils"
	"github.com/twinj/uuid"
	"golang.org/x/crypto/bcrypt"
	"time"
)

type User struct {
	Id       int64
	Name     string
	Slug     string
	Avatar   string
	Email    string
	Image    string // NULL
	Cover    string // NULL
	Bio      string // NULL
	Website  string // NULL
	Location string // NULL
	Role     int    //1 = Administrator, 2 = Editor, 3 = Author, 4 = Owner
}

var ghostUser = &User{Id: 0, Name: "Dingo User", Email: "example@example.com"}

func NewUser(email, name string) *User {
	user := new(User)
	user.Email = email
	user.Name = name
	return user
}

func (u *User) Create(password string) error {
	hashedPassword, err := EncryptPassword(password)
	if err != nil {
		return err
	}
	return u.Save(hashedPassword, 0)
}

func (u *User) Save(hashedPassword string, createdBy int64) error {
	id, err := InsertUser(u.Name, u.Slug, hashedPassword, u.Email, u.Image, u.Cover, time.Now(), createdBy)
	if err != nil {
		return err
	}
	u.Id = id
	//	err = InsertRoleUser(u.Role, userId)
	//	if err != nil {
	//		return err
	//	}
	return nil
}

func (u *User) Update() error {
	writeDB, err := db.Begin()
	if err != nil {
		writeDB.Rollback()
		return err
	}
	_, err = writeDB.Exec(stmtUpdateUser, u.Name, u.Slug, u.Email, u.Image, u.Cover, u.Bio, u.Website, u.Location, time.Now(), u.Id, u.Id)
	if err != nil {
		writeDB.Rollback()
		return err
	}
	return writeDB.Commit()
}

func (u *User) ChangePassword(password string) error {
	hashedPassword, err := EncryptPassword(password)
	if err != nil {
		return err
	}
	WriteDB, err := db.Begin()
	if err != nil {
		WriteDB.Rollback()
		return err
	}
	_, err = db.Exec("UPDATE users SET password = ? WHERE id = ?", hashedPassword, u.Id)
	if err != nil {
		WriteDB.Rollback()
		return err
	}
	return WriteDB.Commit()
}

func EncryptPassword(password string) (string, error) {
	hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), 10)
	if err != nil {
		return "", err
	}
	return string(hashedPassword), nil
}

func (u *User) CheckPassword(password string) bool {
	hashedPassword, err := GetHashedPasswordForUser(string(u.Email))
	if err != nil {
		return false
	}
	err = bcrypt.CompareHashAndPassword(hashedPassword, []byte(password))
	if err != nil {
		return false
	}
	return true
}

func scanUser(user *User, row *sql.Row) error {
	var (
		nullImage    sql.NullString
		nullCover    sql.NullString
		nullBio      sql.NullString
		nullWebsite  sql.NullString
		nullLocation sql.NullString
	)
	err := row.Scan(&user.Id, &user.Name, &user.Slug, &user.Email, &nullImage, &nullCover, &nullBio, &nullWebsite, &nullLocation)
	user.Avatar = utils.Gravatar(user.Email, "150")
	user.Image = nullImage.String
	user.Cover = nullCover.String
	user.Bio = nullBio.String
	user.Website = nullWebsite.String
	user.Location = nullLocation.String
	return err
}

func GetHashedPasswordForUser(email string) ([]byte, error) {
	var hashedPassword []byte
	row := db.QueryRow(stmtGetHashedPasswordByEmail, email)
	err := row.Scan(&hashedPassword)
	if err != nil {
		return []byte{}, err
	}
	return hashedPassword, nil
}

func GetUserById(id int64) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserById, id)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func GetUserBySlug(slug string) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserBySlug, slug)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func GetUserByName(name string) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserByName, name)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func GetUserByEmail(email string) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserByEmail, email)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func InsertUser(name string, slug string, password string, email string, image string, cover string, created_at time.Time, created_by int64) (int64, error) {
	writeDB, err := db.Begin()
	if err != nil {
		writeDB.Rollback()
		return 0, err
	}
	result, err := writeDB.Exec(stmtInsertUser, nil, uuid.Formatter(uuid.NewV4(), uuid.CleanHyphen), name, slug, password, email, image, cover, created_at, created_by, created_at, created_by)
	if err != nil {
		writeDB.Rollback()
		return 0, err
	}
	userId, err := result.LastInsertId()
	if err != nil {
		writeDB.Rollback()
		return 0, err
	}
	return userId, writeDB.Commit()
}

func InsertRoleUser(role_id int, user_id int64) error {
	writeDB, err := db.Begin()
	if err != nil {
		writeDB.Rollback()
		return err
	}
	_, err = writeDB.Exec(stmtInsertRoleUser, nil, role_id, user_id)
	if err != nil {
		writeDB.Rollback()
		return err
	}
	return writeDB.Commit()
}

func UserEmailExist(email string) bool {
	var count int64
	row := db.QueryRow(stmtGetUsersCountByEmail, email)
	err := row.Scan(&count)
	if count > 0 || err != nil {
		return true
	}
	return false
}

func GetNumberOfUsers() (int64, error) {
	var count int64
	row := db.QueryRow("SELECT COUNT(*) FROM users")
	err := row.Scan(&count)
	return count, err
}
<commit_msg>[refactor] refactoring User struct and Insert method<commit_after>package model

import (
	"database/sql"
	"github.com/dinever/dingo/app/utils"
	"github.com/twinj/uuid"
	"golang.org/x/crypto/bcrypt"
	"time"
)

type User struct {
	Id        int64
	Name      string
	Slug      string
	Avatar    string
	Email     string
	Image     string // NULL
	Cover     string // NULL
	Bio       string // NULL
	Website   string // NULL
	Location  string // NULL
	Role      int    //1 = Administrator, 2 = Editor, 3 = Author, 4 = Owner
	CreatedAt *time.Time
	UpdatedAt *time.Time
}

var ghostUser = &User{Id: 0, Name: "Dingo User", Email: "example@example.com"}

func NewUser(email, name string) *User {
	return &User{
		Email:     email,
		Name:      name,
		CreatedAt: utils.Now(),
		UpdatedAt: utils.Now(),
	}
}

func (u *User) Create(password string) error {
	hashedPassword, err := EncryptPassword(password)
	if err != nil {
		return err
	}
	return u.Save(hashedPassword, 0)
}

func (u *User) Save(hashedPassword string, createdBy int64) error {
	err := u.Insert(hashedPassword, createdBy)
	if err != nil {
		return err
	}
	//	err = InsertRoleUser(u.Role, userId)
	//	if err != nil {
	//		return err
	//	}
	return nil
}

func (u *User) Update() error {
	writeDB, err := db.Begin()
	if err != nil {
		writeDB.Rollback()
		return err
	}
	u.UpdatedAt = utils.Now()
	_, err = writeDB.Exec(stmtUpdateUser, u.Name, u.Slug, u.Email, u.Image, u.Cover, u.Bio, u.Website, u.Location, u.UpdatedAt, u.Id, u.Id)
	if err != nil {
		writeDB.Rollback()
		return err
	}
	return writeDB.Commit()
}

func (u *User) ChangePassword(password string) error {
	hashedPassword, err := EncryptPassword(password)
	if err != nil {
		return err
	}
	WriteDB, err := db.Begin()
	if err != nil {
		WriteDB.Rollback()
		return err
	}
	_, err = db.Exec("UPDATE users SET password = ? WHERE id = ?", hashedPassword, u.Id)
	if err != nil {
		WriteDB.Rollback()
		return err
	}
	return WriteDB.Commit()
}

func EncryptPassword(password string) (string, error) {
	hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), 10)
	if err != nil {
		return "", err
	}
	return string(hashedPassword), nil
}

func (u *User) CheckPassword(password string) bool {
	hashedPassword, err := GetHashedPasswordForUser(string(u.Email))
	if err != nil {
		return false
	}
	err = bcrypt.CompareHashAndPassword(hashedPassword, []byte(password))
	if err != nil {
		return false
	}
	return true
}

func scanUser(user *User, row *sql.Row) error {
	var (
		nullImage    sql.NullString
		nullCover    sql.NullString
		nullBio      sql.NullString
		nullWebsite  sql.NullString
		nullLocation sql.NullString
	)
	err := row.Scan(&user.Id, &user.Name, &user.Slug, &user.Email, &nullImage, &nullCover, &nullBio, &nullWebsite, &nullLocation)
	user.Avatar = utils.Gravatar(user.Email, "150")
	user.Image = nullImage.String
	user.Cover = nullCover.String
	user.Bio = nullBio.String
	user.Website = nullWebsite.String
	user.Location = nullLocation.String
	return err
}

func GetHashedPasswordForUser(email string) ([]byte, error) {
	var hashedPassword []byte
	row := db.QueryRow(stmtGetHashedPasswordByEmail, email)
	err := row.Scan(&hashedPassword)
	if err != nil {
		return []byte{}, err
	}
	return hashedPassword, nil
}

func GetUserById(id int64) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserById, id)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func GetUserBySlug(slug string) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserBySlug, slug)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func GetUserByName(name string) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserByName, name)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func GetUserByEmail(email string) (*User, error) {
	user := new(User)
	// Get user
	row := db.QueryRow(stmtGetUserByEmail, email)
	err := scanUser(user, row)
	if err != nil {
		return nil, err
	}
	return user, nil
}

func (u *User) Insert(password string, created_by int64) error {
	writeDB, err := db.Begin()
	if err != nil {
		writeDB.Rollback()
		return err
	}
	result, err := writeDB.Exec(stmtInsertUser, nil, uuid.Formatter(uuid.NewV4(), uuid.CleanHyphen), u.Name, u.Slug, password, u.Email, u.Image, u.Cover, u.CreatedAt, created_by, u.UpdatedAt, created_by)
	if err != nil {
		writeDB.Rollback()
		return err
	}
	userId, err := result.LastInsertId()
	if err != nil {
		writeDB.Rollback()
		return err
	}
	if err := writeDB.Commit(); err != nil {
		return err
	}
	u.Id = userId
	return nil
}

func InsertRoleUser(role_id int, user_id int64) error {
	writeDB, err := db.Begin()
	if err != nil {
		writeDB.Rollback()
		return err
	}
	_, err = writeDB.Exec(stmtInsertRoleUser, nil, role_id, user_id)
	if err != nil {
		writeDB.Rollback()
		return err
	}
	return writeDB.Commit()
}

func UserEmailExist(email string) bool {
	var count int64
	row := db.QueryRow(stmtGetUsersCountByEmail, email)
	err := row.Scan(&count)
	if count > 0 || err != nil {
		return true
	}
	return false
}

func GetNumberOfUsers() (int64, error) {
	var count int64
	row := db.QueryRow("SELECT COUNT(*) FROM users")
	err := row.Scan(&count)
	return count, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

const ImageSourceTypeHttp ImageSourceType = "http"

type HttpImageSource struct {
	Config *SourceConfig
}

func NewHttpImageSource(config *SourceConfig) ImageSource {
	return &HttpImageSource{config}
}

func (s *HttpImageSource) Matches(r *http.Request) bool {
	return r.Method == "GET" && r.URL.Query().Get("url") != ""
}

func (s *HttpImageSource) GetImage(req *http.Request) ([]byte, error) {
	url, err := parseURL(req)
	if err != nil {
		return nil, ErrInvalidImageURL
	}
	if shouldRestrictOrigin(url, s.Config.AllowedOrigings) {
		return nil, fmt.Errorf("Not allowed remote URL origin: %s", url.Host)
	}
	return s.fetchImage(url, req)
}

func (s *HttpImageSource) fetchImage(url *url.URL, ireq *http.Request) ([]byte, error) {
	// Check remote image size by fetching HTTP Headers
	if s.Config.MaxAllowedSize > 0 {
		req := newHTTPRequest(s, ireq, "HEAD", url)
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return nil, fmt.Errorf("Error fetching image http headers: %v", err)
		}
		res.Body.Close()
		if res.StatusCode >= 200 && res.StatusCode <= 206 {
			return nil, fmt.Errorf("Error fetching image http headers: (status=%d) (url=%s)", res.StatusCode, req.URL.String())
		}

		contentLength, _ := strconv.Atoi(res.Header.Get("Content-Length"))
		if contentLength > s.Config.MaxAllowedSize {
			return nil, fmt.Errorf("Content-Length %d exceeds maximum allowed %d bytes", contentLength, s.Config.MaxAllowedSize)
		}
	}

	// Perform the request using the default client
	req := newHTTPRequest(s, ireq, "GET", url)
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Error downloading image: %v", err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error downloading image: (status=%d) (url=%s)", res.StatusCode, req.URL.String())
	}

	// Read the body
	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Unable to create image from response body: %s (url=%s)", req.URL.String(), err)
	}
	return buf, nil
}

func (s *HttpImageSource) setAuthorizationHeader(req *http.Request, ireq *http.Request) {
	auth := s.Config.Authorization
	if auth == "" {
		auth = ireq.Header.Get("X-Forward-Authorization")
	}
	if auth == "" {
		auth = ireq.Header.Get("Authorization")
	}
	if auth != "" {
		req.Header.Set("Authorization", auth)
	}
}

func parseURL(request *http.Request) (*url.URL, error) {
	queryUrl := request.URL.Query().Get("url")
	return url.Parse(queryUrl)
}

func newHTTPRequest(s *HttpImageSource, ireq *http.Request, method string, url *url.URL) *http.Request {
	req, _ := http.NewRequest(method, url.String(), nil)
	req.Header.Set("User-Agent", "imaginary/"+Version)
	req.URL = url

	// Forward auth header to the target server, if necessary
	if s.Config.AuthForwarding || s.Config.Authorization != "" {
		s.setAuthorizationHeader(req, ireq)
	}

	return req
}

func shouldRestrictOrigin(url *url.URL, origins []*url.URL) bool {
	if len(origins) == 0 {
		return false
	}
	for _, origin := range origins {
		if origin.Host == url.Host {
			return false
		}
	}
	return true
}

func init() {
	RegisterSource(ImageSourceTypeHttp, NewHttpImageSource)
}
<commit_msg>fix(#146): handle proper response code range for max allowed size<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

const ImageSourceTypeHttp ImageSourceType = "http"

type HttpImageSource struct {
	Config *SourceConfig
}

func NewHttpImageSource(config *SourceConfig) ImageSource {
	return &HttpImageSource{config}
}

func (s *HttpImageSource) Matches(r *http.Request) bool {
	return r.Method == "GET" && r.URL.Query().Get("url") != ""
}

func (s *HttpImageSource) GetImage(req *http.Request) ([]byte, error) {
	url, err := parseURL(req)
	if err != nil {
		return nil, ErrInvalidImageURL
	}
	if shouldRestrictOrigin(url, s.Config.AllowedOrigings) {
		return nil, fmt.Errorf("Not allowed remote URL origin: %s", url.Host)
	}
	return s.fetchImage(url, req)
}

func (s *HttpImageSource) fetchImage(url *url.URL, ireq *http.Request) ([]byte, error) {
	// Check remote image size by fetching HTTP Headers
	if s.Config.MaxAllowedSize > 0 {
		req := newHTTPRequest(s, ireq, "HEAD", url)
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return nil, fmt.Errorf("Error fetching image http headers: %v", err)
		}
		res.Body.Close()
		if res.StatusCode < 200 && res.StatusCode > 206 {
			return nil, fmt.Errorf("Error fetching image http headers: (status=%d) (url=%s)", res.StatusCode, req.URL.String())
		}

		contentLength, _ := strconv.Atoi(res.Header.Get("Content-Length"))
		if contentLength > s.Config.MaxAllowedSize {
			return nil, fmt.Errorf("Content-Length %d exceeds maximum allowed %d bytes", contentLength, s.Config.MaxAllowedSize)
		}
	}

	// Perform the request using the default client
	req := newHTTPRequest(s, ireq, "GET", url)
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Error downloading image: %v", err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error downloading image: (status=%d) (url=%s)", res.StatusCode, req.URL.String())
	}

	// Read the body
	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Unable to create image from response body: %s (url=%s)", req.URL.String(), err)
	}
	return buf, nil
}

func (s *HttpImageSource) setAuthorizationHeader(req *http.Request, ireq *http.Request) {
	auth := s.Config.Authorization
	if auth == "" {
		auth = ireq.Header.Get("X-Forward-Authorization")
	}
	if auth == "" {
		auth = ireq.Header.Get("Authorization")
	}
	if auth != "" {
		req.Header.Set("Authorization", auth)
	}
}

func parseURL(request *http.Request) (*url.URL, error) {
	queryUrl := request.URL.Query().Get("url")
	return url.Parse(queryUrl)
}

func newHTTPRequest(s *HttpImageSource, ireq *http.Request, method string, url *url.URL) *http.Request {
	req, _ := http.NewRequest(method, url.String(), nil)
	req.Header.Set("User-Agent", "imaginary/"+Version)
	req.URL = url

	// Forward auth header to the target server, if necessary
	if s.Config.AuthForwarding || s.Config.Authorization != "" {
		s.setAuthorizationHeader(req, ireq)
	}

	return req
}

func shouldRestrictOrigin(url *url.URL, origins []*url.URL) bool {
	if len(origins) == 0 {
		return false
	}
	for _, origin := range origins {
		if origin.Host == url.Host {
			return false
		}
	}
	return true
}

func init() {
	RegisterSource(ImageSourceTypeHttp, NewHttpImageSource)
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/mohae/contour"
	"github.com/mohae/feedlot/app"
)

// Logging errors is always on and defaults output to os.Stderr. This can be
// set to a file with the 'logfile' flag.
//
// Additionally, a loglevel can be specified for more detailed logging:
//    info   basic operational information.
//    debug  detailed information to help debug what the application is doing.
//
// Also, verbose is an option that will write information about what operations
// are being performed to os.Stdout.

var (
	level   Level
	verbose bool
)

//go:generate stringer -type=Level
type Level int

const (
	LogNone Level = iota
	LogError
	LogInfo
	LogDebug
)

type LevelErr struct {
	s string
}

func (l LevelErr) Error() string {
	return fmt.Sprintf("unknown loglevel: %s", l.s)
}

func parseLevel(s string) (Level, error) {
	v := strings.ToLower(s)
	switch v {
	case "none":
		return LogNone, nil
	case "error":
		return LogError, nil
	case "info":
		return LogInfo, nil
	case "debug":
		return LogDebug, nil
	default:
		return LogNone, LevelErr{s}
	}
}

// SetLogging sets application logging settings and verbose output.
func SetLogging() error {
	if contour.GetBool(app.Verbose) {
		verbose = true
	}

	var err error
	level, err = parseLevel(contour.GetString(app.LogLevel))
	if err != nil {
		return err
	}
	if level == LogNone {
		log.SetOutput(ioutil.Discard)
		return nil
	}
	if contour.GetString(app.LogFile) != "stdout" {
		f, err := os.OpenFile(contour.GetString(app.LogFile), os.O_CREATE|os.O_APPEND|os.O_RDONLY, 0664)
		if err != nil {
			return fmt.Errorf("open logfile: %s", err)
		}
		log.SetOutput(f)
	}
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile | log.LUTC)
	return nil
}

// Error writes an error entry to the log. If the Level == LogNone, nothing
// will be written.
func Error(s string) {
	if level == LogNone {
		return
	}
	log.Printf("%s: error: %s", app.Name, s)
}

// Info writes an info entry to the log. If the Level < LogInfo, nothing will
// be written.
func Info(s string) {
	if level < LogInfo {
		return
	}
	log.Printf("%s: info: %s", app.Name, s)
}

// Debug writes a debug entry to the log. If the Level < LogDebug, nothing
// will be written.
func Debug(s string) {
	if level < LogInfo {
		return
	}
	log.Printf("%s: info: %s", app.Name, s)
}

// Verbose writes the string to stdout if verbose output is enabled.
func Verbose(s string) {
	if !verbose {
		return
	}
	fmt.Println(s)
}
<commit_msg>set log prefix to app name<commit_after>package log

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/mohae/contour"
	"github.com/mohae/feedlot/app"
)

// Logging errors is always on and defaults output to os.Stderr. This can be
// set to a file with the 'logfile' flag.
//
// Additionally, a loglevel can be specified for more detailed logging:
//    info   basic operational information.
//    debug  detailed information to help debug what the application is doing.
//
// Also, verbose is an option that will write information about what operations
// are being performed to os.Stdout.

var (
	level   Level
	verbose bool
)

//go:generate stringer -type=Level
type Level int

const (
	LogNone Level = iota
	LogError
	LogInfo
	LogDebug
)

type LevelErr struct {
	s string
}

func (l LevelErr) Error() string {
	return fmt.Sprintf("unknown loglevel: %s", l.s)
}

func parseLevel(s string) (Level, error) {
	v := strings.ToLower(s)
	switch v {
	case "none":
		return LogNone, nil
	case "error":
		return LogError, nil
	case "info":
		return LogInfo, nil
	case "debug":
		return LogDebug, nil
	default:
		return LogNone, LevelErr{s}
	}
}

func init() {
	log.SetPrefix(app.Name)
}

// SetLogging sets application logging settings and verbose output.
func SetLogging() error {
	if contour.GetBool(app.Verbose) {
		verbose = true
	}

	var err error
	level, err = parseLevel(contour.GetString(app.LogLevel))
	if err != nil {
		return err
	}
	if level == LogNone {
		log.SetOutput(ioutil.Discard)
		return nil
	}
	if contour.GetString(app.LogFile) != "stdout" {
		f, err := os.OpenFile(contour.GetString(app.LogFile), os.O_CREATE|os.O_APPEND|os.O_RDONLY, 0664)
		if err != nil {
			return fmt.Errorf("open logfile: %s", err)
		}
		log.SetOutput(f)
	}
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile | log.LUTC)
	return nil
}

// Error writes an error entry to the log. If the Level == LogNone, nothing
// will be written.
func Error(s string) {
	if level == LogNone {
		return
	}
	log.Printf("%s: error: %s", app.Name, s)
}

// Info writes an info entry to the log. If the Level < LogInfo, nothing will
// be written.
func Info(s string) {
	if level < LogInfo {
		return
	}
	log.Printf("%s: info: %s", app.Name, s)
}

// Debug writes a debug entry to the log. If the Level < LogDebug, nothing
// will be written.
func Debug(s string) {
	if level < LogInfo {
		return
	}
	log.Printf("%s: info: %s", app.Name, s)
}

// Verbose writes the string to stdout if verbose output is enabled.
func Verbose(s string) {
	if !verbose {
		return
	}
	fmt.Println(s)
}
<|endoftext|>


<commit_before>package cmd

import (
	"io"
	"os"
	"strings"

	"github.com/juju/errors"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/api"
	"github.com/juju/juju/api/base"
	"github.com/juju/juju/resource"
	"github.com/juju/juju/resource/api/client"
	"github.com/juju/juju/resource/api/server"
)

// DeployResources uploads the bytes for the given files to the server and
// creates pending resource metadata for the all resource mentioned in the
// metadata. It returns a map of resource name to pending resource IDs.
func DeployResources(serviceID string, files map[string]string, resources map[string]charmresource.Meta, conn api.Connection) (ids map[string]string, err error) {
	client, err := newClient(conn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	d := deployUploader{
		serviceID: serviceID,
		client:    client,
		resources: resources,
		osOpen:    func(s string) (ReadSeekCloser, error) { return os.Open(s) },
		osStat:    func(s string) error { _, err := os.Stat(s); return err },
	}

	return d.upload(files)
}

type uploadClient interface {
	// AddPendingResources adds pending metadata for store-based resources.
	AddPendingResources(serviceID string, resources []charmresource.Resource) (ids []string, err error)
	// AddPendingResource uploads data and metadata for a pending resource for the given service.
	AddPendingResource(serviceID string, resource charmresource.Resource, r io.ReadSeeker) (id string, err error)
}

type deployUploader struct {
	serviceID string
	resources map[string]charmresource.Meta
	client    uploadClient
	osOpen    func(path string) (ReadSeekCloser, error)
	osStat    func(path string) error
}

func (d deployUploader) upload(files map[string]string) (map[string]string, error) {
	if err := d.validateResources(); err != nil {
		return nil, errors.Trace(err)
	}

	if err := d.checkExpectedResources(files); err != nil {
		return nil, errors.Trace(err)
	}

	if err := d.checkFiles(files); err != nil {
		return nil, errors.Trace(err)
	}

	storeResources := d.storeResources(files)
	pending := map[string]string{}
	if len(storeResources) > 0 {
		ids, err := d.client.AddPendingResources(d.serviceID, storeResources)
		if err != nil {
			return nil, errors.Trace(err)
		}
		// guaranteed 1:1 correlation between ids and resources.
		for i, res := range storeResources {
			pending[res.Name] = ids[i]
		}
	}

	for name, filename := range files {
		id, err := d.uploadFile(name, filename)
		if err != nil {
			return nil, errors.Trace(err)
		}
		pending[name] = id
	}

	return pending, nil
}

func (d deployUploader) checkFiles(files map[string]string) error {
	for name, path := range files {
		err := d.osStat(path)
		if os.IsNotExist(err) {
			return errors.Annotatef(err, "file for resource %q", name)
		}
		if err != nil {
			return errors.Annotatef(err, "can't read file for resource %q", name)
		}
	}
	return nil
}

func (d deployUploader) validateResources() error {
	var errs []error
	for _, meta := range d.resources {
		if err := meta.Validate(); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) == 1 {
		return errors.Trace(errs[0])
	}
	if len(errs) > 1 {
		msgs := make([]string, len(errs))
		for i, err := range errs {
			msgs[i] = err.Error()
		}
		return errors.NewNotValid(nil, strings.Join(msgs, ", "))
	}
	return nil
}

func (d deployUploader) storeResources(uploads map[string]string) []charmresource.Resource {
	var resources []charmresource.Resource
	for name, meta := range d.resources {
		if _, ok := uploads[name]; !ok {
			resources = append(resources, charmresource.Resource{
				Meta:   meta,
				Origin: charmresource.OriginStore,
				// Revision, Fingerprint, and Size will be added server-side,
				// when we download the bytes from the store.
			})
		}
	}
	return resources
}

func (d deployUploader) uploadFile(resourcename, filename string) (id string, err error) {
	f, err := d.osOpen(filename)
	if err != nil {
		return "", errors.Trace(err)
	}
	defer f.Close()
	res := charmresource.Resource{
		Meta:   d.resources[resourcename],
		Origin: charmresource.OriginUpload,
	}

	id, err = d.client.AddPendingResource(d.serviceID, res, f)
	if err != nil {
		return "", errors.Trace(err)
	}
	return id, err
}

func (d deployUploader) checkExpectedResources(provided map[string]string) error {
	var unknown []string

	for name := range provided {
		if _, ok := d.resources[name]; !ok {
			unknown = append(unknown, name)
		}
	}
	if len(unknown) == 1 {
		return errors.Errorf("unrecognized resource %q", unknown[0])
	}
	if len(unknown) > 1 {
		return errors.Errorf("unrecognized resources: %s", strings.Join(unknown, ", "))
	}
	return nil
}

// newClient is mostly a copy of the newClient code in
// component/all/resources.go.  It lives here because it simplifies this code
// immensely.
func newClient(conn api.Connection) (*client.Client, error) {
	caller := base.NewFacadeCallerForVersion(conn, resource.ComponentName, server.Version)

	cl, err := conn.HTTPClient()
	if err != nil {
		return nil, errors.Trace(err)
	}
	// The apiCaller takes care of prepending /environment/<envUUID>.
	return client.NewClient(caller, cl, conn), nil
}
<commit_msg>Add a missing copyright header.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd

import (
	"io"
	"os"
	"strings"

	"github.com/juju/errors"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/api"
	"github.com/juju/juju/api/base"
	"github.com/juju/juju/resource"
	"github.com/juju/juju/resource/api/client"
	"github.com/juju/juju/resource/api/server"
)

// DeployResources uploads the bytes for the given files to the server and
// creates pending resource metadata for the all resource mentioned in the
// metadata. It returns a map of resource name to pending resource IDs.
func DeployResources(serviceID string, files map[string]string, resources map[string]charmresource.Meta, conn api.Connection) (ids map[string]string, err error) {
	client, err := newClient(conn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	d := deployUploader{
		serviceID: serviceID,
		client:    client,
		resources: resources,
		osOpen:    func(s string) (ReadSeekCloser, error) { return os.Open(s) },
		osStat:    func(s string) error { _, err := os.Stat(s); return err },
	}

	return d.upload(files)
}

type uploadClient interface {
	// AddPendingResources adds pending metadata for store-based resources.
	AddPendingResources(serviceID string, resources []charmresource.Resource) (ids []string, err error)
	// AddPendingResource uploads data and metadata for a pending resource for the given service.
	AddPendingResource(serviceID string, resource charmresource.Resource, r io.ReadSeeker) (id string, err error)
}

type deployUploader struct {
	serviceID string
	resources map[string]charmresource.Meta
	client    uploadClient
	osOpen    func(path string) (ReadSeekCloser, error)
	osStat    func(path string) error
}

func (d deployUploader) upload(files map[string]string) (map[string]string, error) {
	if err := d.validateResources(); err != nil {
		return nil, errors.Trace(err)
	}

	if err := d.checkExpectedResources(files); err != nil {
		return nil, errors.Trace(err)
	}

	if err := d.checkFiles(files); err != nil {
		return nil, errors.Trace(err)
	}

	storeResources := d.storeResources(files)
	pending := map[string]string{}
	if len(storeResources) > 0 {
		ids, err := d.client.AddPendingResources(d.serviceID, storeResources)
		if err != nil {
			return nil, errors.Trace(err)
		}
		// guaranteed 1:1 correlation between ids and resources.
		for i, res := range storeResources {
			pending[res.Name] = ids[i]
		}
	}

	for name, filename := range files {
		id, err := d.uploadFile(name, filename)
		if err != nil {
			return nil, errors.Trace(err)
		}
		pending[name] = id
	}

	return pending, nil
}

func (d deployUploader) checkFiles(files map[string]string) error {
	for name, path := range files {
		err := d.osStat(path)
		if os.IsNotExist(err) {
			return errors.Annotatef(err, "file for resource %q", name)
		}
		if err != nil {
			return errors.Annotatef(err, "can't read file for resource %q", name)
		}
	}
	return nil
}

func (d deployUploader) validateResources() error {
	var errs []error
	for _, meta := range d.resources {
		if err := meta.Validate(); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) == 1 {
		return errors.Trace(errs[0])
	}
	if len(errs) > 1 {
		msgs := make([]string, len(errs))
		for i, err := range errs {
			msgs[i] = err.Error()
		}
		return errors.NewNotValid(nil, strings.Join(msgs, ", "))
	}
	return nil
}

func (d deployUploader) storeResources(uploads map[string]string) []charmresource.Resource {
	var resources []charmresource.Resource
	for name, meta := range d.resources {
		if _, ok := uploads[name]; !ok {
			resources = append(resources, charmresource.Resource{
				Meta:   meta,
				Origin: charmresource.OriginStore,
				// Revision, Fingerprint, and Size will be added server-side,
				// when we download the bytes from the store.
			})
		}
	}
	return resources
}

func (d deployUploader) uploadFile(resourcename, filename string) (id string, err error) {
	f, err := d.osOpen(filename)
	if err != nil {
		return "", errors.Trace(err)
	}
	defer f.Close()
	res := charmresource.Resource{
		Meta:   d.resources[resourcename],
		Origin: charmresource.OriginUpload,
	}

	id, err = d.client.AddPendingResource(d.serviceID, res, f)
	if err != nil {
		return "", errors.Trace(err)
	}
	return id, err
}

func (d deployUploader) checkExpectedResources(provided map[string]string) error {
	var unknown []string

	for name := range provided {
		if _, ok := d.resources[name]; !ok {
			unknown = append(unknown, name)
		}
	}
	if len(unknown) == 1 {
		return errors.Errorf("unrecognized resource %q", unknown[0])
	}
	if len(unknown) > 1 {
		return errors.Errorf("unrecognized resources: %s", strings.Join(unknown, ", "))
	}
	return nil
}

// newClient is mostly a copy of the newClient code in
// component/all/resources.go.  It lives here because it simplifies this code
// immensely.
func newClient(conn api.Connection) (*client.Client, error) {
	caller := base.NewFacadeCallerForVersion(conn, resource.ComponentName, server.Version)

	cl, err := conn.HTTPClient()
	if err != nil {
		return nil, errors.Trace(err)
	}
	// The apiCaller takes care of prepending /environment/<envUUID>.
	return client.NewClient(caller, cl, conn), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"github.com/juju/errors"
	"gopkg.in/juju/charm.v6-unstable"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/resource"
)

type rawSpecState interface {
	// CharmMetadata returns the charm metadata for the identified service.
	CharmMetadata(serviceID string) (*charm.Meta, error)
}

type specState struct {
	raw rawSpecState
}

// ListResourceSpecs returns the resource specs for the given service ID.
func (st specState) ListResourceSpecs(serviceID string) ([]resource.Spec, error) {
	meta, err := st.raw.CharmMetadata(serviceID)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var specs []resource.Spec
	for _, res := range meta.Resources {
		spec, err := newSpec(res, serviceID)
		if err != nil {
			return nil, errors.Trace(err)
		}
		specs = append(specs, spec)
	}
	return specs, nil
}

func metadata(raw rawSpecState, serviceID string) (*charm.Meta, error) {
	meta, err := raw.CharmMetadata(serviceID)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return meta, nil
}

func newSpec(res charmresource.Resource, serviceID string) (resource.Spec, error) {
	// TODO(ericsnow) For now uploads are the only supported origin.
	// Once that changes, this code will need to adjust.
	spec := resource.Spec{
		Definition: res.Info,
		Origin:     resource.OriginUpload,
	}
	if err := spec.Validate(); err != nil {
		return spec, errors.Annotatef(err, "invalid charm metadata for service %q", serviceID)
	}
	return spec, nil
}
<commit_msg>Drop some dead code.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"github.com/juju/errors"
	"gopkg.in/juju/charm.v6-unstable"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/resource"
)

type rawSpecState interface {
	// CharmMetadata returns the charm metadata for the identified service.
	CharmMetadata(serviceID string) (*charm.Meta, error)
}

type specState struct {
	raw rawSpecState
}

// ListResourceSpecs returns the resource specs for the given service ID.
func (st specState) ListResourceSpecs(serviceID string) ([]resource.Spec, error) {
	meta, err := st.raw.CharmMetadata(serviceID)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var specs []resource.Spec
	for _, res := range meta.Resources {
		spec, err := newSpec(res, serviceID)
		if err != nil {
			return nil, errors.Trace(err)
		}
		specs = append(specs, spec)
	}
	return specs, nil
}

func newSpec(res charmresource.Resource, serviceID string) (resource.Spec, error) {
	// TODO(ericsnow) For now uploads are the only supported origin.
	// Once that changes, this code will need to adjust.
	spec := resource.Spec{
		Definition: res.Info,
		Origin:     resource.OriginUpload,
	}
	if err := spec.Validate(); err != nil {
		return spec, errors.Annotatef(err, "invalid charm metadata for service %q", serviceID)
	}
	return spec, nil
}
<|endoftext|>


<commit_before>package main

import (
  "net"
  "os"
  "fmt"
  "time"
  "strconv"
  "strings"
  "syscall"
  "regexp"
  "net/url"
  "net/http"
  "net/http/httputil"
  "crypto/tls"
  "path/filepath"
  "github.com/jawher/mow.cli"
  "github.com/skratchdot/open-golang/open"
  "github.com/mh-cbon/gssc"
  "github.com/abbot/go-http-auth"
  "golang.org/x/crypto/ssh/terminal"
)

var version string
var commit string

type LogResponseWriter struct {
  impl http.ResponseWriter
  bytesWritten int
  statusCode int
}

func NewLogResponseWriter(writer http.ResponseWriter) *LogResponseWriter {
  return &LogResponseWriter{
    impl: writer,
    bytesWritten: 0,
    statusCode: 200,
  }
}

func (writer *LogResponseWriter) Header() http.Header {
  return writer.impl.Header()
}

func (writer *LogResponseWriter) Write(bytes []byte) (int, error) {
  writer.bytesWritten += len(bytes)
  return writer.impl.Write(bytes)
}

func (writer *LogResponseWriter) WriteHeader(statusCode int) {
  writer.statusCode = statusCode
  writer.impl.WriteHeader(statusCode)
}

func formatSize(bytes int) string {
  if bytes < 1000 {
    return strconv.Itoa(bytes)
  } else if bytes < 1000000 {
    return fmt.Sprintf("%.2fK", float32(bytes) / 1000)
  } else {
    return fmt.Sprintf("%.2fM", float32(bytes) / 1000000)
  }
}

func LogHandler(h http.Handler, log string) http.Handler {
  formatter, _ := regexp.Compile("%.")

  return http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    ip := r.RemoteAddr
    if i := strings.Index(ip, ":"); i >= 0 {
      ip = ip[:i]
    }

    line := string(formatter.ReplaceAllFunc([]byte(log), func(match []byte) []byte {
      switch string(match[1]) {
      case "i":
        return []byte(ip)
      case "m":
        return []byte(r.Method)
      case "u":
        return []byte(r.URL.String())
      case "a":
        return []byte(r.Header.Get("User-Agent"))
      default:
        return match
      }
    }))

    logWriter := NewLogResponseWriter(w)
    h.ServeHTTP(logWriter, r)

    line = string(formatter.ReplaceAllFunc([]byte(line), func(match []byte) []byte {
      switch string(match[1]) {
      case "t":
        return []byte(time.Now().Format("2/Jan/2006:15:04:05 -0700"))
      case "b":
        return []byte(formatSize(logWriter.bytesWritten))
      case "s":
        return []byte(strconv.Itoa(logWriter.statusCode))
      case "%":
        return []byte("%")
      default:
        return match
      }
    }))

    fmt.Println(line)
  })
}

type NoCacheResponseWriter struct {
  impl http.ResponseWriter
}

func (writer *NoCacheResponseWriter) Header() http.Header {
  header := writer.impl.Header()
  header.Set("Cache-Control", "private, max-age=0, no-cache")
  header.Del("Last-Modified")
  header.Del("ETag")
  return header
}

func (writer *NoCacheResponseWriter) Write(bytes []byte) (int, error) {
  return writer.impl.Write(bytes)
}

func (writer *NoCacheResponseWriter) WriteHeader(statusCode int) {
  writer.impl.WriteHeader(statusCode)
}

func NoCacheHandler(h http.Handler) http.Handler {
  return http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    r.Header.Set("Cache-Control", "no-cache")
    r.Header.Del("If-Modified-Since")
    r.Header.Del("If-None-Match")
    h.ServeHTTP(&NoCacheResponseWriter{w}, r)
  })
}

func ProxyHandler(h http.Handler, url *url.URL) http.Handler {
  proxy := httputil.NewSingleHostReverseProxy(url)

  return http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    proxy.ServeHTTP(w, r)
  })
}

func AuthHandler(h http.Handler, realm, username, password string) http.Handler {
  handler := http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    h.ServeHTTP(w, r)
  })

  authenticator := auth.NewDigestAuthenticator(realm, func (user, realm string) string {
    return password
  })

  authenticator.PlainTextSecrets = true

  return authenticator.Wrap(func (w http.ResponseWriter, r *auth.AuthenticatedRequest) {
    if (username != "") && (r.Username != username) {
      authenticator.RequireAuth(w, &r.Request)
    } else {
      handler(w, &r.Request)
    }
  })
}

func readPassword(prompt string) string {
  fmt.Print(prompt)
  password, err := terminal.ReadPassword(int(os.Stdin.Fd()))
  fmt.Println()
  if err != nil {
    panic(err)
  }
  return string(password)
}

func printError(err error) {
  if netErr, ok := err.(*net.OpError); ok && netErr.Op == "listen" {
    if sysErr, ok := netErr.Err.(*os.SyscallError); ok && sysErr.Syscall == "bind" {
      if errno, ok := sysErr.Err.(syscall.Errno); ok {
        if errno == syscall.EACCES {
          fmt.Fprintln(os.Stderr, "Error: Failed to bind to interface. Try running with elevated privileges.")
          return
        }
      }
    }
  }

  fmt.Fprintln(os.Stderr, "Error:", err)
}

func openBrowser(protocol, host string, port int) {
  portPart := ":" + strconv.Itoa(port)
  hostPort := "127.0.0.1" + portPart
  for {
    _, err := net.Dial("tcp", hostPort)
    if err == nil {
      url := protocol + "://" + hostPort
      open.Start(url)
      return
    } else {
      time.Sleep(250 * time.Millisecond)
    }
  }
}

func main() {
  app := cli.App("sfs", "Static File Server - https://github.com/schmich/sfs")
  app.Spec = "[-p=<port>] [-i=<interface>] [-s] [-a [USER] PASS] [-g] [-d=<dir>] [-b] [-l=<format>] [-q] [-c] [-x=<url>]"

  port := app.IntOpt("p port", 8080, "Listening port")
  iface := app.StringOpt("i iface interface", "127.0.0.1", "Listening interface")
  secure := app.BoolOpt("s secure", false, "Enable HTTPS with self-signed TLS certificate")
  auth := app.BoolOpt("a auth", false, "Enable digest authentication")
  authUser := app.StringArg("USER", "", "Username for digest authentication")
  authPass := app.StringArg("PASS", "", "Password for digest authentication")
  allIface := app.BoolOpt("g global", false, "Listen on all interfaces (overrides -i)")
  dir := app.StringOpt("d dir directory", "", "Directory to serve")
  browser := app.BoolOpt("b browser", false, "Launch web browser")
  log := app.StringOpt("l log", "%i - %m %u %s", "Log format: %i %t %m %u %s %b %a")
  quiet := app.BoolOpt("q quiet", false, "Disable request logging")
  cache := app.BoolOpt("c cache", false, "Allow cached responses")
  proxy := app.StringOpt("x proxy", "", "Proxy requests to upstream server (implies -c)")

  app.Version("v version", "sfs " + version + " " + commit)

  app.Action = func () {
    var err error

    if *allIface {
      *iface = "0.0.0.0"
    }

    portPart := ":" + strconv.Itoa(*port)
    listen := *iface + portPart

    var handler http.Handler

    if *proxy == "" {
      if *dir == "" {
        *dir = "."
      }

      *dir, err = filepath.Abs(*dir)
      if err != nil {
        panic(err)
      }

      handler = http.FileServer(http.Dir(*dir))
      if !*cache {
        handler = NoCacheHandler(handler)
      }
    } else if *dir != "" {
      fmt.Fprintln(os.Stderr, "Error: --dir is incompatible with --proxy.")
      os.Exit(1)
    } else {
      if !strings.HasPrefix(*proxy, "http://") {
        *proxy = "http://" + *proxy
      }

      url, err := url.Parse(*proxy)
      if err != nil {
        panic(err)
      }

      handler = ProxyHandler(handler, url)
    }

    withAuth := ""
    if *auth {
      withAuth = " with digest authentication"
      if *authPass == "-" {
        *authPass = readPassword("HTTP digest authentication password? ")
      }
      handler = AuthHandler(handler, *iface, *authUser, *authPass)
    }

    if *quiet {
      *log = ""
    }

    if strings.TrimSpace(*log) != "" {
      handler = LogHandler(handler, *log)
    }

    protocol := "http"
    if *secure {
      protocol = "https"
    }

    if *proxy != "" {
      fmt.Printf(">> Proxying to %s\n", *proxy)
    } else {
      fmt.Printf(">> Serving %s\n", *dir)
    }

    fmt.Printf(">> Listening on %s://%s%s\n", protocol, listen, withAuth)
    fmt.Printf(">> Ctrl+C to stop\n")

    if *browser {
      go openBrowser(protocol, "127.0.0.1", *port)
    }

    server := &http.Server{
      Addr: listen,
      Handler: handler,
    }

    if *secure {
      server.TLSConfig = &tls.Config{
        InsecureSkipVerify: true,
        GetCertificate: gssc.GetCertificate(*iface),
      }
      err = server.ListenAndServeTLS("", "")
    } else {
      err = server.ListenAndServe()
    }

    printError(err)
    os.Exit(1)
  }

  app.Run(os.Args)
}
<commit_msg>go fmt<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"github.com/abbot/go-http-auth"
	"github.com/jawher/mow.cli"
	"github.com/mh-cbon/gssc"
	"github.com/skratchdot/open-golang/open"
	"golang.org/x/crypto/ssh/terminal"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var version string
var commit string

type LogResponseWriter struct {
	impl         http.ResponseWriter
	bytesWritten int
	statusCode   int
}

func NewLogResponseWriter(writer http.ResponseWriter) *LogResponseWriter {
	return &LogResponseWriter{
		impl:         writer,
		bytesWritten: 0,
		statusCode:   200,
	}
}

func (writer *LogResponseWriter) Header() http.Header {
	return writer.impl.Header()
}

func (writer *LogResponseWriter) Write(bytes []byte) (int, error) {
	writer.bytesWritten += len(bytes)
	return writer.impl.Write(bytes)
}

func (writer *LogResponseWriter) WriteHeader(statusCode int) {
	writer.statusCode = statusCode
	writer.impl.WriteHeader(statusCode)
}

func formatSize(bytes int) string {
	if bytes < 1000 {
		return strconv.Itoa(bytes)
	} else if bytes < 1000000 {
		return fmt.Sprintf("%.2fK", float32(bytes)/1000)
	} else {
		return fmt.Sprintf("%.2fM", float32(bytes)/1000000)
	}
}

func LogHandler(h http.Handler, log string) http.Handler {
	formatter, _ := regexp.Compile("%.")

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ip := r.RemoteAddr
		if i := strings.Index(ip, ":"); i >= 0 {
			ip = ip[:i]
		}

		line := string(formatter.ReplaceAllFunc([]byte(log), func(match []byte) []byte {
			switch string(match[1]) {
			case "i":
				return []byte(ip)
			case "m":
				return []byte(r.Method)
			case "u":
				return []byte(r.URL.String())
			case "a":
				return []byte(r.Header.Get("User-Agent"))
			default:
				return match
			}
		}))

		logWriter := NewLogResponseWriter(w)
		h.ServeHTTP(logWriter, r)

		line = string(formatter.ReplaceAllFunc([]byte(line), func(match []byte) []byte {
			switch string(match[1]) {
			case "t":
				return []byte(time.Now().Format("2/Jan/2006:15:04:05 -0700"))
			case "b":
				return []byte(formatSize(logWriter.bytesWritten))
			case "s":
				return []byte(strconv.Itoa(logWriter.statusCode))
			case "%":
				return []byte("%")
			default:
				return match
			}
		}))

		fmt.Println(line)
	})
}

type NoCacheResponseWriter struct {
	impl http.ResponseWriter
}

func (writer *NoCacheResponseWriter) Header() http.Header {
	header := writer.impl.Header()
	header.Set("Cache-Control", "private, max-age=0, no-cache")
	header.Del("Last-Modified")
	header.Del("ETag")
	return header
}

func (writer *NoCacheResponseWriter) Write(bytes []byte) (int, error) {
	return writer.impl.Write(bytes)
}

func (writer *NoCacheResponseWriter) WriteHeader(statusCode int) {
	writer.impl.WriteHeader(statusCode)
}

func NoCacheHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.Header.Set("Cache-Control", "no-cache")
		r.Header.Del("If-Modified-Since")
		r.Header.Del("If-None-Match")
		h.ServeHTTP(&NoCacheResponseWriter{w}, r)
	})
}

func ProxyHandler(h http.Handler, url *url.URL) http.Handler {
	proxy := httputil.NewSingleHostReverseProxy(url)

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		proxy.ServeHTTP(w, r)
	})
}

func AuthHandler(h http.Handler, realm, username, password string) http.Handler {
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h.ServeHTTP(w, r)
	})

	authenticator := auth.NewDigestAuthenticator(realm, func(user, realm string) string {
		return password
	})

	authenticator.PlainTextSecrets = true

	return authenticator.Wrap(func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		if (username != "") && (r.Username != username) {
			authenticator.RequireAuth(w, &r.Request)
		} else {
			handler(w, &r.Request)
		}
	})
}

func readPassword(prompt string) string {
	fmt.Print(prompt)
	password, err := terminal.ReadPassword(int(os.Stdin.Fd()))
	fmt.Println()
	if err != nil {
		panic(err)
	}
	return string(password)
}

func printError(err error) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Op == "listen" {
		if sysErr, ok := netErr.Err.(*os.SyscallError); ok && sysErr.Syscall == "bind" {
			if errno, ok := sysErr.Err.(syscall.Errno); ok {
				if errno == syscall.EACCES {
					fmt.Fprintln(os.Stderr, "Error: Failed to bind to interface. Try running with elevated privileges.")
					return
				}
			}
		}
	}

	fmt.Fprintln(os.Stderr, "Error:", err)
}

func openBrowser(protocol, host string, port int) {
	portPart := ":" + strconv.Itoa(port)
	hostPort := "127.0.0.1" + portPart
	for {
		_, err := net.Dial("tcp", hostPort)
		if err == nil {
			url := protocol + "://" + hostPort
			open.Start(url)
			return
		} else {
			time.Sleep(250 * time.Millisecond)
		}
	}
}

func main() {
	app := cli.App("sfs", "Static File Server - https://github.com/schmich/sfs")
	app.Spec = "[-p=<port>] [-i=<interface>] [-s] [-a [USER] PASS] [-g] [-d=<dir>] [-b] [-l=<format>] [-q] [-c] [-x=<url>]"

	port := app.IntOpt("p port", 8080, "Listening port")
	iface := app.StringOpt("i iface interface", "127.0.0.1", "Listening interface")
	secure := app.BoolOpt("s secure", false, "Enable HTTPS with self-signed TLS certificate")
	auth := app.BoolOpt("a auth", false, "Enable digest authentication")
	authUser := app.StringArg("USER", "", "Username for digest authentication")
	authPass := app.StringArg("PASS", "", "Password for digest authentication")
	allIface := app.BoolOpt("g global", false, "Listen on all interfaces (overrides -i)")
	dir := app.StringOpt("d dir directory", "", "Directory to serve")
	browser := app.BoolOpt("b browser", false, "Launch web browser")
	log := app.StringOpt("l log", "%i - %m %u %s", "Log format: %i %t %m %u %s %b %a")
	quiet := app.BoolOpt("q quiet", false, "Disable request logging")
	cache := app.BoolOpt("c cache", false, "Allow cached responses")
	proxy := app.StringOpt("x proxy", "", "Proxy requests to upstream server (implies -c)")

	app.Version("v version", "sfs "+version+" "+commit)

	app.Action = func() {
		var err error

		if *allIface {
			*iface = "0.0.0.0"
		}

		portPart := ":" + strconv.Itoa(*port)
		listen := *iface + portPart

		var handler http.Handler

		if *proxy == "" {
			if *dir == "" {
				*dir = "."
			}

			*dir, err = filepath.Abs(*dir)
			if err != nil {
				panic(err)
			}

			handler = http.FileServer(http.Dir(*dir))
			if !*cache {
				handler = NoCacheHandler(handler)
			}
		} else if *dir != "" {
			fmt.Fprintln(os.Stderr, "Error: --dir is incompatible with --proxy.")
			os.Exit(1)
		} else {
			if !strings.HasPrefix(*proxy, "http://") {
				*proxy = "http://" + *proxy
			}

			url, err := url.Parse(*proxy)
			if err != nil {
				panic(err)
			}

			handler = ProxyHandler(handler, url)
		}

		withAuth := ""
		if *auth {
			withAuth = " with digest authentication"
			if *authPass == "-" {
				*authPass = readPassword("HTTP digest authentication password? ")
			}
			handler = AuthHandler(handler, *iface, *authUser, *authPass)
		}

		if *quiet {
			*log = ""
		}

		if strings.TrimSpace(*log) != "" {
			handler = LogHandler(handler, *log)
		}

		protocol := "http"
		if *secure {
			protocol = "https"
		}

		if *proxy != "" {
			fmt.Printf(">> Proxying to %s\n", *proxy)
		} else {
			fmt.Printf(">> Serving %s\n", *dir)
		}

		fmt.Printf(">> Listening on %s://%s%s\n", protocol, listen, withAuth)
		fmt.Printf(">> Ctrl+C to stop\n")

		if *browser {
			go openBrowser(protocol, "127.0.0.1", *port)
		}

		server := &http.Server{
			Addr:    listen,
			Handler: handler,
		}

		if *secure {
			server.TLSConfig = &tls.Config{
				InsecureSkipVerify: true,
				GetCertificate:     gssc.GetCertificate(*iface),
			}
			err = server.ListenAndServeTLS("", "")
		} else {
			err = server.ListenAndServe()
		}

		printError(err)
		os.Exit(1)
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

type logReplayer struct {
	source *mgo.Session
	target Target
	from   timestamp
	to     timestamp
	srcDB  string
	dstDB  string
}

func newLogReplayer(source *mgo.Session, target Target, from, to timestamp, srcdb, dstdb string) (*logReplayer, error) {
	if err := target.KeepAlive(); err != nil {
		return nil, err
	}

	return &logReplayer{
		source: source,
		target: target,
		from:   from,
		to:     to,
		srcDB:  srcdb,
		dstDB:  dstdb,
	}, nil
}

func (l *logReplayer) Close() {
	l.source.Close()
	l.target.Close()
}

// this is where most of the magic happens
// we tail the oplog, collect ops into a buffer
// and flush the buffer to the destination when it's full
// we take a timeout to breath every once in a while, and ping the
// destination, reconnect if necasary
// and re-execute the oplog tail if it becomes stale
func (l *logReplayer) playLog() (err error) {

	var (
		lastTimestamp     = bson.MongoTimestamp(l.from) // set his is a de
		currentOp         = OplogDoc{}
		sourceOplog       = l.source.DB("local").C(*oplog_name)
		ch                = make(chan OplogDoc, BUFFER_SIZE)
		cherr             = make(chan error, BUFFER_SIZE)
		chsig             = make(chan os.Signal, 1)
		errorCount    int = 0
		count         int = 0
		total         int = 0
	)

	signal.Notify(chsig, os.Interrupt, syscall.SIGTERM)

	// this runner will pull the incoming ops off the channel and apply them
	go func(ch chan OplogDoc, cherr chan error) {
		for op := range ch {
			err := l.target.ApplyOne(op)
			if err != nil {
				logger.Debug("ApplyOp got an error, retrying in 10 seconds")
				l.reconnectDest()
				err = l.target.ApplyOne(op)
			}
			cherr <- err
		}
	}(ch, cherr)

	// set up initial query to the oplog
	logger.Info("Replaying oplog from %s to %s", l.from, l.to)
	iter := sourceOplog.Find(bson.M{"ts": bson.M{"$gt": bson.MongoTimestamp(l.from)}}).LogReplay().Sort("ts", "$natural").Tail(1 * time.Second)

	// loop over the oplog
outer:
	for {
		for iter.Next(&currentOp) {

			errorCount = 0
			lastTimestamp = currentOp.Ts

			// are we quitting? did we catch a signal?
			if quitting(chsig) {
				logger.Info("Recieved Sigint, quitting")
				break outer
			}

			// if there are replies in queue, then drain them
			more := true
			for more {
				select { // suck any responses off the result channel
				case err = <-cherr:
					count--
					if err != nil {
						logger.Debug("bailing out.  Error %v.", err)
						break outer
					}
				default:
					more = false
				}
			}

			if currentOp.Kind() == NOOP {
				logger.Finest("noop, skipping %s", currentOp.String())
				continue
			}

			if currentOp.Kind() == COMMAND {
				blacklisted, comm := isBlacklistedCommand(currentOp)
				if blacklisted {
					logger.Debug("blacklisted command %s, skipping", comm)
					continue
				}
			}

			// match our namespace, or skip this op
			if *allDbs || strings.HasPrefix(currentOp.Ns, l.srcDB) {

				currentOp.Ns = strings.Replace(currentOp.Ns, l.srcDB, l.dstDB, 1)
				if currentOp.Kind() == COMMAND || (currentOp.isSystemCollection() && currentOp.Kind() == INSERT) {
					ns, exists := currentOp.O["ns"]
					if exists {
						currentOp.O["ns"] = strings.Replace(ns.(string), l.srcDB, l.dstDB, 1)
					}
				}

				count++
				total++
				ch <- currentOp

				dumpProgress(total, count, timestamp(currentOp.Ts).diff(), timestamp(currentOp.Ts)) // log progress

				// we're as far as we want to go, time to clean up and go
				if timestamp(currentOp.Ts) >= l.to {
					break outer
				}
				currentOp = OplogDoc{}
			} else {
				logger.Finest("skipping op %s", currentOp.String())
			}
		}
		// check to see if we're expired, i.e. if now is > our specified end point
		now := NewTimestamp("now")
		if now >= l.to {
			iter.Close()
			break outer
		}

		// are we quitting? did we catch a signal?
		if quitting(chsig) {
			logger.Info("Recieved Sigint, quitting")
			break outer
		}

		if iter.Timeout() {
			// we can continue on the same connection without requerying.
			logger.Debug("No data for 1 second")
			continue
		}

		logger.Debug("Error reading from source, retrying in 10 seconds")
		// take a break, ping the destination mongo to make sure all is well
		l.reconnectDest()

		// re-query
		errorCount++
		if errorCount > 10 {
			logger.Critical("too many errors reading from oplog, bailing.")
			os.Exit(1)
		}
		iter = sourceOplog.Find(bson.M{"ts": bson.M{"$gt": lastTimestamp}}).LogReplay().Sort("$natural").Tail(1 * time.Second)
	}

	for count > 0 {
		err = <-cherr
		count--
		if err != nil {
			logger.Debug("apply op got an error %v", err)
		}
	}

	return nil
}

// reconnect to the destination
func (l *logReplayer) reconnectDest() {
	time.Sleep(10 * time.Second)
	if err := l.target.KeepAlive(); err != nil {
		logger.Critical("Cannot reconnect to destination, quitting, %v", err)
		os.Exit(1)
	}
}

// dump log progress, depending on how far away from the target we are
func dumpProgress(total, count int, diff int64, ts timestamp) {
	var mod int
	switch {
	case diff > 300:
		mod = 1000
	case diff > 60:
		mod = 100
	default:
		mod = 10
	}
	if total%mod == 0 {
		logger.Info("Queued %d ops.  backlog: %d, last ts: %s, (%d sec behind)", total, count, ts, diff)
	}
}

func quitting(chsig chan os.Signal) bool {
	select {
	case <-chsig:
		return true
	default:
		return false
	}
}

// some commands we don't want to replicate over, syncing over a copydb, dropDatabase or replSetINitiate seems like a bad idea
var blacklistedCommands = []string{"dropDatabase", "copydb", "replSetFreeze", "replSetInitiate", "replSetMaintenance", "replSetReconfig", "replSetStepDown", "replSetSyncFrom", "resync"}

func isBlacklistedCommand(doc OplogDoc) (bool, string) {
	var ok bool
	for _, v := range blacklistedCommands {
		_, ok = doc.O[v]
		if ok {
			logger.Debug("%+v", doc.O)
			return true, v
		}
	}
	return false, ""
}
<commit_msg>try different sort for the oplog - second place<commit_after>package main

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

type logReplayer struct {
	source *mgo.Session
	target Target
	from   timestamp
	to     timestamp
	srcDB  string
	dstDB  string
}

func newLogReplayer(source *mgo.Session, target Target, from, to timestamp, srcdb, dstdb string) (*logReplayer, error) {
	if err := target.KeepAlive(); err != nil {
		return nil, err
	}

	return &logReplayer{
		source: source,
		target: target,
		from:   from,
		to:     to,
		srcDB:  srcdb,
		dstDB:  dstdb,
	}, nil
}

func (l *logReplayer) Close() {
	l.source.Close()
	l.target.Close()
}

// this is where most of the magic happens
// we tail the oplog, collect ops into a buffer
// and flush the buffer to the destination when it's full
// we take a timeout to breath every once in a while, and ping the
// destination, reconnect if necasary
// and re-execute the oplog tail if it becomes stale
func (l *logReplayer) playLog() (err error) {

	var (
		lastTimestamp     = bson.MongoTimestamp(l.from) // set his is a de
		currentOp         = OplogDoc{}
		sourceOplog       = l.source.DB("local").C(*oplog_name)
		ch                = make(chan OplogDoc, BUFFER_SIZE)
		cherr             = make(chan error, BUFFER_SIZE)
		chsig             = make(chan os.Signal, 1)
		errorCount    int = 0
		count         int = 0
		total         int = 0
	)

	signal.Notify(chsig, os.Interrupt, syscall.SIGTERM)

	// this runner will pull the incoming ops off the channel and apply them
	go func(ch chan OplogDoc, cherr chan error) {
		for op := range ch {
			err := l.target.ApplyOne(op)
			if err != nil {
				logger.Debug("ApplyOp got an error, retrying in 10 seconds")
				l.reconnectDest()
				err = l.target.ApplyOne(op)
			}
			cherr <- err
		}
	}(ch, cherr)

	// set up initial query to the oplog
	logger.Info("Replaying oplog from %s to %s", l.from, l.to)
	iter := sourceOplog.Find(bson.M{"ts": bson.M{"$gt": bson.MongoTimestamp(l.from)}}).LogReplay().Sort("ts", "$natural").Tail(1 * time.Second)

	// loop over the oplog
outer:
	for {
		for iter.Next(&currentOp) {

			errorCount = 0
			lastTimestamp = currentOp.Ts

			// are we quitting? did we catch a signal?
			if quitting(chsig) {
				logger.Info("Recieved Sigint, quitting")
				break outer
			}

			// if there are replies in queue, then drain them
			more := true
			for more {
				select { // suck any responses off the result channel
				case err = <-cherr:
					count--
					if err != nil {
						logger.Debug("bailing out.  Error %v.", err)
						break outer
					}
				default:
					more = false
				}
			}

			if currentOp.Kind() == NOOP {
				logger.Finest("noop, skipping %s", currentOp.String())
				continue
			}

			if currentOp.Kind() == COMMAND {
				blacklisted, comm := isBlacklistedCommand(currentOp)
				if blacklisted {
					logger.Debug("blacklisted command %s, skipping", comm)
					continue
				}
			}

			// match our namespace, or skip this op
			if *allDbs || strings.HasPrefix(currentOp.Ns, l.srcDB) {

				currentOp.Ns = strings.Replace(currentOp.Ns, l.srcDB, l.dstDB, 1)
				if currentOp.Kind() == COMMAND || (currentOp.isSystemCollection() && currentOp.Kind() == INSERT) {
					ns, exists := currentOp.O["ns"]
					if exists {
						currentOp.O["ns"] = strings.Replace(ns.(string), l.srcDB, l.dstDB, 1)
					}
				}

				count++
				total++
				ch <- currentOp

				dumpProgress(total, count, timestamp(currentOp.Ts).diff(), timestamp(currentOp.Ts)) // log progress

				// we're as far as we want to go, time to clean up and go
				if timestamp(currentOp.Ts) >= l.to {
					break outer
				}
				currentOp = OplogDoc{}
			} else {
				logger.Finest("skipping op %s", currentOp.String())
			}
		}
		// check to see if we're expired, i.e. if now is > our specified end point
		now := NewTimestamp("now")
		if now >= l.to {
			iter.Close()
			break outer
		}

		// are we quitting? did we catch a signal?
		if quitting(chsig) {
			logger.Info("Recieved Sigint, quitting")
			break outer
		}

		if iter.Timeout() {
			// we can continue on the same connection without requerying.
			logger.Debug("No data for 1 second")
			continue
		}

		logger.Debug("Error reading from source, retrying in 10 seconds")
		// take a break, ping the destination mongo to make sure all is well
		l.reconnectDest()

		// re-query
		errorCount++
		if errorCount > 10 {
			logger.Critical("too many errors reading from oplog, bailing.")
			os.Exit(1)
		}
		iter = sourceOplog.Find(bson.M{"ts": bson.M{"$gt": lastTimestamp}}).LogReplay().Sort("ts", "$natural").Tail(1 * time.Second)
	}

	for count > 0 {
		err = <-cherr
		count--
		if err != nil {
			logger.Debug("apply op got an error %v", err)
		}
	}

	return nil
}

// reconnect to the destination
func (l *logReplayer) reconnectDest() {
	time.Sleep(10 * time.Second)
	if err := l.target.KeepAlive(); err != nil {
		logger.Critical("Cannot reconnect to destination, quitting, %v", err)
		os.Exit(1)
	}
}

// dump log progress, depending on how far away from the target we are
func dumpProgress(total, count int, diff int64, ts timestamp) {
	var mod int
	switch {
	case diff > 300:
		mod = 1000
	case diff > 60:
		mod = 100
	default:
		mod = 10
	}
	if total%mod == 0 {
		logger.Info("Queued %d ops.  backlog: %d, last ts: %s, (%d sec behind)", total, count, ts, diff)
	}
}

func quitting(chsig chan os.Signal) bool {
	select {
	case <-chsig:
		return true
	default:
		return false
	}
}

// some commands we don't want to replicate over, syncing over a copydb, dropDatabase or replSetINitiate seems like a bad idea
var blacklistedCommands = []string{"dropDatabase", "copydb", "replSetFreeze", "replSetInitiate", "replSetMaintenance", "replSetReconfig", "replSetStepDown", "replSetSyncFrom", "resync"}

func isBlacklistedCommand(doc OplogDoc) (bool, string) {
	var ok bool
	for _, v := range blacklistedCommands {
		_, ok = doc.O[v]
		if ok {
			logger.Debug("%+v", doc.O)
			return true, v
		}
	}
	return false, ""
}
<|endoftext|>


<commit_before>package main

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

type logReplayer struct {
	source *mgo.Session
	target Target
	from   timestamp
	to     timestamp
	srcDB  string
	dstDB  string
}

func newLogReplayer(source *mgo.Session, target Target, from, to timestamp, srcdb, dstdb string) (*logReplayer, error) {
	if err := target.KeepAlive(); err != nil {
		return nil, err
	}

	return &logReplayer{
		source: source,
		target: target,
		from:   from,
		to:     to,
		srcDB:  srcdb,
		dstDB:  dstdb,
	}, nil
}

func (l *logReplayer) Close() {
	l.source.Close()
	l.target.Close()
}

// this is where most of the magic happens
// we tail the oplog, collect ops into a buffer
// and flush the buffer to the destination when it's full
// we take a timeout to breath every once in a while, and ping the
// destination, reconnect if necasary
// and re-execute the oplog tail if it becomes stale
func (l *logReplayer) playLog() (err error) {

	var (
		lastTimestamp     = bson.MongoTimestamp(l.from) // set his is a de
		currentOp         = OplogDoc{}
		sourceOplog       = l.source.DB("local").C(*oplog_name)
		ch                = make(chan OplogDoc, BUFFER_SIZE)
		cherr             = make(chan error, BUFFER_SIZE)
		chsig             = make(chan os.Signal, 1)
		errorCount    int = 0
		count         int = 0
		total         int = 0
	)

	signal.Notify(chsig, os.Interrupt, syscall.SIGTERM)

	// this runner will pull the incoming ops off the channel and apply them
	go func(ch chan OplogDoc, cherr chan error) {
		for op := range ch {
			err := l.target.ApplyOne(op)
			if err != nil {
				logger.Debug("ApplyOp got an error, retrying in 10 seconds")
				l.reconnectDest()
				err = l.target.ApplyOne(op)
			}
			cherr <- err
		}
	}(ch, cherr)

	// set up initial query to the oplog
	logger.Info("Replaying oplog from %s to %s", l.from, l.to)
	iter := sourceOplog.Find(bson.M{"ts": bson.M{"$gt": bson.MongoTimestamp(l.from)}}).Sort("$natural").Tail(1 * time.Second)

	// loop over the oplog
outer:
	for {
		for iter.Next(&currentOp) {

			errorCount = 0
			lastTimestamp = currentOp.Ts

			// are we quitting? did we catch a signal?
			if quitting(chsig) {
				logger.Info("Recieved Sigint, quitting")
				break outer
			}

			// if there are replies in queue, then drain them
			more := true
			for more {
				select { // suck any responses off the result channel
				case err = <-cherr:
					count--
					if err != nil {
						logger.Debug("bailing out.  Error %v.", err)
						break outer
					}
				default:
					more = false
				}
			}

			if currentOp.Kind() == NOOP {
				logger.Finest("noop, skipping %s", currentOp.String())
				continue
			}

			if currentOp.Kind() == COMMAND {
				blacklisted, comm := isBlacklistedCommand(currentOp)
				if blacklisted {
					logger.Debug("blacklisted command %s, skipping", comm)
					continue
				}
			}

			// match our namespace, or skip this op
			if *allDbs || strings.HasPrefix(currentOp.Ns, l.srcDB) {

				currentOp.Ns = strings.Replace(currentOp.Ns, l.srcDB, l.dstDB, 1)
				if currentOp.Kind() == COMMAND || (currentOp.isSystemCollection() && currentOp.Kind() == INSERT) {
					ns, exists := currentOp.O["ns"]
					if exists {
						currentOp.O["ns"] = strings.Replace(ns.(string), l.srcDB, l.dstDB, 1)
					}
				}

				count++
				total++
				ch <- currentOp

				dumpProgress(total, count, timestamp(currentOp.Ts).diff(), timestamp(currentOp.Ts)) // log progress

				// we're as far as we want to go, time to clean up and go
				if timestamp(currentOp.Ts) >= l.to {
					break outer
				}
				currentOp = OplogDoc{}
			} else {
				logger.Finest("skipping op %s", currentOp.String())
			}
		}
		// check to see if we're expired, i.e. if now is > our specified end point
		now := NewTimestamp("now")
		if now >= l.to {
			iter.Close()
			break outer
		}

		// are we quitting? did we catch a signal?
		if quitting(chsig) {
			logger.Info("Recieved Sigint, quitting")
			break outer
		}

		if iter.Timeout() {
			// we can continue on the same connection without requerying.
			logger.Debug("No data for 1 second")
			continue
		}

		logger.Debug("Error reading from source, retrying in 10 seconds")
		// take a break, ping the destination mongo to make sure all is well
		l.reconnectDest()

		// re-query
		errorCount++
		if errorCount > 10 {
			logger.Critical("too many errors reading from oplog, bailing.")
			os.Exit(1)
		}
		iter = sourceOplog.Find(bson.M{"ts": bson.M{"$gt": lastTimestamp}}).Sort("$natural").Tail(1 * time.Second)
	}

	for count > 0 {
		err = <-cherr
		count--
		if err != nil {
			logger.Debug("apply op got an error %v", err)
		}
	}

	return nil
}

// reconnect to the destination
func (l *logReplayer) reconnectDest() {
	time.Sleep(10 * time.Second)
	if err := l.target.KeepAlive(); err != nil {
		logger.Critical("Cannot reconnect to destination, quitting, %v", err)
		os.Exit(1)
	}
}

// dump log progress, depending on how far away from the target we are
func dumpProgress(total, count int, diff int64, ts timestamp) {
	var mod int
	switch {
	case diff > 300:
		mod = 1000
	case diff > 60:
		mod = 100
	default:
		mod = 10
	}
	if total%mod == 0 {
		logger.Info("Queued %d ops.  backlog: %d, last ts: %s, (%d sec behind)", total, count, ts, diff)
	}
}

func quitting(chsig chan os.Signal) bool {
	select {
	case <-chsig:
		return true
	default:
		return false
	}
}

// some commands we don't want to replicate over, syncing over a copydb, dropDatabase or replSetINitiate seems like a bad idea
var blacklistedCommands = []string{"dropDatabase", "copydb", "replSetFreeze", "replSetInitiate", "replSetMaintenance", "replSetReconfig", "replSetStepDown", "replSetSyncFrom", "resync"}

func isBlacklistedCommand(doc OplogDoc) (bool, string) {
	var ok bool
	for _, v := range blacklistedCommands {
		_, ok = doc.O[v]
		if ok {
			logger.Debug("%+v", doc.O)
			return true, v
		}
	}
	return false, ""
}
<commit_msg>add LogReplay open to the cursor.  fixes #3, thanks to Bowbaq for the suggestion<commit_after>package main

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

type logReplayer struct {
	source *mgo.Session
	target Target
	from   timestamp
	to     timestamp
	srcDB  string
	dstDB  string
}

func newLogReplayer(source *mgo.Session, target Target, from, to timestamp, srcdb, dstdb string) (*logReplayer, error) {
	if err := target.KeepAlive(); err != nil {
		return nil, err
	}

	return &logReplayer{
		source: source,
		target: target,
		from:   from,
		to:     to,
		srcDB:  srcdb,
		dstDB:  dstdb,
	}, nil
}

func (l *logReplayer) Close() {
	l.source.Close()
	l.target.Close()
}

// this is where most of the magic happens
// we tail the oplog, collect ops into a buffer
// and flush the buffer to the destination when it's full
// we take a timeout to breath every once in a while, and ping the
// destination, reconnect if necasary
// and re-execute the oplog tail if it becomes stale
func (l *logReplayer) playLog() (err error) {

	var (
		lastTimestamp     = bson.MongoTimestamp(l.from) // set his is a de
		currentOp         = OplogDoc{}
		sourceOplog       = l.source.DB("local").C(*oplog_name)
		ch                = make(chan OplogDoc, BUFFER_SIZE)
		cherr             = make(chan error, BUFFER_SIZE)
		chsig             = make(chan os.Signal, 1)
		errorCount    int = 0
		count         int = 0
		total         int = 0
	)

	signal.Notify(chsig, os.Interrupt, syscall.SIGTERM)

	// this runner will pull the incoming ops off the channel and apply them
	go func(ch chan OplogDoc, cherr chan error) {
		for op := range ch {
			err := l.target.ApplyOne(op)
			if err != nil {
				logger.Debug("ApplyOp got an error, retrying in 10 seconds")
				l.reconnectDest()
				err = l.target.ApplyOne(op)
			}
			cherr <- err
		}
	}(ch, cherr)

	// set up initial query to the oplog
	logger.Info("Replaying oplog from %s to %s", l.from, l.to)
	iter := sourceOplog.Find(bson.M{"ts": bson.M{"$gt": bson.MongoTimestamp(l.from)}}).LogReplay().Sort("$natural").Tail(1 * time.Second)

	// loop over the oplog
outer:
	for {
		for iter.Next(&currentOp) {

			errorCount = 0
			lastTimestamp = currentOp.Ts

			// are we quitting? did we catch a signal?
			if quitting(chsig) {
				logger.Info("Recieved Sigint, quitting")
				break outer
			}

			// if there are replies in queue, then drain them
			more := true
			for more {
				select { // suck any responses off the result channel
				case err = <-cherr:
					count--
					if err != nil {
						logger.Debug("bailing out.  Error %v.", err)
						break outer
					}
				default:
					more = false
				}
			}

			if currentOp.Kind() == NOOP {
				logger.Finest("noop, skipping %s", currentOp.String())
				continue
			}

			if currentOp.Kind() == COMMAND {
				blacklisted, comm := isBlacklistedCommand(currentOp)
				if blacklisted {
					logger.Debug("blacklisted command %s, skipping", comm)
					continue
				}
			}

			// match our namespace, or skip this op
			if *allDbs || strings.HasPrefix(currentOp.Ns, l.srcDB) {

				currentOp.Ns = strings.Replace(currentOp.Ns, l.srcDB, l.dstDB, 1)
				if currentOp.Kind() == COMMAND || (currentOp.isSystemCollection() && currentOp.Kind() == INSERT) {
					ns, exists := currentOp.O["ns"]
					if exists {
						currentOp.O["ns"] = strings.Replace(ns.(string), l.srcDB, l.dstDB, 1)
					}
				}

				count++
				total++
				ch <- currentOp

				dumpProgress(total, count, timestamp(currentOp.Ts).diff(), timestamp(currentOp.Ts)) // log progress

				// we're as far as we want to go, time to clean up and go
				if timestamp(currentOp.Ts) >= l.to {
					break outer
				}
				currentOp = OplogDoc{}
			} else {
				logger.Finest("skipping op %s", currentOp.String())
			}
		}
		// check to see if we're expired, i.e. if now is > our specified end point
		now := NewTimestamp("now")
		if now >= l.to {
			iter.Close()
			break outer
		}

		// are we quitting? did we catch a signal?
		if quitting(chsig) {
			logger.Info("Recieved Sigint, quitting")
			break outer
		}

		if iter.Timeout() {
			// we can continue on the same connection without requerying.
			logger.Debug("No data for 1 second")
			continue
		}

		logger.Debug("Error reading from source, retrying in 10 seconds")
		// take a break, ping the destination mongo to make sure all is well
		l.reconnectDest()

		// re-query
		errorCount++
		if errorCount > 10 {
			logger.Critical("too many errors reading from oplog, bailing.")
			os.Exit(1)
		}
		iter = sourceOplog.Find(bson.M{"ts": bson.M{"$gt": lastTimestamp}}).LogReplay().Sort("$natural").Tail(1 * time.Second)
	}

	for count > 0 {
		err = <-cherr
		count--
		if err != nil {
			logger.Debug("apply op got an error %v", err)
		}
	}

	return nil
}

// reconnect to the destination
func (l *logReplayer) reconnectDest() {
	time.Sleep(10 * time.Second)
	if err := l.target.KeepAlive(); err != nil {
		logger.Critical("Cannot reconnect to destination, quitting, %v", err)
		os.Exit(1)
	}
}

// dump log progress, depending on how far away from the target we are
func dumpProgress(total, count int, diff int64, ts timestamp) {
	var mod int
	switch {
	case diff > 300:
		mod = 1000
	case diff > 60:
		mod = 100
	default:
		mod = 10
	}
	if total%mod == 0 {
		logger.Info("Queued %d ops.  backlog: %d, last ts: %s, (%d sec behind)", total, count, ts, diff)
	}
}

func quitting(chsig chan os.Signal) bool {
	select {
	case <-chsig:
		return true
	default:
		return false
	}
}

// some commands we don't want to replicate over, syncing over a copydb, dropDatabase or replSetINitiate seems like a bad idea
var blacklistedCommands = []string{"dropDatabase", "copydb", "replSetFreeze", "replSetInitiate", "replSetMaintenance", "replSetReconfig", "replSetStepDown", "replSetSyncFrom", "resync"}

func isBlacklistedCommand(doc OplogDoc) (bool, string) {
	var ok bool
	for _, v := range blacklistedCommands {
		_, ok = doc.O[v]
		if ok {
			logger.Debug("%+v", doc.O)
			return true, v
		}
	}
	return false, ""
}
<|endoftext|>