<commit_before>// +build ignore

package main

import (
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/qor/qor-example/app/models"
	"github.com/qor/qor-example/db"
	"github.com/qor/qor-example/db/seeds"
	"github.com/qor/qor/admin"
	"github.com/qor/qor/publish"
	"github.com/qor/slug"
)

var (
	fake           = seeds.Fake
	truncateTables = seeds.TruncateTables

	Seeds  = seeds.Seeds
	Tables = []interface{}{
		&models.User{}, &models.Address{},
		&models.Category{}, &models.Color{}, &models.Size{},
		&models.Product{}, &models.ColorVariation{}, &models.ColorVariationImage{}, &models.SizeVariation{},
		&models.Store{},
		&models.Order{}, &models.OrderItem{},
		&models.Setting{},
		&models.Newsletter{},

		&admin.AssetManager{},
		&publish.PublishEvent{},
	}
)

func main() {
	truncateTables(Tables...)
	createRecords()
}

func createRecords() {
	fmt.Println("Start create sample data...")

	createSetting()
	fmt.Println("--> Created setting.")

	createUsers()
	fmt.Println("--> Created users.")
	createAddresses()
	fmt.Println("--> Created addresses.")
	createNewsletters()
	fmt.Println("--> Created newsletters.")

	createCategories()
	fmt.Println("--> Created categories.")
	createColors()
	fmt.Println("--> Created colors.")
	createSizes()
	fmt.Println("--> Created sizes.")
	createProducts()
	fmt.Println("--> Created products.")
	createStores()
	fmt.Println("--> Created stores.")

	createOrders()
	fmt.Println("--> Created orders.")

	fmt.Println("--> Done!")
}

func createSetting() {
	setting := models.Setting{}

	setting.ShippingFee = Seeds.Setting.ShippingFee
	setting.GiftWrappingFee = Seeds.Setting.GiftWrappingFee
	setting.CODFee = Seeds.Setting.CODFee
	setting.TaxRate = Seeds.Setting.TaxRate
	setting.Address = Seeds.Setting.Address
	setting.Region = Seeds.Setting.Region
	setting.City = Seeds.Setting.City
	setting.Country = Seeds.Setting.Country
	setting.Zip = Seeds.Setting.Zip
	setting.Latitude = Seeds.Setting.Latitude
	setting.Longitude = Seeds.Setting.Longitude

	if err := db.DB.Create(&setting).Error; err != nil {
		log.Fatalf("create setting (%v) failure, got err %v", setting, err)
	}
}

func createUsers() {
	for i := 0; i < 500; i++ {
		user := models.User{}
		user.Email = fake.Email()
		user.Name = fake.Name()
		user.Gender = []string{"Female", "Male"}[i%2]
		if err := db.DB.Create(&user).Error; err != nil {
			log.Fatalf("create user (%v) failure, got err %v", user, err)
		}

		user.CreatedAt = randTime()
		if err := db.DB.Save(&user).Error; err != nil {
			log.Fatalf("Save user (%v) failure, got err %v", user, err)
		}
	}
}

func createAddresses() {
	var users []models.User
	if err := db.DB.Find(&users).Error; err != nil {
		log.Fatalf("query users (%v) failure, got err %v", users, err)
	}

	for _, user := range users {
		address := models.Address{}
		address.UserID = user.ID
		address.ContactName = user.Name
		address.Phone = fake.PhoneNumber()
		address.City = fake.City()
		address.Address1 = fake.StreetAddress()
		address.Address2 = fmt.Sprintf("%s, %s", address.City, fake.PostCode())
		if err := db.DB.Create(&address).Error; err != nil {
			log.Fatalf("create address (%v) failure, got err %v", address, err)
		}
	}
}

func createNewsletters() {
	var users []models.User
	if err := db.DB.Find(&users).Error; err != nil {
		log.Fatalf("query users (%v) failure, got err %v", users, err)
	}

	for index, user := range users {
		newsletter := models.Newsletter{}
		newsletter.NewsletterType = []string{"Weekly", "Monthly", "Promotions"}[(rand.Intn(9)+1)%3]
		newsletter.MailType = []string{"HTML", "Text"}[(rand.Intn(9)+1)%2]
		subscribedAt := randTime()
		newsletter.SubscribedAt = &subscribedAt
		newsletter.Email = fake.Email()
		if (index % (rand.Intn(9) + 1)) <= 4 {
			newsletter.UserID = user.ID
			newsletter.Email = user.Email
			subscribedAt := user.CreatedAt.Add(time.Duration(rand.Intn(24)) * time.Hour)
			newsletter.SubscribedAt = &subscribedAt
		}

		if index&(rand.Intn(9)+1) == 0 {
			unsubscribedAt := newsletter.SubscribedAt.Add(time.Duration(rand.Intn(24)) * time.Hour)
			newsletter.UnsubscribedAt = &unsubscribedAt
		}
		if err := db.DB.Create(&newsletter).Error; err != nil {
			log.Fatalf("create newsletter (%v) failure, got err %v", newsletter, err)
		}
	}
}

func createCategories() {
	for _, c := range Seeds.Categories {
		category := models.Category{}
		category.Name = c.Name
		if err := db.DB.Create(&category).Error; err != nil {
			log.Fatalf("create category (%v) failure, got err %v", category, err)
		}
	}
}

func createColors() {
	for _, c := range Seeds.Colors {
		color := models.Color{}
		color.Name = c.Name
		color.Code = c.Code
		if err := db.DB.Create(&color).Error; err != nil {
			log.Fatalf("create color (%v) failure, got err %v", color, err)
		}
	}
}

func createSizes() {
	for _, s := range Seeds.Sizes {
		size := models.Size{}
		size.Name = s.Name
		size.Code = s.Code
		if err := db.DB.Create(&size).Error; err != nil {
			log.Fatalf("create size (%v) failure, got err %v", size, err)
		}
	}
}

func createProducts() {
	for _, p := range Seeds.Products {
		category := findCategoryByName(p.CategoryName)

		product := models.Product{}
		product.CategoryID = category.ID
		product.Name = p.Name
		product.NameWithSlug = slug.Slug{p.NameWithSlug}
		product.Code = p.Code
		product.Price = p.Price
		product.Description = p.Description
		product.MadeCountry = p.MadeCountry

		if err := db.DB.Create(&product).Error; err != nil {
			log.Fatalf("create product (%v) failure, got err %v", product, err)
		}

		for _, cv := range p.ColorVariations {
			color := findColorByName(cv.ColorName)

			colorVariation := models.ColorVariation{}
			colorVariation.ProductID = product.ID
			colorVariation.ColorID = color.ID
			if err := db.DB.Create(&colorVariation).Error; err != nil {
				log.Fatalf("create color_variation (%v) failure, got err %v", colorVariation, err)
			}

			for _, i := range cv.Images {
				image := models.ColorVariationImage{}
				if file, err := openFileByURL(i.URL); err != nil {
					fmt.Printf("open file (%q) failure, got err %v", i.URL, err)
				} else {
					defer file.Close()
					image.Image.Scan(file)
				}
				image.ColorVariationID = colorVariation.ID
				if err := db.DB.Create(&image).Error; err != nil {
					log.Fatalf("create color_variation_image (%v) failure, got err %v", image, err)
				}
			}

			for _, sv := range p.SizeVariations {
				size := findSizeByName(sv.SizeName)

				sizeVariation := models.SizeVariation{}
				sizeVariation.ColorVariationID = colorVariation.ID
				sizeVariation.SizeID = size.ID
				sizeVariation.AvailableQuantity = 20
				if err := db.DB.Create(&sizeVariation).Error; err != nil {
					log.Fatalf("create size_variation (%v) failure, got err %v", sizeVariation, err)
				}
			}
		}
	}
}

func createStores() {
	for _, s := range Seeds.Stores {
		store := models.Store{}
		store.Name = s.Name
		store.Phone = s.Phone
		store.Email = s.Email
		store.Country = s.Country
		store.City = s.City
		store.Region = s.Region
		store.Address = s.Address
		store.Zip = s.Zip
		store.Latitude = s.Latitude
		store.Longitude = s.Longitude
		if err := db.DB.Create(&store).Error; err != nil {
			log.Fatalf("create store (%v) failure, got err %v", store, err)
		}
	}
}

func createOrders() {
	var users []models.User
	if err := db.DB.Limit(480).Preload("Addresses").Find(&users).Error; err != nil {
		log.Fatalf("query users (%v) failure, got err %v", users, err)
	}

	var sizeVariations []models.SizeVariation
	if err := db.DB.Find(&sizeVariations).Error; err != nil {
		log.Fatalf("query sizeVariations (%v) failure, got err %v", sizeVariations, err)
	}
	var sizeVariationsCount = len(sizeVariations)

	for i, user := range users {
		order := models.Order{}
		state := []string{"draft", "checkout", "cancelled", "paid", "paid_cancelled", "processing", "shipped", "returned"}[rand.Intn(10)%8]
		abandonedReason := []string{
			"Doesn't complete payment flow.",
			"Payment failure due to using an invalid credit card.",
			"Invalid shipping address.",
			"Invalid contact information.",
			"Doesn't complete checkout flow.",
		}[rand.Intn(10)%5]

		order.UserID = user.ID
		order.ShippingAddressID = user.Addresses[0].ID
		order.BillingAddressID = user.Addresses[0].ID
		order.State = state
		if rand.Intn(15)%15 == 3 && state == "checkout" || state == "processing" || state == "paid_cancelled" {
			order.AbandonedReason = abandonedReason
		}
		if err := db.DB.Create(&order).Error; err != nil {
			log.Fatalf("create order (%v) failure, got err %v", order, err)
		}

		sizeVariation := sizeVariations[i%sizeVariationsCount]
		product := findProductByColorVariationID(sizeVariation.ColorVariationID)
		quantity := []uint{1, 2, 3, 4, 5}[rand.Intn(10)%5]
		discountRate := []uint{0, 5, 10, 15, 20, 25}[rand.Intn(10)%6]

		orderItem := models.OrderItem{}
		orderItem.OrderID = order.ID
		orderItem.SizeVariationID = sizeVariation.ID
		orderItem.Quantity = quantity
		orderItem.Price = product.Price
		orderItem.DiscountRate = discountRate
		if err := db.DB.Create(&orderItem).Error; err != nil {
			log.Fatalf("create orderItem (%v) failure, got err %v", orderItem, err)
		}

		order.OrderItems = append(order.OrderItems, orderItem)
		order.CreatedAt = user.CreatedAt.Add(time.Duration(rand.Intn(24)) * time.Hour)
		order.PaymentAmount = order.Amount()
		if err := db.DB.Save(&order).Error; err != nil {
			log.Fatalf("Save order (%v) failure, got err %v", order, err)
		}
	}
}

func findCategoryByName(name string) *models.Category {
	category := &models.Category{}
	if err := db.DB.Where(&models.Category{Name: name}).First(category).Error; err != nil {
		log.Fatalf("can't find category with name = %q, got err %v", name, err)
	}
	return category
}

func findColorByName(name string) *models.Color {
	color := &models.Color{}
	if err := db.DB.Where(&models.Color{Name: name}).First(color).Error; err != nil {
		log.Fatalf("can't find color with name = %q, got err %v", name, err)
	}
	return color
}

func findSizeByName(name string) *models.Size {
	size := &models.Size{}
	if err := db.DB.Where(&models.Size{Name: name}).First(size).Error; err != nil {
		log.Fatalf("can't find size with name = %q, got err %v", name, err)
	}
	return size
}

func findProductByColorVariationID(colorVariationID uint) *models.Product {
	colorVariation := models.ColorVariation{}
	product := models.Product{}

	if err := db.DB.Find(&colorVariation, colorVariationID).Error; err != nil {
		log.Fatalf("query colorVariation (%v) failure, got err %v", colorVariation, err)
		return &product
	}
	if err := db.DB.Find(&product, colorVariation.ProductID).Error; err != nil {
		log.Fatalf("query product (%v) failure, got err %v", product, err)
		return &product
	}
	return &product
}

func randTime() time.Time {
	return time.Now().Add((time.Duration(-rand.Intn(7*24)) * time.Hour))
}

func openFileByURL(rawURL string) (*os.File, error) {
	if fileURL, err := url.Parse(rawURL); err != nil {
		return nil, err
	} else {
		path := fileURL.Path
		segments := strings.Split(path, "/")
		fileName := segments[len(segments)-1]

		basePath, _ := filepath.Abs(".")
		filePath := fmt.Sprintf("%s/tmp/%s", basePath, fileName)

		if _, err := os.Stat(filePath); err == nil {
			return os.Open(filePath)
		}

		file, err := os.Create(filePath)
		if err != nil {
			return file, err
		}

		check := http.Client{
			CheckRedirect: func(r *http.Request, via []*http.Request) error {
				r.URL.Opaque = r.URL.Path
				return nil
			},
		}
		resp, err := check.Get(rawURL) // add a filter to check redirect
		if err != nil {
			return file, err
		}
		defer resp.Body.Close()
		fmt.Printf("----> Downloaded %v\n", rawURL)

		_, err = io.Copy(file, resp.Body)
		if err != nil {
			return file, err
		}
		return file, nil
	}
}
<commit_msg>Generate two weeks of dummy orders/users data<commit_after>// +build ignore

package main

import (
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/jinzhu/now"
	"github.com/qor/qor-example/app/models"
	"github.com/qor/qor-example/db"
	"github.com/qor/qor-example/db/seeds"
	"github.com/qor/qor/admin"
	"github.com/qor/qor/publish"
	"github.com/qor/slug"
)

var (
	fake           = seeds.Fake
	truncateTables = seeds.TruncateTables

	Seeds  = seeds.Seeds
	Tables = []interface{}{
		&models.User{}, &models.Address{},
		&models.Category{}, &models.Color{}, &models.Size{},
		&models.Product{}, &models.ColorVariation{}, &models.ColorVariationImage{}, &models.SizeVariation{},
		&models.Store{},
		&models.Order{}, &models.OrderItem{},
		&models.Setting{},
		&models.Newsletter{},

		&admin.AssetManager{},
		&publish.PublishEvent{},
	}
)

func main() {
	truncateTables(Tables...)
	createRecords()
}

func createRecords() {
	fmt.Println("Start create sample data...")

	createSetting()
	fmt.Println("--> Created setting.")

	createUsers()
	fmt.Println("--> Created users.")
	createAddresses()
	fmt.Println("--> Created addresses.")
	createNewsletters()
	fmt.Println("--> Created newsletters.")

	createCategories()
	fmt.Println("--> Created categories.")
	createColors()
	fmt.Println("--> Created colors.")
	createSizes()
	fmt.Println("--> Created sizes.")
	createProducts()
	fmt.Println("--> Created products.")
	createStores()
	fmt.Println("--> Created stores.")

	createOrders()
	fmt.Println("--> Created orders.")

	fmt.Println("--> Done!")
}

func createSetting() {
	setting := models.Setting{}

	setting.ShippingFee = Seeds.Setting.ShippingFee
	setting.GiftWrappingFee = Seeds.Setting.GiftWrappingFee
	setting.CODFee = Seeds.Setting.CODFee
	setting.TaxRate = Seeds.Setting.TaxRate
	setting.Address = Seeds.Setting.Address
	setting.Region = Seeds.Setting.Region
	setting.City = Seeds.Setting.City
	setting.Country = Seeds.Setting.Country
	setting.Zip = Seeds.Setting.Zip
	setting.Latitude = Seeds.Setting.Latitude
	setting.Longitude = Seeds.Setting.Longitude

	if err := db.DB.Create(&setting).Error; err != nil {
		log.Fatalf("create setting (%v) failure, got err %v", setting, err)
	}
}

func createUsers() {
	totalCount := 600
	for i := 0; i < totalCount; i++ {
		user := models.User{}
		user.Email = fake.Email()
		user.Name = fake.Name()
		user.Gender = []string{"Female", "Male"}[i%2]
		if err := db.DB.Create(&user).Error; err != nil {
			log.Fatalf("create user (%v) failure, got err %v", user, err)
		}

		day := (-14 + i/45)
		user.CreatedAt = now.EndOfDay().Add(time.Duration(day*rand.Intn(24)) * time.Hour)
		if user.CreatedAt.After(time.Now()) {
			user.CreatedAt = time.Now()
		}
		if err := db.DB.Save(&user).Error; err != nil {
			log.Fatalf("Save user (%v) failure, got err %v", user, err)
		}
	}
}

func createAddresses() {
	var users []models.User
	if err := db.DB.Find(&users).Error; err != nil {
		log.Fatalf("query users (%v) failure, got err %v", users, err)
	}

	for _, user := range users {
		address := models.Address{}
		address.UserID = user.ID
		address.ContactName = user.Name
		address.Phone = fake.PhoneNumber()
		address.City = fake.City()
		address.Address1 = fake.StreetAddress()
		address.Address2 = fmt.Sprintf("%s, %s", address.City, fake.PostCode())
		if err := db.DB.Create(&address).Error; err != nil {
			log.Fatalf("create address (%v) failure, got err %v", address, err)
		}
	}
}

func createNewsletters() {
	var users []models.User
	if err := db.DB.Find(&users).Error; err != nil {
		log.Fatalf("query users (%v) failure, got err %v", users, err)
	}

	for index, user := range users {
		newsletter := models.Newsletter{}
		newsletter.NewsletterType = []string{"Weekly", "Monthly", "Promotions"}[(rand.Intn(9)+1)%3]
		newsletter.MailType = []string{"HTML", "Text"}[(rand.Intn(9)+1)%2]
		subscribedAt := randTime()
		newsletter.SubscribedAt = &subscribedAt
		newsletter.Email = fake.Email()
		if (index % (rand.Intn(9) + 1)) <= 4 {
			newsletter.UserID = user.ID
			newsletter.Email = user.Email
			subscribedAt := user.CreatedAt.Add(time.Duration(rand.Intn(24)) * time.Hour)
			newsletter.SubscribedAt = &subscribedAt
		}

		if index&(rand.Intn(9)+1) == 0 {
			unsubscribedAt := newsletter.SubscribedAt.Add(time.Duration(rand.Intn(24)) * time.Hour)
			newsletter.UnsubscribedAt = &unsubscribedAt
		}
		if err := db.DB.Create(&newsletter).Error; err != nil {
			log.Fatalf("create newsletter (%v) failure, got err %v", newsletter, err)
		}
	}
}

func createCategories() {
	for _, c := range Seeds.Categories {
		category := models.Category{}
		category.Name = c.Name
		if err := db.DB.Create(&category).Error; err != nil {
			log.Fatalf("create category (%v) failure, got err %v", category, err)
		}
	}
}

func createColors() {
	for _, c := range Seeds.Colors {
		color := models.Color{}
		color.Name = c.Name
		color.Code = c.Code
		if err := db.DB.Create(&color).Error; err != nil {
			log.Fatalf("create color (%v) failure, got err %v", color, err)
		}
	}
}

func createSizes() {
	for _, s := range Seeds.Sizes {
		size := models.Size{}
		size.Name = s.Name
		size.Code = s.Code
		if err := db.DB.Create(&size).Error; err != nil {
			log.Fatalf("create size (%v) failure, got err %v", size, err)
		}
	}
}

func createProducts() {
	for _, p := range Seeds.Products {
		category := findCategoryByName(p.CategoryName)

		product := models.Product{}
		product.CategoryID = category.ID
		product.Name = p.Name
		product.NameWithSlug = slug.Slug{p.NameWithSlug}
		product.Code = p.Code
		product.Price = p.Price
		product.Description = p.Description
		product.MadeCountry = p.MadeCountry

		if err := db.DB.Create(&product).Error; err != nil {
			log.Fatalf("create product (%v) failure, got err %v", product, err)
		}

		for _, cv := range p.ColorVariations {
			color := findColorByName(cv.ColorName)

			colorVariation := models.ColorVariation{}
			colorVariation.ProductID = product.ID
			colorVariation.ColorID = color.ID
			if err := db.DB.Create(&colorVariation).Error; err != nil {
				log.Fatalf("create color_variation (%v) failure, got err %v", colorVariation, err)
			}

			for _, i := range cv.Images {
				image := models.ColorVariationImage{}
				if file, err := openFileByURL(i.URL); err != nil {
					fmt.Printf("open file (%q) failure, got err %v", i.URL, err)
				} else {
					defer file.Close()
					image.Image.Scan(file)
				}
				image.ColorVariationID = colorVariation.ID
				if err := db.DB.Create(&image).Error; err != nil {
					log.Fatalf("create color_variation_image (%v) failure, got err %v", image, err)
				}
			}

			for _, sv := range p.SizeVariations {
				size := findSizeByName(sv.SizeName)

				sizeVariation := models.SizeVariation{}
				sizeVariation.ColorVariationID = colorVariation.ID
				sizeVariation.SizeID = size.ID
				sizeVariation.AvailableQuantity = 20
				if err := db.DB.Create(&sizeVariation).Error; err != nil {
					log.Fatalf("create size_variation (%v) failure, got err %v", sizeVariation, err)
				}
			}
		}
	}
}

func createStores() {
	for _, s := range Seeds.Stores {
		store := models.Store{}
		store.Name = s.Name
		store.Phone = s.Phone
		store.Email = s.Email
		store.Country = s.Country
		store.City = s.City
		store.Region = s.Region
		store.Address = s.Address
		store.Zip = s.Zip
		store.Latitude = s.Latitude
		store.Longitude = s.Longitude
		if err := db.DB.Create(&store).Error; err != nil {
			log.Fatalf("create store (%v) failure, got err %v", store, err)
		}
	}
}

func createOrders() {
	var users []models.User
	if err := db.DB.Preload("Addresses").Find(&users).Error; err != nil {
		log.Fatalf("query users (%v) failure, got err %v", users, err)
	}

	var sizeVariations []models.SizeVariation
	if err := db.DB.Find(&sizeVariations).Error; err != nil {
		log.Fatalf("query sizeVariations (%v) failure, got err %v", sizeVariations, err)
	}
	var sizeVariationsCount = len(sizeVariations)

	for i, user := range users {
		order := models.Order{}
		state := []string{"draft", "checkout", "cancelled", "paid", "paid_cancelled", "processing", "shipped", "returned"}[rand.Intn(10)%8]
		abandonedReason := []string{
			"Doesn't complete payment flow.",
			"Payment failure due to using an invalid credit card.",
			"Invalid shipping address.",
			"Invalid contact information.",
			"Doesn't complete checkout flow.",
		}[rand.Intn(10)%5]

		order.UserID = user.ID
		order.ShippingAddressID = user.Addresses[0].ID
		order.BillingAddressID = user.Addresses[0].ID
		order.State = state
		if rand.Intn(15)%15 == 3 && state == "checkout" || state == "processing" || state == "paid_cancelled" {
			order.AbandonedReason = abandonedReason
		}
		if err := db.DB.Create(&order).Error; err != nil {
			log.Fatalf("create order (%v) failure, got err %v", order, err)
		}

		sizeVariation := sizeVariations[i%sizeVariationsCount]
		product := findProductByColorVariationID(sizeVariation.ColorVariationID)
		quantity := []uint{1, 2, 3, 4, 5}[rand.Intn(10)%5]
		discountRate := []uint{0, 5, 10, 15, 20, 25}[rand.Intn(10)%6]

		orderItem := models.OrderItem{}
		orderItem.OrderID = order.ID
		orderItem.SizeVariationID = sizeVariation.ID
		orderItem.Quantity = quantity
		orderItem.Price = product.Price
		orderItem.DiscountRate = discountRate
		if err := db.DB.Create(&orderItem).Error; err != nil {
			log.Fatalf("create orderItem (%v) failure, got err %v", orderItem, err)
		}

		order.OrderItems = append(order.OrderItems, orderItem)
		order.CreatedAt = user.CreatedAt.Add(1 * time.Hour)
		order.PaymentAmount = order.Amount()
		if err := db.DB.Save(&order).Error; err != nil {
			log.Fatalf("Save order (%v) failure, got err %v", order, err)
		}
	}
}

func findCategoryByName(name string) *models.Category {
	category := &models.Category{}
	if err := db.DB.Where(&models.Category{Name: name}).First(category).Error; err != nil {
		log.Fatalf("can't find category with name = %q, got err %v", name, err)
	}
	return category
}

func findColorByName(name string) *models.Color {
	color := &models.Color{}
	if err := db.DB.Where(&models.Color{Name: name}).First(color).Error; err != nil {
		log.Fatalf("can't find color with name = %q, got err %v", name, err)
	}
	return color
}

func findSizeByName(name string) *models.Size {
	size := &models.Size{}
	if err := db.DB.Where(&models.Size{Name: name}).First(size).Error; err != nil {
		log.Fatalf("can't find size with name = %q, got err %v", name, err)
	}
	return size
}

func findProductByColorVariationID(colorVariationID uint) *models.Product {
	colorVariation := models.ColorVariation{}
	product := models.Product{}

	if err := db.DB.Find(&colorVariation, colorVariationID).Error; err != nil {
		log.Fatalf("query colorVariation (%v) failure, got err %v", colorVariation, err)
		return &product
	}
	if err := db.DB.Find(&product, colorVariation.ProductID).Error; err != nil {
		log.Fatalf("query product (%v) failure, got err %v", product, err)
		return &product
	}
	return &product
}

func randTime() time.Time {
	num := rand.Intn(10)
	return time.Now().Add(-time.Duration(num*24) * time.Hour)
}

func openFileByURL(rawURL string) (*os.File, error) {
	if fileURL, err := url.Parse(rawURL); err != nil {
		return nil, err
	} else {
		path := fileURL.Path
		segments := strings.Split(path, "/")
		fileName := segments[len(segments)-1]

		basePath, _ := filepath.Abs(".")
		filePath := fmt.Sprintf("%s/tmp/%s", basePath, fileName)

		if _, err := os.Stat(filePath); err == nil {
			return os.Open(filePath)
		}

		file, err := os.Create(filePath)
		if err != nil {
			return file, err
		}

		check := http.Client{
			CheckRedirect: func(r *http.Request, via []*http.Request) error {
				r.URL.Opaque = r.URL.Path
				return nil
			},
		}
		resp, err := check.Get(rawURL) // add a filter to check redirect
		if err != nil {
			return file, err
		}
		defer resp.Body.Close()
		fmt.Printf("----> Downloaded %v\n", rawURL)

		_, err = io.Copy(file, resp.Body)
		if err != nil {
			return file, err
		}
		return file, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"restic"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"restic/archiver"
	"restic/debug"
	"restic/errors"
	"restic/filter"
	"restic/fs"
)

var cmdBackup = &cobra.Command{
	Use:   "backup [flags] FILE/DIR [FILE/DIR] ...",
	Short: "create a new backup of files and/or directories",
	Long: `
The "backup" command creates a new snapshot and saves the files and directories
given as the arguments.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
		}

		if backupOptions.Stdin {
			return readBackupFromStdin(backupOptions, globalOptions, args)
		}

		return runBackup(backupOptions, globalOptions, args)
	},
}

// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
	Parent         string
	Force          bool
	Excludes       []string
	ExcludeFile    string
	ExcludeOtherFS bool
	Stdin          bool
	StdinFilename  string
	Tags           []string
	Hostname       string
	FilesFrom      string
}

var backupOptions BackupOptions

func init() {
	cmdRoot.AddCommand(cmdBackup)

	hostname, err := os.Hostname()
	if err != nil {
		debug.Log("os.Hostname() returned err: %v", err)
		hostname = ""
	}

	f := cmdBackup.Flags()
	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
	f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
	f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
	f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}

func newScanProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	p := restic.NewProgress()
	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
	}

	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
	}

	return p
}

func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps, eta uint64
	itemsTodo := todo.Files + todo.Dirs

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if todo.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
			if s.Bytes >= todo.Bytes {
				eta = 0
			} else if bps > 0 {
				eta = (todo.Bytes - s.Bytes) / bps
			}
		}

		itemsDone := s.Files + s.Dirs

		status1 := fmt.Sprintf("[%s] %s  %s/s  %s / %s  %d / %d items  %d errors  ",
			formatDuration(d),
			formatPercent(s.Bytes, todo.Bytes),
			formatBytes(bps),
			formatBytes(s.Bytes), formatBytes(todo.Bytes),
			itemsDone, itemsTodo,
			s.Errors)
		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))

		if w := stdoutTerminalWidth(); w > 0 {
			maxlen := w - len(status2) - 1

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s%s", status1, status2)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(todo.Bytes, d))
	}

	return archiveProgress
}

func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps uint64

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if s.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
		}

		status1 := fmt.Sprintf("[%s] %s  %s/s", formatDuration(d),
			formatBytes(s.Bytes),
			formatBytes(bps))

		if w := stdoutTerminalWidth(); w > 0 {
			maxlen := w - len(status1)

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s", status1)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
	}

	return archiveProgress
}

// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string) (result []string, err error) {
	for _, item := range items {
		_, err := fs.Lstat(item)
		if err != nil && os.IsNotExist(errors.Cause(err)) {
			continue
		}

		result = append(result, item)
	}

	if len(result) == 0 {
		return nil, errors.Fatal("all target directories/files do not exist")
	}

	return
}

// gatherDevices returns the set of unique device ids of the files and/or
// directory paths listed in "items".
func gatherDevices(items []string) (deviceMap map[uint64]struct{}, err error) {
	deviceMap = make(map[uint64]struct{})
	for _, item := range items {
		fi, err := fs.Lstat(item)
		if err != nil {
			return nil, err
		}
		id, err := fs.DeviceID(fi)
		if err != nil {
			return nil, err
		}
		deviceMap[id] = struct{}{}
	}
	if len(deviceMap) == 0 {
		return nil, errors.New("zero allowed devices")
	}
	return deviceMap, nil
}

func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 0 {
		return errors.Fatal("when reading from stdin, no additional files can be specified")
	}

	if opts.StdinFilename == "" {
		return errors.Fatal("filename for backup from stdin must not be empty")
	}

	if gopts.password == "" && gopts.PasswordFile == "" {
		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	r := &archiver.Reader{
		Repository: repo,
		Tags:       opts.Tags,
		Hostname:   opts.Hostname,
	}

	_, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
	if err != nil {
		return err
	}

	Verbosef("archived as %v\n", id.Str())
	return nil
}

// readFromFile will read all lines from the given filename and write them to a
// string array, if filename is empty readFromFile returns and empty string
// array. If filename is a dash (-), readFromFile will read the lines from
// the standard input.
func readLinesFromFile(filename string) ([]string, error) {
	if filename == "" {
		return nil, nil
	}

	var r io.Reader = os.Stdin
	if filename != "-" {
		f, err := os.Open(filename)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		r = f
	}

	var lines []string

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" {
			continue
		}
		lines = append(lines, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return lines, nil
}

func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
	if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
		return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
	}

	fromfile, err := readLinesFromFile(opts.FilesFrom)
	if err != nil {
		return err
	}

	// merge files from files-from into normal args so we can reuse the normal
	// args checks and have the ability to use both files-from and args at the
	// same time
	args = append(args, fromfile...)
	if len(args) == 0 {
		return errors.Fatal("wrong number of parameters")
	}

	target := make([]string, 0, len(args))
	for _, d := range args {
		if a, err := filepath.Abs(d); err == nil {
			d = a
		}
		target = append(target, d)
	}

	target, err = filterExisting(target)
	if err != nil {
		return err
	}

	// allowed devices
	var allowedDevs map[uint64]struct{}
	if opts.ExcludeOtherFS {
		allowedDevs, err = gatherDevices(target)
		if err != nil {
			return err
		}
		debug.Log("allowed devices: %v\n", allowedDevs)
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	var parentSnapshotID *restic.ID

	// Force using a parent
	if !opts.Force && opts.Parent != "" {
		id, err := restic.FindSnapshot(repo, opts.Parent)
		if err != nil {
			return errors.Fatalf("invalid id %q: %v", opts.Parent, err)
		}

		parentSnapshotID = &id
	}

	// Find last snapshot to set it as parent, if not already set
	if !opts.Force && parentSnapshotID == nil {
		id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
		if err == nil {
			parentSnapshotID = &id
		} else if err != restic.ErrNoSnapshotFound {
			return err
		}
	}

	if parentSnapshotID != nil {
		Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
	}

	Verbosef("scan %v\n", target)

	// add patterns from file
	if opts.ExcludeFile != "" {
		file, err := fs.Open(opts.ExcludeFile)
		if err != nil {
			Warnf("error reading exclude patterns: %v", err)
			return nil
		}

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			line := scanner.Text()
			if !strings.HasPrefix(line, "#") {
				line = os.ExpandEnv(line)
				opts.Excludes = append(opts.Excludes, line)
			}
		}
	}

	selectFilter := func(item string, fi os.FileInfo) bool {
		matched, err := filter.List(opts.Excludes, item)
		if err != nil {
			Warnf("error for exclude pattern: %v", err)
		}

		if matched {
			debug.Log("path %q excluded by a filter", item)
			return false
		}

		if !opts.ExcludeOtherFS || fi == nil {
			return true
		}

		id, err := fs.DeviceID(fi)
		if err != nil {
			// This should never happen because gatherDevices() would have
			// errored out earlier. If it still does that's a reason to panic.
			panic(err)
		}
		_, found := allowedDevs[id]
		if !found {
			debug.Log("path %q on disallowed device %d", item, id)
			return false
		}

		return true
	}

	stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts))
	if err != nil {
		return err
	}

	arch := archiver.New(repo)
	arch.Excludes = opts.Excludes
	arch.SelectFilter = selectFilter

	arch.Warn = func(dir string, fi os.FileInfo, err error) {
		// TODO: make ignoring errors configurable
		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
	}

	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
	if err != nil {
		return err
	}

	Verbosef("snapshot %s saved\n", id.Str())

	return nil
}
<commit_msg>Check allowed devices per path<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"restic"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"restic/archiver"
	"restic/debug"
	"restic/errors"
	"restic/filter"
	"restic/fs"
)

var cmdBackup = &cobra.Command{
	Use:   "backup [flags] FILE/DIR [FILE/DIR] ...",
	Short: "create a new backup of files and/or directories",
	Long: `
The "backup" command creates a new snapshot and saves the files and directories
given as the arguments.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
		}

		if backupOptions.Stdin {
			return readBackupFromStdin(backupOptions, globalOptions, args)
		}

		return runBackup(backupOptions, globalOptions, args)
	},
}

// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
	Parent         string
	Force          bool
	Excludes       []string
	ExcludeFile    string
	ExcludeOtherFS bool
	Stdin          bool
	StdinFilename  string
	Tags           []string
	Hostname       string
	FilesFrom      string
}

var backupOptions BackupOptions

func init() {
	cmdRoot.AddCommand(cmdBackup)

	hostname, err := os.Hostname()
	if err != nil {
		debug.Log("os.Hostname() returned err: %v", err)
		hostname = ""
	}

	f := cmdBackup.Flags()
	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
	f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
	f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
	f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}

func newScanProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	p := restic.NewProgress()
	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
	}

	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
	}

	return p
}

func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps, eta uint64
	itemsTodo := todo.Files + todo.Dirs

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if todo.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
			if s.Bytes >= todo.Bytes {
				eta = 0
			} else if bps > 0 {
				eta = (todo.Bytes - s.Bytes) / bps
			}
		}

		itemsDone := s.Files + s.Dirs

		status1 := fmt.Sprintf("[%s] %s  %s/s  %s / %s  %d / %d items  %d errors  ",
			formatDuration(d),
			formatPercent(s.Bytes, todo.Bytes),
			formatBytes(bps),
			formatBytes(s.Bytes), formatBytes(todo.Bytes),
			itemsDone, itemsTodo,
			s.Errors)
		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))

		if w := stdoutTerminalWidth(); w > 0 {
			maxlen := w - len(status2) - 1

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s%s", status1, status2)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(todo.Bytes, d))
	}

	return archiveProgress
}

func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps uint64

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if s.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
		}

		status1 := fmt.Sprintf("[%s] %s  %s/s", formatDuration(d),
			formatBytes(s.Bytes),
			formatBytes(bps))

		if w := stdoutTerminalWidth(); w > 0 {
			maxlen := w - len(status1)

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s", status1)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
	}

	return archiveProgress
}

// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string) (result []string, err error) {
	for _, item := range items {
		_, err := fs.Lstat(item)
		if err != nil && os.IsNotExist(errors.Cause(err)) {
			continue
		}

		result = append(result, item)
	}

	if len(result) == 0 {
		return nil, errors.Fatal("all target directories/files do not exist")
	}

	return
}

// gatherDevices returns the set of unique device ids of the files and/or
// directory paths listed in "items".
func gatherDevices(items []string) (deviceMap map[string]uint64, err error) {
	deviceMap = make(map[string]uint64)
	for _, item := range items {
		fi, err := fs.Lstat(item)
		if err != nil {
			return nil, err
		}
		id, err := fs.DeviceID(fi)
		if err != nil {
			return nil, err
		}
		deviceMap[item] = id
	}
	if len(deviceMap) == 0 {
		return nil, errors.New("zero allowed devices")
	}
	return deviceMap, nil
}

func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 0 {
		return errors.Fatal("when reading from stdin, no additional files can be specified")
	}

	if opts.StdinFilename == "" {
		return errors.Fatal("filename for backup from stdin must not be empty")
	}

	if gopts.password == "" && gopts.PasswordFile == "" {
		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	r := &archiver.Reader{
		Repository: repo,
		Tags:       opts.Tags,
		Hostname:   opts.Hostname,
	}

	_, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
	if err != nil {
		return err
	}

	Verbosef("archived as %v\n", id.Str())
	return nil
}

// readFromFile will read all lines from the given filename and write them to a
// string array, if filename is empty readFromFile returns and empty string
// array. If filename is a dash (-), readFromFile will read the lines from
// the standard input.
func readLinesFromFile(filename string) ([]string, error) {
	if filename == "" {
		return nil, nil
	}

	var r io.Reader = os.Stdin
	if filename != "-" {
		f, err := os.Open(filename)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		r = f
	}

	var lines []string

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" {
			continue
		}
		lines = append(lines, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return lines, nil
}

func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
	if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
		return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
	}

	fromfile, err := readLinesFromFile(opts.FilesFrom)
	if err != nil {
		return err
	}

	// merge files from files-from into normal args so we can reuse the normal
	// args checks and have the ability to use both files-from and args at the
	// same time
	args = append(args, fromfile...)
	if len(args) == 0 {
		return errors.Fatal("wrong number of parameters")
	}

	target := make([]string, 0, len(args))
	for _, d := range args {
		if a, err := filepath.Abs(d); err == nil {
			d = a
		}
		target = append(target, d)
	}

	target, err = filterExisting(target)
	if err != nil {
		return err
	}

	// allowed devices
	var allowedDevs map[string]uint64
	if opts.ExcludeOtherFS {
		allowedDevs, err = gatherDevices(target)
		if err != nil {
			return err
		}
		debug.Log("allowed devices: %v\n", allowedDevs)
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	var parentSnapshotID *restic.ID

	// Force using a parent
	if !opts.Force && opts.Parent != "" {
		id, err := restic.FindSnapshot(repo, opts.Parent)
		if err != nil {
			return errors.Fatalf("invalid id %q: %v", opts.Parent, err)
		}

		parentSnapshotID = &id
	}

	// Find last snapshot to set it as parent, if not already set
	if !opts.Force && parentSnapshotID == nil {
		id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
		if err == nil {
			parentSnapshotID = &id
		} else if err != restic.ErrNoSnapshotFound {
			return err
		}
	}

	if parentSnapshotID != nil {
		Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
	}

	Verbosef("scan %v\n", target)

	// add patterns from file
	if opts.ExcludeFile != "" {
		file, err := fs.Open(opts.ExcludeFile)
		if err != nil {
			Warnf("error reading exclude patterns: %v", err)
			return nil
		}

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			line := scanner.Text()
			if !strings.HasPrefix(line, "#") {
				line = os.ExpandEnv(line)
				opts.Excludes = append(opts.Excludes, line)
			}
		}
	}

	selectFilter := func(item string, fi os.FileInfo) bool {
		matched, err := filter.List(opts.Excludes, item)
		if err != nil {
			Warnf("error for exclude pattern: %v", err)
		}

		if matched {
			debug.Log("path %q excluded by a filter", item)
			return false
		}

		if !opts.ExcludeOtherFS || fi == nil {
			return true
		}

		id, err := fs.DeviceID(fi)
		if err != nil {
			// This should never happen because gatherDevices() would have
			// errored out earlier. If it still does that's a reason to panic.
			panic(err)
		}

		for dir := item; dir != ""; dir = filepath.Dir(dir) {
			debug.Log("item %v, test dir %v", item, dir)

			allowedID, ok := allowedDevs[dir]
			if !ok {
				continue
			}

			if allowedID != id {
				debug.Log("path %q on disallowed device %d", item, id)
				return false
			}

			return true
		}

		panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowedDevs))
	}

	stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts))
	if err != nil {
		return err
	}

	arch := archiver.New(repo)
	arch.Excludes = opts.Excludes
	arch.SelectFilter = selectFilter

	arch.Warn = func(dir string, fi os.FileInfo, err error) {
		// TODO: make ignoring errors configurable
		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
	}

	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
	if err != nil {
		return err
	}

	Verbosef("snapshot %s saved\n", id.Str())

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"time"
)

// Deck is an array of Card objects.
type Deck []Card

// InitializeDeck will create a deck of 52 cards and shuffle them.
func InitializeDeck() (deck Deck) {
	deck = CreateDeckOfCards()
	deck.Shuffle()
	return
}

// Shuffle does a random swap of each element in the array.
func (deck *Deck) Shuffle() Deck {
	rand.Seed(time.Now().UTC().UnixNano())
	d := *deck
	for i := range d {
		r := rand.Intn(len(d))
		d[i], d[r] = d[r], d[i]
	}
	return d
}

// Deal cards to player's hands
func (deck *Deck) Deal(p1, p2 *Player) {
	count := 0
	d := *deck
	for len(p1.Hand) < 10 || len(p2.Hand) < 10 {
		if count%2 == 0 {
			d = d.DrawCard(&p1.Hand)
		} else {
			d = d.DrawCard(&p2.Hand)
		}
		count++
	}
}

// DrawCard by popping a card from the Deck and appending it to a player's hand.
func (deck *Deck) DrawCard(hand *Hand) Deck {
	d := *deck
	card := d[len(d)-1]
	d = d[:len(d)-1]
	*hand = append(*hand, card)
	return d
}
<commit_msg>Slight refactoring to get rid of extra variable instantiation.<commit_after>package main

import (
	"math/rand"
	"time"
)

// Deck is an array of Card objects.
type Deck []Card

// InitializeDeck will create a deck of 52 cards and shuffle them.
func InitializeDeck() (deck Deck) {
	deck = CreateDeckOfCards()
	deck.Shuffle()
	return
}

// Shuffle does a random swap of each element in the array.
func (deck *Deck) Shuffle() (d Deck) {
	rand.Seed(time.Now().UTC().UnixNano())
	d = *deck
	for i := range d {
		r := rand.Intn(len(d))
		d[i], d[r] = d[r], d[i]
	}
	return
}

// Deal cards to player's hands
func (deck *Deck) Deal(p1, p2 *Player) {
	count := 0
	for len(p1.Hand) < 10 || len(p2.Hand) < 10 {
		if count%2 == 0 {
			*deck = deck.DrawCard(&p1.Hand)
		} else {
			*deck = deck.DrawCard(&p2.Hand)
		}
		count++
	}
}

// DrawCard by popping a card from the Deck and appending it to a player's hand.
func (deck *Deck) DrawCard(hand *Hand) (d Deck) {
	d = *deck
	card := d[len(d)-1]
	d = d[:len(d)-1]
	*hand = append(*hand, card)
	return
}
<|endoftext|>


<commit_before>
// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spec

import (
	"strings"

	"github.com/go-swagger/go-swagger/swag"
)

// type operationRef struct {
// 	operation *Operation
// 	parameter *Parameter
// }

// specAnalyzer takes a swagger spec object and turns it into a registry
// with a bunch of utility methods to act on the information in the spec
type specAnalyzer struct {
	spec        *Swagger
	consumes    map[string]struct{}
	produces    map[string]struct{}
	authSchemes map[string]struct{}
	operations  map[string]map[string]*Operation
}

func (s *specAnalyzer) initialize() {
	for _, c := range s.spec.Consumes {
		s.consumes[c] = struct{}{}
	}
	for _, c := range s.spec.Produces {
		s.produces[c] = struct{}{}
	}
	for _, ss := range s.spec.Security {
		for k := range ss {
			s.authSchemes[k] = struct{}{}
		}
	}
	for path, pathItem := range s.AllPaths() {
		s.analyzeOperations(path, &pathItem)
	}
}

func (s *specAnalyzer) analyzeOperations(path string, op *PathItem) {
	s.analyzeOperation("GET", path, op.Get)
	s.analyzeOperation("PUT", path, op.Put)
	s.analyzeOperation("POST", path, op.Post)
	s.analyzeOperation("PATCH", path, op.Patch)
	s.analyzeOperation("DELETE", path, op.Delete)
	s.analyzeOperation("HEAD", path, op.Head)
	s.analyzeOperation("OPTIONS", path, op.Options)
}

func (s *specAnalyzer) analyzeOperation(method, path string, op *Operation) {
	if op != nil {
		for _, c := range op.Consumes {
			s.consumes[c] = struct{}{}
		}
		for _, c := range op.Produces {
			s.produces[c] = struct{}{}
		}
		for _, ss := range op.Security {
			for k := range ss {
				s.authSchemes[k] = struct{}{}
			}
		}
		if _, ok := s.operations[method]; !ok {
			s.operations[method] = make(map[string]*Operation)
		}
		s.operations[method][path] = op
	}
}

// SecurityRequirement is a representation of a security requirement for an operation
type SecurityRequirement struct {
	Name   string
	Scopes []string
}

// SecurityRequirementsFor gets the security requirements for the operation
func (s *specAnalyzer) SecurityRequirementsFor(operation *Operation) []SecurityRequirement {
	if s.spec.Security == nil && operation.Security == nil {
		return nil
	}

	schemes := s.spec.Security
	if operation.Security != nil {
		schemes = operation.Security
	}

	unique := make(map[string]SecurityRequirement)
	for _, scheme := range schemes {
		for k, v := range scheme {
			if _, ok := unique[k]; !ok {
				unique[k] = SecurityRequirement{Name: k, Scopes: v}
			}
		}
	}

	var result []SecurityRequirement
	for _, v := range unique {
		result = append(result, v)
	}
	return result
}

// SecurityDefinitionsFor gets the matching security definitions for a set of requirements
func (s *specAnalyzer) SecurityDefinitionsFor(operation *Operation) map[string]SecurityScheme {
	requirements := s.SecurityRequirementsFor(operation)
	if len(requirements) == 0 {
		return nil
	}
	result := make(map[string]SecurityScheme)
	for _, v := range requirements {
		if definition, ok := s.spec.SecurityDefinitions[v.Name]; ok {
			if definition != nil {
				result[v.Name] = *definition
			}
		}
	}
	return result
}

// ConsumesFor gets the mediatypes for the operation
func (s *specAnalyzer) ConsumesFor(operation *Operation) []string {
	cons := make(map[string]struct{})
	for k := range s.consumes {
		cons[k] = struct{}{}
	}
	for _, c := range operation.Consumes {
		cons[c] = struct{}{}
	}
	return s.structMapKeys(cons)
}

// ProducesFor gets the mediatypes for the operation
func (s *specAnalyzer) ProducesFor(operation *Operation) []string {
	prod := make(map[string]struct{})
	for k := range s.produces {
		prod[k] = struct{}{}
	}
	for _, c := range operation.Produces {
		prod[c] = struct{}{}
	}
	return s.structMapKeys(prod)
}

func fieldNameFromParam(param *Parameter) string {
	if nm, ok := param.Extensions.GetString("go-name"); ok {
		return nm
	}
	return swag.ToGoName(param.Name)
}

func (s *specAnalyzer) paramsAsMap(parameters []Parameter, res map[string]Parameter) {
	for _, param := range parameters {
		pr := param
		if pr.Ref.String() != "" {
			obj, _, err := pr.Ref.GetPointer().Get(s.spec)
			if err != nil {
				panic(err)
			}
			pr = obj.(Parameter)
		}
		res[fieldNameFromParam(&pr)] = pr
	}
}

func (s *specAnalyzer) ParametersFor(operationID string) []Parameter {
	gatherParams := func(pi *PathItem, op *Operation) []Parameter {
		bag := make(map[string]Parameter)
		s.paramsAsMap(pi.Parameters, bag)
		s.paramsAsMap(op.Parameters, bag)

		var res []Parameter
		for _, v := range bag {
			res = append(res, v)
		}
		return res
	}
	for _, pi := range s.spec.Paths.Paths {
		if pi.Get != nil && pi.Get.ID == operationID {
			return gatherParams(&pi, pi.Get)
		}
		if pi.Head != nil && pi.Head.ID == operationID {
			return gatherParams(&pi, pi.Head)
		}
		if pi.Options != nil && pi.Options.ID == operationID {
			return gatherParams(&pi, pi.Options)
		}
		if pi.Post != nil && pi.Post.ID == operationID {
			return gatherParams(&pi, pi.Post)
		}
		if pi.Patch != nil && pi.Patch.ID == operationID {
			return gatherParams(&pi, pi.Patch)
		}
		if pi.Put != nil && pi.Put.ID == operationID {
			return gatherParams(&pi, pi.Put)
		}
		if pi.Delete != nil && pi.Delete.ID == operationID {
			return gatherParams(&pi, pi.Delete)
		}
	}
	return nil
}

func (s *specAnalyzer) ParamsFor(method, path string) map[string]Parameter {
	res := make(map[string]Parameter)
	if pi, ok := s.spec.Paths.Paths[path]; ok {
		s.paramsAsMap(pi.Parameters, res)
		s.paramsAsMap(s.operations[strings.ToUpper(method)][path].Parameters, res)
	}
	return res
}

func (s *specAnalyzer) OperationForName(operationID string) (*Operation, bool) {
	for _, v := range s.operations {
		for _, vv := range v {
			if operationID == vv.ID {
				return vv, true
			}
		}
	}
	return nil, false
}

func (s *specAnalyzer) OperationFor(method, path string) (*Operation, bool) {
	if mp, ok := s.operations[strings.ToUpper(method)]; ok {
		op, fn := mp[path]
		return op, fn
	}
	return nil, false
}

func (s *specAnalyzer) Operations() map[string]map[string]*Operation {
	return s.operations
}

func (s *specAnalyzer) structMapKeys(mp map[string]struct{}) []string {
	var result []string
	for k := range mp {
		result = append(result, k)
	}
	return result
}

// AllPaths returns all the paths in the swagger spec
func (s *specAnalyzer) AllPaths() map[string]PathItem {
	if s.spec == nil || s.spec.Paths == nil {
		return nil
	}
	return s.spec.Paths.Paths
}

func (s *specAnalyzer) OperationIDs() []string {
	var result []string
	for _, v := range s.operations {
		for _, vv := range v {
			result = append(result, vv.ID)
		}
	}
	return result
}

func (s *specAnalyzer) RequiredConsumes() []string {
	return s.structMapKeys(s.consumes)
}

func (s *specAnalyzer) RequiredProduces() []string {
	return s.structMapKeys(s.produces)
}

func (s *specAnalyzer) RequiredSchemes() []string {
	return s.structMapKeys(s.authSchemes)
}
<commit_msg>contributes to #90 renders all the known annotations<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spec

import (
	"strings"

	"github.com/go-swagger/go-swagger/swag"
)

// type operationRef struct {
// 	operation *Operation
// 	parameter *Parameter
// }

// specAnalyzer takes a swagger spec object and turns it into a registry
// with a bunch of utility methods to act on the information in the spec
type specAnalyzer struct {
	spec        *Swagger
	consumes    map[string]struct{}
	produces    map[string]struct{}
	authSchemes map[string]struct{}
	operations  map[string]map[string]*Operation
}

func (s *specAnalyzer) initialize() {
	for _, c := range s.spec.Consumes {
		s.consumes[c] = struct{}{}
	}
	for _, c := range s.spec.Produces {
		s.produces[c] = struct{}{}
	}
	for _, ss := range s.spec.Security {
		for k := range ss {
			s.authSchemes[k] = struct{}{}
		}
	}
	for path, pathItem := range s.AllPaths() {
		s.analyzeOperations(path, &pathItem)
	}
}

func (s *specAnalyzer) analyzeOperations(path string, op *PathItem) {
	s.analyzeOperation("GET", path, op.Get)
	s.analyzeOperation("PUT", path, op.Put)
	s.analyzeOperation("POST", path, op.Post)
	s.analyzeOperation("PATCH", path, op.Patch)
	s.analyzeOperation("DELETE", path, op.Delete)
	s.analyzeOperation("HEAD", path, op.Head)
	s.analyzeOperation("OPTIONS", path, op.Options)
}

func (s *specAnalyzer) analyzeOperation(method, path string, op *Operation) {
	if op != nil {
		for _, c := range op.Consumes {
			s.consumes[c] = struct{}{}
		}
		for _, c := range op.Produces {
			s.produces[c] = struct{}{}
		}
		for _, ss := range op.Security {
			for k := range ss {
				s.authSchemes[k] = struct{}{}
			}
		}
		if _, ok := s.operations[method]; !ok {
			s.operations[method] = make(map[string]*Operation)
		}
		s.operations[method][path] = op
	}
}

// SecurityRequirement is a representation of a security requirement for an operation
type SecurityRequirement struct {
	Name   string
	Scopes []string
}

// SecurityRequirementsFor gets the security requirements for the operation
func (s *specAnalyzer) SecurityRequirementsFor(operation *Operation) []SecurityRequirement {
	if s.spec.Security == nil && operation.Security == nil {
		return nil
	}

	schemes := s.spec.Security
	if operation.Security != nil {
		schemes = operation.Security
	}

	unique := make(map[string]SecurityRequirement)
	for _, scheme := range schemes {
		for k, v := range scheme {
			if _, ok := unique[k]; !ok {
				unique[k] = SecurityRequirement{Name: k, Scopes: v}
			}
		}
	}

	var result []SecurityRequirement
	for _, v := range unique {
		result = append(result, v)
	}
	return result
}

// SecurityDefinitionsFor gets the matching security definitions for a set of requirements
func (s *specAnalyzer) SecurityDefinitionsFor(operation *Operation) map[string]SecurityScheme {
	requirements := s.SecurityRequirementsFor(operation)
	if len(requirements) == 0 {
		return nil
	}
	result := make(map[string]SecurityScheme)
	for _, v := range requirements {
		if definition, ok := s.spec.SecurityDefinitions[v.Name]; ok {
			if definition != nil {
				result[v.Name] = *definition
			}
		}
	}
	return result
}

// ConsumesFor gets the mediatypes for the operation
func (s *specAnalyzer) ConsumesFor(operation *Operation) []string {
	cons := make(map[string]struct{})
	for k := range s.consumes {
		cons[k] = struct{}{}
	}
	for _, c := range operation.Consumes {
		cons[c] = struct{}{}
	}
	return s.structMapKeys(cons)
}

// ProducesFor gets the mediatypes for the operation
func (s *specAnalyzer) ProducesFor(operation *Operation) []string {
	prod := make(map[string]struct{})
	for k := range s.produces {
		prod[k] = struct{}{}
	}
	for _, c := range operation.Produces {
		prod[c] = struct{}{}
	}
	return s.structMapKeys(prod)
}

func fieldNameFromParam(param *Parameter) string {
	if nm, ok := param.Extensions.GetString("go-name"); ok {
		return nm
	}
	return swag.ToGoName(param.Name)
}

func (s *specAnalyzer) paramsAsMap(parameters []Parameter, res map[string]Parameter) {
	for _, param := range parameters {
		pr := param
		if pr.Ref.String() != "" {
			obj, _, err := pr.Ref.GetPointer().Get(s.spec)
			if err != nil {
				panic(err)
			}
			pr = obj.(Parameter)
		}
		res[fieldNameFromParam(&pr)] = pr
	}
}

func (s *specAnalyzer) ParametersFor(operationID string) []Parameter {
	gatherParams := func(pi *PathItem, op *Operation) []Parameter {
		bag := make(map[string]Parameter)
		s.paramsAsMap(pi.Parameters, bag)
		s.paramsAsMap(op.Parameters, bag)

		var res []Parameter
		for _, v := range bag {
			res = append(res, v)
		}
		return res
	}
	for _, pi := range s.spec.Paths.Paths {
		if pi.Get != nil && pi.Get.ID == operationID {
			return gatherParams(&pi, pi.Get)
		}
		if pi.Head != nil && pi.Head.ID == operationID {
			return gatherParams(&pi, pi.Head)
		}
		if pi.Options != nil && pi.Options.ID == operationID {
			return gatherParams(&pi, pi.Options)
		}
		if pi.Post != nil && pi.Post.ID == operationID {
			return gatherParams(&pi, pi.Post)
		}
		if pi.Patch != nil && pi.Patch.ID == operationID {
			return gatherParams(&pi, pi.Patch)
		}
		if pi.Put != nil && pi.Put.ID == operationID {
			return gatherParams(&pi, pi.Put)
		}
		if pi.Delete != nil && pi.Delete.ID == operationID {
			return gatherParams(&pi, pi.Delete)
		}
	}
	return nil
}

func (s *specAnalyzer) ParamsFor(method, path string) map[string]Parameter {
	res := make(map[string]Parameter)
	if pi, ok := s.spec.Paths.Paths[path]; ok {
		s.paramsAsMap(pi.Parameters, res)
		s.paramsAsMap(s.operations[strings.ToUpper(method)][path].Parameters, res)
	}
	return res
}

func (s *specAnalyzer) OperationForName(operationID string) (string, string, *Operation, bool) {
	for method, pathItem := range s.operations {
		for path, op := range pathItem {
			if operationID == op.ID {
				return method, path, op, true
			}
		}
	}
	return "", "", nil, false
}

func (s *specAnalyzer) OperationFor(method, path string) (*Operation, bool) {
	if mp, ok := s.operations[strings.ToUpper(method)]; ok {
		op, fn := mp[path]
		return op, fn
	}
	return nil, false
}

func (s *specAnalyzer) Operations() map[string]map[string]*Operation {
	return s.operations
}

func (s *specAnalyzer) structMapKeys(mp map[string]struct{}) []string {
	var result []string
	for k := range mp {
		result = append(result, k)
	}
	return result
}

// AllPaths returns all the paths in the swagger spec
func (s *specAnalyzer) AllPaths() map[string]PathItem {
	if s.spec == nil || s.spec.Paths == nil {
		return nil
	}
	return s.spec.Paths.Paths
}

func (s *specAnalyzer) OperationIDs() []string {
	var result []string
	for _, v := range s.operations {
		for _, vv := range v {
			result = append(result, vv.ID)
		}
	}
	return result
}

func (s *specAnalyzer) RequiredConsumes() []string {
	return s.structMapKeys(s.consumes)
}

func (s *specAnalyzer) RequiredProduces() []string {
	return s.structMapKeys(s.produces)
}

func (s *specAnalyzer) RequiredSchemes() []string {
	return s.structMapKeys(s.authSchemes)
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013-2016, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package util

import (
	"fmt"
	"golang.org/x/exp/utf8string"
	"regexp"
	"strings"
)

// StringSlice makes it possible to scan Posgres arrays directly into a golang
// slice. Borrowed from https://gist.github.com/adharris/4163702.
type StringSlice []string

// Scan implements sql.Scanner for the StringSlice type.
func (s *StringSlice) Scan(src interface{}) error {
	asBytes, ok := src.([]byte)
	if !ok {
		return error(New("Scan source was not []bytes"))
	}

	asString := string(asBytes)
	parsed := parseArray(asString)
	(*s) = StringSlice(parsed)

	return nil
}

// construct a regexp to extract values:
var (
	// unquoted array values must not contain: (" , \ { } whitespace NULL)
	// and must be at least one char
	unquotedChar  = `[^",\\{}\s(NULL)]`
	unquotedValue = fmt.Sprintf("(%s)+", unquotedChar)

	// quoted array values are surrounded by double quotes, can be any
	// character except " or \, which must be backslash escaped:
	quotedChar  = `[^"\\]|\\"|\\\\`
	quotedValue = fmt.Sprintf("\"(%s)*\"", quotedChar)

	// an array value may be either quoted or unquoted:
	arrayValue = fmt.Sprintf("(?P<value>(%s|%s))", unquotedValue, quotedValue)

	// Array values are separated with a comma IF there is more than one value:
	arrayExp = regexp.MustCompile(fmt.Sprintf("((%s)(,)?)", arrayValue))

	valueIndex int
)

// Find the index of the 'value' named expression
func init() {
	for i, subexp := range arrayExp.SubexpNames() {
		if subexp == "value" {
			valueIndex = i
			break
		}
	}
}

// Parse the output string from the array type.
// Regex used: (((?P<value>(([^",\\{}\s(NULL)])+|"([^"\\]|\\"|\\\\)*")))(,)?)
func parseArray(array string) []string {
	results := make([]string, 0)
	matches := arrayExp.FindAllStringSubmatch(array, -1)
	for _, match := range matches {
		s := match[valueIndex]
		// the string _might_ be wrapped in quotes, so trim them:
		s = strings.Trim(s, "\"")
		results = append(results, s)
	}
	return results
}

// TrimStringMax trims a string down if its length is over a certain amount
func TrimStringMax(s string, strLength int) string {
	r := utf8string.NewString(s)
	if r.RuneCount() > strLength {
		return r.Slice(0, strLength)
	}
	return s
}

// RemoveDupStrings removes duplicates from a slice of strings. The slice of
// strings must be sorted before it's used with this function.
func RemoveDupStrings(strs []string) []string {
	for i, v := range strs {
		if i+1 >= len(strs) {
			break
		}
		if v == strs[i+1] {
			strs = DelSliceElement(i+1, strs)
		}
	}
	return strs
}

// DelSliceElement removes an element from a slice of strings.
func DelSliceElement(pos int, strs []string) []string {
	strs = append(strs[:pos], strs[pos+1:]...)
	return strs
}
<commit_msg>more than two duplicate strings in an indexed slice would end up leaving exactly two duplicates remaining, thus breaking it still<commit_after>/*
 * Copyright (c) 2013-2016, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package util

import (
	"fmt"
	"golang.org/x/exp/utf8string"
	"regexp"
	"strings"
)

// StringSlice makes it possible to scan Posgres arrays directly into a golang
// slice. Borrowed from https://gist.github.com/adharris/4163702.
type StringSlice []string

// Scan implements sql.Scanner for the StringSlice type.
func (s *StringSlice) Scan(src interface{}) error {
	asBytes, ok := src.([]byte)
	if !ok {
		return error(New("Scan source was not []bytes"))
	}

	asString := string(asBytes)
	parsed := parseArray(asString)
	(*s) = StringSlice(parsed)

	return nil
}

// construct a regexp to extract values:
var (
	// unquoted array values must not contain: (" , \ { } whitespace NULL)
	// and must be at least one char
	unquotedChar  = `[^",\\{}\s(NULL)]`
	unquotedValue = fmt.Sprintf("(%s)+", unquotedChar)

	// quoted array values are surrounded by double quotes, can be any
	// character except " or \, which must be backslash escaped:
	quotedChar  = `[^"\\]|\\"|\\\\`
	quotedValue = fmt.Sprintf("\"(%s)*\"", quotedChar)

	// an array value may be either quoted or unquoted:
	arrayValue = fmt.Sprintf("(?P<value>(%s|%s))", unquotedValue, quotedValue)

	// Array values are separated with a comma IF there is more than one value:
	arrayExp = regexp.MustCompile(fmt.Sprintf("((%s)(,)?)", arrayValue))

	valueIndex int
)

// Find the index of the 'value' named expression
func init() {
	for i, subexp := range arrayExp.SubexpNames() {
		if subexp == "value" {
			valueIndex = i
			break
		}
	}
}

// Parse the output string from the array type.
// Regex used: (((?P<value>(([^",\\{}\s(NULL)])+|"([^"\\]|\\"|\\\\)*")))(,)?)
func parseArray(array string) []string {
	results := make([]string, 0)
	matches := arrayExp.FindAllStringSubmatch(array, -1)
	for _, match := range matches {
		s := match[valueIndex]
		// the string _might_ be wrapped in quotes, so trim them:
		s = strings.Trim(s, "\"")
		results = append(results, s)
	}
	return results
}

// TrimStringMax trims a string down if its length is over a certain amount
func TrimStringMax(s string, strLength int) string {
	r := utf8string.NewString(s)
	if r.RuneCount() > strLength {
		return r.Slice(0, strLength)
	}
	return s
}

// RemoveDupStrings removes duplicates from a slice of strings. The slice of
// strings must be sorted before it's used with this function.
func RemoveDupStrings(strs []string) []string {
	for i, v := range strs {
		// catches the case where we've sliced off all the duplicates,
		// but if we don't break here checking the last element will
		// needlessly keep marching down the remainder of the slice for
		// no effect
		if i > len(strs) {
			break
		}
		j := 1
		s := 0
		for {
			if i+1 >= len(strs) {
				break
			}
			if v == strs[i+j] {
				j++
				s++
			} else {
				break
			}
		}
		if s == 0 {
			continue
		}
		strs = delTwoPosElements(i+i, s, strs)
	}
	return strs
}

// DelSliceElement removes an element from a slice of strings.
func DelSliceElement(pos int, strs []string) []string {
	return delTwoPosElements(pos, 1, strs)
}

func delTwoPosElements(pos int, skip int, strs []string) []string {
	strs = append(strs[:pos], strs[pos+skip:]...)
	return strs
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"strings"
	"testing"

	"cloud.google.com/go/bigquery"
	"cloud.google.com/go/datastore"
	"cloud.google.com/go/storage"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	"golang.org/x/net/context"
	dlppb "google.golang.org/genproto/googleapis/privacy/dlp/v2"
)

func TestInspectString(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectString(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, test.s)
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
	}
}

func TestInspectFile(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, dlppb.ByteContentItem_TEXT_UTF8, strings.NewReader(test.s))
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
	}
}

const (
	ssnFileName             = "fake_ssn.txt"
	nothingEventfulFileName = "nothing_eventful.txt"
	bucketName              = "golang-samples-dlp-test"
)

func writeTestGCSFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	bucket := client.Bucket(bucketName)
	_, err = bucket.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			if err := bucket.Create(ctx, projectID, nil); err != nil {
				t.Fatalf("Failed to create bucket: %v", err)
			}
		default:
			t.Fatalf("error getting bucket attrs: %v", err)
		}
	}
	if err := writeObject(ctx, bucket, ssnFileName, "My SSN is 111222333"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
	if err := writeObject(ctx, bucket, nothingEventfulFileName, "Nothing eventful"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
}

func writeObject(ctx context.Context, bucket *storage.BucketHandle, fileName, content string) error {
	obj := bucket.Object(fileName)
	_, err := obj.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			w := obj.NewWriter(ctx)
			w.Write([]byte(content))
			if err := w.Close(); err != nil {
				return err
			}
		default:
			return err
		}
	}
	return nil
}

func TestInspectGCS(t *testing.T) {
	testutil.SystemTest(t)
	writeTestGCSFiles(t, projectID)
	tests := []struct {
		fileName string
		want     string
	}{
		{
			fileName: ssnFileName,
			want:     "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			fileName: nothingEventfulFileName,
			want:     "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectGCSFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, "test-topic", "test-sub", bucketName, test.fileName)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectString(%s) = %q, want %q substring", test.fileName, got, test.want)
		}
	}
}

type SSNTask struct {
	Description string
}

type BoringTask struct {
	Description string
}

func writeTestDatastoreFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	kind := "SSNTask"
	name := "ssntask1"
	ssnKey := datastore.NameKey(kind, name, nil)
	task := SSNTask{
		Description: "My SSN is 111222333",
	}
	if _, err := client.Put(ctx, ssnKey, &task); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}

	kind = "BoringTask"
	name = "boringtask1"
	boringKey := datastore.NameKey(kind, name, nil)
	boringTask := BoringTask{
		Description: "Nothing meaningful",
	}
	if _, err := client.Put(ctx, boringKey, &boringTask); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}
}

func TestInspectDatastore(t *testing.T) {
	testutil.SystemTest(t)
	writeTestDatastoreFiles(t, projectID)
	tests := []struct {
		kind string
		want string
	}{
		{
			kind: "SSNTask",
			want: "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			kind: "BoringTask",
			want: "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectDatastore(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, "test-topic", "test-sub", projectID, "", test.kind)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectDatastore(%s) = %q, want %q substring", test.kind, got, test.want)
		}
	}
}

type Item struct {
	Description string
}

const (
	harmlessTable = "harmless"
	harmfulTable  = "harmful"
	bqDatasetID   = "golang_samples_dlp"
)

func createBigqueryTestFiles(projectID, datasetID string) error {
	ctx := context.Background()
	client, err := bigquery.NewClient(ctx, projectID)
	if err != nil {
		return err
	}
	defer client.Close()
	d := client.Dataset(datasetID)
	if _, err := d.Metadata(ctx); err != nil {
		if err := d.Create(ctx, &bigquery.DatasetMetadata{}); err != nil {
			return err
		}
	}
	schema, err := bigquery.InferSchema(Item{})
	if err != nil {
		return err
	}
	if err := uploadBigQuery(ctx, d, schema, harmlessTable, "Nothing meaningful"); err != nil {
		return err
	}
	return uploadBigQuery(ctx, d, schema, harmfulTable, "My SSN is 111222333")
}

func uploadBigQuery(ctx context.Context, d *bigquery.Dataset, schema bigquery.Schema, table, content string) error {
	t := d.Table(table)
	if _, err := t.Metadata(ctx); err == nil {
		return nil
	}
	if err := t.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
		return err
	}
	source := bigquery.NewReaderSource(strings.NewReader(content))
	l := t.LoaderFrom(source)
	job, err := l.Run(ctx)
	if err != nil {
		return err
	}
	status, err := job.Wait(ctx)
	if err != nil {
		return err
	}
	return status.Err()
}

func TestInspectBigquery(t *testing.T) {
	testutil.SystemTest(t)
	if err := createBigqueryTestFiles(projectID, bqDatasetID); err != nil {
		t.Fatalf("error creating test BigQuery files: %v", err)
	}
	tests := []struct {
		table string
		want  string
	}{
		{
			table: harmfulTable,
			want:  "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			table: harmlessTable,
			want:  "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectBigquery(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, "test-topic", "test-sub", projectID, bqDatasetID, test.table)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectBigquery(%s) = %q, want %q substring", test.table, got, test.want)
		}
	}
}
<commit_msg>dlp: skip flaky tests (#500)<commit_after>// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"strings"
	"testing"

	"cloud.google.com/go/bigquery"
	"cloud.google.com/go/datastore"
	"cloud.google.com/go/storage"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	"golang.org/x/net/context"
	dlppb "google.golang.org/genproto/googleapis/privacy/dlp/v2"
)

func TestInspectString(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectString(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, test.s)
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
	}
}

func TestInspectFile(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, dlppb.ByteContentItem_TEXT_UTF8, strings.NewReader(test.s))
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
	}
}

const (
	ssnFileName             = "fake_ssn.txt"
	nothingEventfulFileName = "nothing_eventful.txt"
	bucketName              = "golang-samples-dlp-test"
)

func writeTestGCSFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	bucket := client.Bucket(bucketName)
	_, err = bucket.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			if err := bucket.Create(ctx, projectID, nil); err != nil {
				t.Fatalf("Failed to create bucket: %v", err)
			}
		default:
			t.Fatalf("error getting bucket attrs: %v", err)
		}
	}
	if err := writeObject(ctx, bucket, ssnFileName, "My SSN is 111222333"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
	if err := writeObject(ctx, bucket, nothingEventfulFileName, "Nothing eventful"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
}

func writeObject(ctx context.Context, bucket *storage.BucketHandle, fileName, content string) error {
	obj := bucket.Object(fileName)
	_, err := obj.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			w := obj.NewWriter(ctx)
			w.Write([]byte(content))
			if err := w.Close(); err != nil {
				return err
			}
		default:
			return err
		}
	}
	return nil
}

func TestInspectGCS(t *testing.T) {
	t.Skip("flaky due to timeout")
	testutil.SystemTest(t)
	writeTestGCSFiles(t, projectID)
	tests := []struct {
		fileName string
		want     string
	}{
		{
			fileName: ssnFileName,
			want:     "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			fileName: nothingEventfulFileName,
			want:     "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectGCSFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, "test-topic", "test-sub", bucketName, test.fileName)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectString(%s) = %q, want %q substring", test.fileName, got, test.want)
		}
	}
}

type SSNTask struct {
	Description string
}

type BoringTask struct {
	Description string
}

func writeTestDatastoreFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	kind := "SSNTask"
	name := "ssntask1"
	ssnKey := datastore.NameKey(kind, name, nil)
	task := SSNTask{
		Description: "My SSN is 111222333",
	}
	if _, err := client.Put(ctx, ssnKey, &task); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}

	kind = "BoringTask"
	name = "boringtask1"
	boringKey := datastore.NameKey(kind, name, nil)
	boringTask := BoringTask{
		Description: "Nothing meaningful",
	}
	if _, err := client.Put(ctx, boringKey, &boringTask); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}
}

func TestInspectDatastore(t *testing.T) {
	t.Skip("flaky due to timeout")
	testutil.SystemTest(t)
	writeTestDatastoreFiles(t, projectID)
	tests := []struct {
		kind string
		want string
	}{
		{
			kind: "SSNTask",
			want: "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			kind: "BoringTask",
			want: "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectDatastore(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, "test-topic", "test-sub", projectID, "", test.kind)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectDatastore(%s) = %q, want %q substring", test.kind, got, test.want)
		}
	}
}

type Item struct {
	Description string
}

const (
	harmlessTable = "harmless"
	harmfulTable  = "harmful"
	bqDatasetID   = "golang_samples_dlp"
)

func createBigqueryTestFiles(projectID, datasetID string) error {
	ctx := context.Background()
	client, err := bigquery.NewClient(ctx, projectID)
	if err != nil {
		return err
	}
	defer client.Close()
	d := client.Dataset(datasetID)
	if _, err := d.Metadata(ctx); err != nil {
		if err := d.Create(ctx, &bigquery.DatasetMetadata{}); err != nil {
			return err
		}
	}
	schema, err := bigquery.InferSchema(Item{})
	if err != nil {
		return err
	}
	if err := uploadBigQuery(ctx, d, schema, harmlessTable, "Nothing meaningful"); err != nil {
		return err
	}
	return uploadBigQuery(ctx, d, schema, harmfulTable, "My SSN is 111222333")
}

func uploadBigQuery(ctx context.Context, d *bigquery.Dataset, schema bigquery.Schema, table, content string) error {
	t := d.Table(table)
	if _, err := t.Metadata(ctx); err == nil {
		return nil
	}
	if err := t.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
		return err
	}
	source := bigquery.NewReaderSource(strings.NewReader(content))
	l := t.LoaderFrom(source)
	job, err := l.Run(ctx)
	if err != nil {
		return err
	}
	status, err := job.Wait(ctx)
	if err != nil {
		return err
	}
	return status.Err()
}

func TestInspectBigquery(t *testing.T) {
	t.Skip("flaky due to timeout")
	testutil.SystemTest(t)
	if err := createBigqueryTestFiles(projectID, bqDatasetID); err != nil {
		t.Fatalf("error creating test BigQuery files: %v", err)
	}
	tests := []struct {
		table string
		want  string
	}{
		{
			table: harmfulTable,
			want:  "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			table: harmlessTable,
			want:  "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectBigquery(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, "test-topic", "test-sub", projectID, bqDatasetID, test.table)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectBigquery(%s) = %q, want %q substring", test.table, got, test.want)
		}
	}
}
<|endoftext|>