<commit_before>package gorjun

import (
	"encoding/json"
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"math/rand"
	"net/http"
	"sort"
	"strconv"
	"testing"
	"time"
)

func TestListUserFiles(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}

	d1 := []byte("This is a test file\n")
	ioutil.WriteFile("/tmp/libgorjun-test", d1, 0644)
	id, err := g.Upload("/tmp/libgorjun-test", "raw","false")
	if err != nil {
		t.Errorf("Failed to upload: %v", err)
	}
	fmt.Printf("File ID: %s", id)

	flist, err := g.ListUserFiles()
	if err != nil {
		t.Errorf("Failed to retrieve user files: %v", err)
	}
	if len(flist) <= 0 {
		t.Errorf("Resulting array is empty")
	}
}

func TestUploadRaw(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	d1 := []byte("This is a test file\n")
	ioutil.WriteFile("/tmp/libgorjun-test", d1, 0644)
	id, err := g.Upload("/tmp/libgorjun-test", "raw","false")
	if err != nil {
		t.Errorf("Failed to upload: %v", err)
	}
	fmt.Printf("File ID: %s", id)
}

func TestGetFileByName(t *testing.T) {
	g := NewGorjunServer()
	file, err := g.GetFileByName("libgorjun-test", "raw")
	if err != nil {
		t.Errorf("Failed to get file by name: %s", err)
	}
	fmt.Printf("File: %+v\n", file)
}

func TestRemoveFile(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	err = g.RemoveFile("libgorjun-test", "raw")
	if err != nil {
		t.Errorf("Failed to remove file: %v", err)
	}
}

func TestRemoveTemplate(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	id, err := g.Upload("data/abdysamat-apache-subutai-template_4.0.0_amd64.tar.gz", "template","false")
	if err != nil {
		t.Errorf("Failed to upload: %v", err)
	}
	fmt.Printf("Template uploaded successfully, id : %s\n", id)
	err = g.RemoveFileByID(id, "template")
	if err != nil {
		t.Errorf("Failed to remove file: %v", err)
	}
	fmt.Printf("Template removed successfully, id : %s\n", id)
}

//TestGorjunServer_CheckTemplatesSignatureExist will check signatures of
//templates, all templates should have more than zero signatures
func TestGorjunServer_CheckTemplatesSignatureExist(t *testing.T) {
	g := NewGorjunServer()
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/list", g.Hostname))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var templates []GorjunFile
	err = json.Unmarshal(data, &templates)
	for _, template := range templates {
		fmt.Printf("ID of templates is %s\n", template.ID)
		resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?id=%s", g.Hostname, template.ID))
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
		}
		var templateInfo []GorjunFile
		err = json.Unmarshal(data, &templateInfo)
		fmt.Printf("Len of signatture  is %d\n", len(templateInfo[0].Signature))
		assert.NotEqual(t, len(templateInfo[0].Signature), 0, "Template with ID = %s should have signature\n", template.ID)
	}
}

func Shuffle(a []string) {
	for i := range a {
		j := rand.Intn(i + 1)
		a[i], a[j] = a[j], a[i]
	}
}

//TestGorjunServer_GetLatestTemplateByVersion will upload templates
//with different version in random order, info rest should return latest by version
//if several version exits it should return by date
func TestGorjunServer_GetLatestTemplateByVersion(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	var dates []int
	templateVersions := []string{"0.1.6", "0.1.7", "0.1.9", "0.1.10", "0.1.11"}
	rand.Seed(time.Now().UnixNano())
	Shuffle(templateVersions)

	for _, version := range templateVersions {
		id, err := g.Upload("data/nginx-subutai-template_"+version+"_amd64.tar.gz", "template","false")
		if err != nil {
			t.Errorf("Failed to upload: %v", err)
		}
		fmt.Printf("Template uploaded successfully, id : %s\n", id)

		resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?id=%s", g.Hostname, id))
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
		}
		var template []GorjunFile
		err = json.Unmarshal(data, &template)
		timestamp, err := strconv.Atoi(template[0].Timestamp)
		dates = append(dates, timestamp)
		time.Sleep(100 * time.Millisecond)
	}
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?name=%s&owner=%s", g.Hostname, "nginx", g.Username))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var template []GorjunFile
	err = json.Unmarshal(data, &template)
	assert.Equal(t, "0.1.11", template[0].Version)
}

//TestGorjunServer_GetLatestRaw will upload raw
//files , info rest should return  by date
func TestGorjunServer_GetLatestRaw(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	var dates []int
	rawNumber := 10

	for i := 1; i <= rawNumber; i++ {
		id, err := g.Upload("data/abdysamat-apache-subutai-template_4.0.0_amd64.tar.gz", "raw","false")
		if err != nil {
			t.Errorf("Failed to upload: %v", err)
		}
		fmt.Printf("Raw uploaded successfully, id : %s\n", id)

		resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/raw/info?id=%s", g.Hostname, id))
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
		}
		var template []GorjunFile
		err = json.Unmarshal(data, &template)
		timestamp, err := strconv.Atoi(template[0].Timestamp)
		dates = append(dates, timestamp)
		time.Sleep(101 * time.Millisecond)
	}
	sort.Ints(dates)
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/raw/info?name=%s&owner=%s", g.Hostname, "abdysamat-apache", g.Username))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var template []GorjunFile
	err = json.Unmarshal(data, &template)
	timestamp, err := strconv.Atoi(template[0].Timestamp)
	fmt.Println(dates)
	fmt.Println(timestamp)
	fmt.Println(dates[rawNumber-1])
	assert.Equal(t, timestamp, dates[rawNumber-1])
}

//TestGorjunServer_SameTemplateUpload will upload
//same template twice, old template should deleted
func TestGorjunServer_SameTemplateUpload(t *testing.T) {
	g := NewGorjunServer()
	templateVersions := []string{"0.1.6", "0.1.7", "0.1.9", "0.1.10", "0.1.11"}
	for _, version := range templateVersions {
		resp, _ := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?name=%s&version=%s", g.Hostname, "nginx", version))
		if resp.StatusCode != http.StatusOK {
			fmt.Println("Test can't be run because templates should uploaded")
			return
		}
	}
	TestGorjunServer_GetLatestTemplateByVersion(t)
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/list", g.Hostname))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var templateList []GorjunFile
	err = json.Unmarshal(data, &templateList)

	m := make(map[string]int)

	for _, template := range templateList {
		s := template.Owner[0] + template.Name + template.Version
		m[s]++
		assert.NotEqual(t, m[s], 2, "Same template exist twice", template.ID)
	}
}<commit_msg>#289<commit_after>package gorjun

import (
	"encoding/json"
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"math/rand"
	"net/http"
	"sort"
	"strconv"
	"testing"
	"time"
)

func TestListUserFiles(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}

	d1 := []byte("This is a test file\n")
	ioutil.WriteFile("/tmp/libgorjun-test", d1, 0644)
	id, err := g.Upload("/tmp/libgorjun-test", "raw", "false")
	if err != nil {
		t.Errorf("Failed to upload: %v", err)
	}
	fmt.Printf("File ID: %s", id)

	flist, err := g.ListUserFiles()
	if err != nil {
		t.Errorf("Failed to retrieve user files: %v", err)
	}
	if len(flist) <= 0 {
		t.Errorf("Resulting array is empty")
	}
	err = g.Deletes("raw", "")
	if err != nil {
		t.Errorf("Failed to delete raw files: %v", err)
	}
}

func TestRemoveTemplate(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	id, err := g.Upload("data/abdysamat-apache-subutai-template_4.0.0_amd64.tar.gz", "template", "false")
	if err != nil {
		t.Errorf("Failed to upload: %v", err)
	}
	fmt.Printf("Template uploaded successfully, id : %s\n", id)
	err = g.RemoveFileByID(id, "template")
	if err != nil {
		t.Errorf("Failed to remove file: %v", err)
	}
	fmt.Printf("Template removed successfully, id : %s\n", id)
}

//TestGorjunServer_CheckTemplatesSignatureExist will check signatures of
//templates, all templates should have more than zero signatures
func TestGorjunServer_CheckTemplatesSignatureExist(t *testing.T) {
	g := NewGorjunServer()
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/list", g.Hostname))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var templates []GorjunFile
	err = json.Unmarshal(data, &templates)
	for _, template := range templates {
		fmt.Printf("ID of templates is %s\n", template.ID)
		resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?id=%s", g.Hostname, template.ID))
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
		}
		var templateInfo []GorjunFile
		err = json.Unmarshal(data, &templateInfo)
		fmt.Printf("Len of signatture  is %d\n", len(templateInfo[0].Signature))
		assert.NotEqual(t, len(templateInfo[0].Signature), 0, "Template with ID = %s should have signature\n", template.ID)
	}
}

func Shuffle(a []string) {
	for i := range a {
		j := rand.Intn(i + 1)
		a[i], a[j] = a[j], a[i]
	}
}

//TestGorjunServer_GetLatestTemplateByVersion will upload templates
//with different version in random order, info rest should return latest by version
//if several version exits it should return by date
func TestGorjunServer_GetLatestTemplateByVersion(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	var dates []int
	templateVersions := []string{"0.1.6", "0.1.7", "0.1.9", "0.1.10", "0.1.11"}
	rand.Seed(time.Now().UnixNano())
	Shuffle(templateVersions)

	for _, version := range templateVersions {
		id, err := g.Upload("data/nginx-subutai-template_"+version+"_amd64.tar.gz", "template", "false")
		if err != nil {
			t.Errorf("Failed to upload: %v", err)
		}
		fmt.Printf("Template uploaded successfully, id : %s\n", id)

		resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?id=%s", g.Hostname, id))
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
		}
		var template []GorjunFile
		err = json.Unmarshal(data, &template)
		timestamp, err := strconv.Atoi(template[0].Timestamp)
		dates = append(dates, timestamp)
		time.Sleep(100 * time.Millisecond)
	}
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?name=%s&owner=%s", g.Hostname, "nginx", g.Username))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var template []GorjunFile
	err = json.Unmarshal(data, &template)
	assert.Equal(t, "0.1.11", template[0].Version)
	g.Deletes("template", "")
}

//TestGorjunServer_GetLatestRaw will upload raw
//files , info rest should return  by date
func TestGorjunServer_GetLatestRaw(t *testing.T) {
	g := NewGorjunServer()
	err := g.AuthenticateUser()
	if err != nil {
		t.Errorf("Authnetication failure: %v", err)
	}
	var dates []int
	rawNumber := 10

	for i := 1; i <= rawNumber; i++ {
		id, err := g.Upload("data/abdysamat-apache-subutai-template_4.0.0_amd64.tar.gz", "raw", "false")
		if err != nil {
			t.Errorf("Failed to upload: %v", err)
		}
		fmt.Printf("Raw uploaded successfully, id : %s\n", id)

		resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/raw/info?id=%s", g.Hostname, id))
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
		}
		var template []GorjunFile
		err = json.Unmarshal(data, &template)
		timestamp, err := strconv.Atoi(template[0].Timestamp)
		dates = append(dates, timestamp)
		time.Sleep(101 * time.Millisecond)
	}
	sort.Ints(dates)
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/raw/info?name=%s&owner=%s", g.Hostname, "abdysamat-apache", g.Username))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var template []GorjunFile
	err = json.Unmarshal(data, &template)
	timestamp, err := strconv.Atoi(template[0].Timestamp)
	fmt.Println(dates)
	fmt.Println(timestamp)
	fmt.Println(dates[rawNumber-1])
	assert.Equal(t, timestamp, dates[rawNumber-1])
	g.Deletes("raw", "")
}

//TestGorjunServer_SameTemplateUpload will upload
//same template twice, old template should deleted
func TestGorjunServer_SameTemplateUpload(t *testing.T) {
	g := NewGorjunServer()
	templateVersions := []string{"0.1.6", "0.1.7", "0.1.9", "0.1.10", "0.1.11"}
	for _, version := range templateVersions {
		resp, _ := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/info?name=%s&version=%s", g.Hostname, "nginx", version))
		if resp.StatusCode != http.StatusOK {
			fmt.Println("Test can't be run because templates should uploaded")
			return
		}
	}
	TestGorjunServer_GetLatestTemplateByVersion(t)
	resp, err := http.Get(fmt.Sprintf("http://%s/kurjun/rest/template/list", g.Hostname))
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Errorf("Failed to read body from %s: %v", g.Hostname, err)
	}
	var templateList []GorjunFile
	err = json.Unmarshal(data, &templateList)

	m := make(map[string]int)

	for _, template := range templateList {
		s := template.Owner[0] + template.Name + template.Version
		m[s]++
		assert.NotEqual(t, m[s], 2, "Same template exist twice", template.ID)
	}
}
<|endoftext|>


<commit_before>package ds

import (
	"context"
	"reflect"

	"cloud.google.com/go/datastore"
)

// GetByKey retrieves model from datastore by key
func (client *Client) GetByKey(ctx context.Context, key *datastore.Key, dst interface{}) error {
	if client.Cache != nil && client.Cache.Get(key, dst) == nil {
		return nil
	}
	err := client.Get(ctx, key, dst)
	SetKey(key, dst)
	if client.Cache != nil {
		client.Cache.Set(key, dst)
	}
	if err != nil {
		return err
	}
	return nil
}

// GetByKeys retrieves models from datastore by keys
func (client *Client) GetByKeys(ctx context.Context, keys []*datastore.Key, dst interface{}) error {
	// prepare slice if dst is pointer to 0 len slice
	if rf := reflect.ValueOf(dst); rf.Kind() == reflect.Ptr {
		rs := rf.Elem()
		if rs.Kind() == reflect.Slice && rs.Len() == 0 {
			l := len(keys)
			rs.Set(reflect.MakeSlice(rs.Type(), l, l))
		}
		dst = rs.Interface()
	}

	if client.Cache != nil {
		err := client.Cache.GetMulti(keys, dst)
		if err == nil {
			nfKeys := []*datastore.Key{}
			nfMap := []int{}
			rf := valueOf(dst)
			for i := 0; i < rf.Len(); i++ {
				if rf.Index(i).IsNil() {
					nfKeys = append(nfKeys, keys[i])
					nfMap = append(nfMap, i)
				}
			}
			l := len(nfKeys)
			nfDstRf := reflect.MakeSlice(rf.Type(), l, l)
			err := client.GetMulti(ctx, keys, nfDstRf.Interface())
			for i, k := range nfMap {
				rf.Index(k).Set(nfDstRf.Index(i))
			}
			SetKeys(keys, dst)
			if err != nil {
				return err
			}
			return nil
		}
	}

	var err error
	l := len(keys)
	p := 1000
	if l > p {
		rfDst := valueOf(dst)
		for i := 0; i < l/p+1; i++ {
			m := (i + 1) * p
			if l-m+1 < p {
				m = l
			}
			if i*p == m {
				break
			}
			e := client.GetMulti(ctx, keys[i*p:m], rfDst.Slice(i*p, m).Interface())
			if e != nil {
				if err == nil {
					err = e
				} else {
					if errs, ok := err.(datastore.MultiError); ok {
						err = append(errs, e)
					} else {
						err = datastore.MultiError{err, e}
					}
				}
			}
		}
	} else {
		err = client.GetMulti(ctx, keys, dst)
	}
	SetKeys(keys, dst)
	if client.Cache != nil {
		client.Cache.SetMulti(keys, dst)
	}
	if err != nil {
		return err
	}
	return nil
}

// GetByModel retrieves model from datastore by key from model
func (client *Client) GetByModel(ctx context.Context, dst interface{}) error {
	key := ExtractKey(dst)
	return client.GetByKey(ctx, key, dst)
}

// GetByModels retrieves models from datastore by keys from models
func (client *Client) GetByModels(ctx context.Context, dst interface{}) error {
	keys := ExtractKeys(dst)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByID retrieves model from datastore by id
func (client *Client) GetByID(ctx context.Context, kind string, id int64, dst interface{}) error {
	return client.GetByKey(ctx, datastore.IDKey(kind, id, nil), dst)
}

// GetByIDs retrieves models from datastore by ids
func (client *Client) GetByIDs(ctx context.Context, kind string, ids []int64, dst interface{}) error {
	keys := BuildIDKeys(kind, ids)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByStringID retrieves model from datastore by string id
func (client *Client) GetByStringID(ctx context.Context, kind string, id string, dst interface{}) error {
	tid := parseID(id)
	if tid == 0 {
		return datastore.ErrInvalidKey
	}
	return client.GetByKey(ctx, datastore.IDKey(kind, tid, nil), dst)
}

// GetByStringIDs retrieves models from datastore by string ids
func (client *Client) GetByStringIDs(ctx context.Context, kind string, ids []string, dst interface{}) error {
	keys := BuildStringIDKeys(kind, ids)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByName retrieves model from datastore by name
func (client *Client) GetByName(ctx context.Context, kind string, name string, dst interface{}) error {
	return client.GetByKey(ctx, datastore.NameKey(kind, name, nil), dst)
}

// GetByNames retrieves models from datastore by names
func (client *Client) GetByNames(ctx context.Context, kind string, names []string, dst interface{}) error {
	keys := BuildNameKeys(kind, names)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByQuery retrieves model from datastore by datastore query
func (client *Client) GetByQuery(ctx context.Context, q *datastore.Query, dst interface{}) error {
	_, err := client.GetAll(ctx, q, dst)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>fix multi<commit_after>package ds

import (
	"context"
	"reflect"

	"cloud.google.com/go/datastore"
)

// GetByKey retrieves model from datastore by key
func (client *Client) GetByKey(ctx context.Context, key *datastore.Key, dst interface{}) error {
	if client.Cache != nil && client.Cache.Get(key, dst) == nil {
		return nil
	}
	err := client.Get(ctx, key, dst)
	SetKey(key, dst)
	if client.Cache != nil {
		client.Cache.Set(key, dst)
	}
	if err != nil {
		return err
	}
	return nil
}

func (client *Client) getByKeys(ctx context.Context, keys []*datastore.Key, dst interface{}) error {
	var err error
	l := len(keys)
	p := 1000
	if l > p {
		rfDst := valueOf(dst)
		for i := 0; i < l/p+1; i++ {
			m := (i + 1) * p
			if m > l {
				m = l
			}
			if i*p == m {
				break
			}
			e := client.GetMulti(ctx, keys[i*p:m], rfDst.Slice(i*p, m).Interface())
			if e != nil {
				if err == nil {
					err = e
				} else {
					if errs, ok := err.(datastore.MultiError); ok {
						err = append(errs, e)
					} else {
						err = datastore.MultiError{err, e}
					}
				}
			}
		}
	} else {
		err = client.GetMulti(ctx, keys, dst)
	}
	SetKeys(keys, dst)
	if client.Cache != nil {
		client.Cache.SetMulti(keys, dst)
	}
	if err != nil {
		return err
	}
	return nil
}

// GetByKeys retrieves models from datastore by keys
func (client *Client) GetByKeys(ctx context.Context, keys []*datastore.Key, dst interface{}) error {
	// prepare slice if dst is pointer to 0 len slice
	if rf := reflect.ValueOf(dst); rf.Kind() == reflect.Ptr {
		rs := rf.Elem()
		if rs.Kind() == reflect.Slice && rs.Len() == 0 {
			l := len(keys)
			rs.Set(reflect.MakeSlice(rs.Type(), l, l))
		}
		dst = rs.Interface()
	}

	if client.Cache != nil {
		err := client.Cache.GetMulti(keys, dst)
		if err == nil {
			nfKeys := []*datastore.Key{}
			nfMap := []int{}
			rf := valueOf(dst)
			for i := 0; i < rf.Len(); i++ {
				if rf.Index(i).IsNil() {
					nfKeys = append(nfKeys, keys[i])
					nfMap = append(nfMap, i)
				}
			}
			l := len(nfKeys)
			nfDstRf := reflect.MakeSlice(rf.Type(), l, l)
			err := client.getByKeys(ctx, keys, nfDstRf.Interface())
			for i, k := range nfMap {
				rf.Index(k).Set(nfDstRf.Index(i))
			}
			SetKeys(keys, dst)
			if err != nil {
				return err
			}
			return nil
		}
	}

	return client.getByKeys(ctx, keys, dst)
}

// GetByModel retrieves model from datastore by key from model
func (client *Client) GetByModel(ctx context.Context, dst interface{}) error {
	key := ExtractKey(dst)
	return client.GetByKey(ctx, key, dst)
}

// GetByModels retrieves models from datastore by keys from models
func (client *Client) GetByModels(ctx context.Context, dst interface{}) error {
	keys := ExtractKeys(dst)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByID retrieves model from datastore by id
func (client *Client) GetByID(ctx context.Context, kind string, id int64, dst interface{}) error {
	return client.GetByKey(ctx, datastore.IDKey(kind, id, nil), dst)
}

// GetByIDs retrieves models from datastore by ids
func (client *Client) GetByIDs(ctx context.Context, kind string, ids []int64, dst interface{}) error {
	keys := BuildIDKeys(kind, ids)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByStringID retrieves model from datastore by string id
func (client *Client) GetByStringID(ctx context.Context, kind string, id string, dst interface{}) error {
	tid := parseID(id)
	if tid == 0 {
		return datastore.ErrInvalidKey
	}
	return client.GetByKey(ctx, datastore.IDKey(kind, tid, nil), dst)
}

// GetByStringIDs retrieves models from datastore by string ids
func (client *Client) GetByStringIDs(ctx context.Context, kind string, ids []string, dst interface{}) error {
	keys := BuildStringIDKeys(kind, ids)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByName retrieves model from datastore by name
func (client *Client) GetByName(ctx context.Context, kind string, name string, dst interface{}) error {
	return client.GetByKey(ctx, datastore.NameKey(kind, name, nil), dst)
}

// GetByNames retrieves models from datastore by names
func (client *Client) GetByNames(ctx context.Context, kind string, names []string, dst interface{}) error {
	keys := BuildNameKeys(kind, names)
	return client.GetByKeys(ctx, keys, dst)
}

// GetByQuery retrieves model from datastore by datastore query
func (client *Client) GetByQuery(ctx context.Context, q *datastore.Query, dst interface{}) error {
	keys, err := client.GetAll(ctx, q.KeysOnly(), nil)
	if err != nil {
		return err
	}
	return client.GetByKeys(ctx, keys, dst)
}
<|endoftext|>


<commit_before>package creeperkeeper

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const baseURL = "https://vine.co"
const vineDateFormat = "2006-01-02T15:04:05.999999"

var fallbackUUIDCount = 0
var uuidMutex = sync.Mutex{}

var userURLRE = regexp.MustCompile(`(?:https?://)?vine\.co/(u/)?([^/]+)(/likes)?/?(\?.*)?$`)
var vineURLRE = regexp.MustCompile(`https://vine\.co/v/([a-zA-Z0-9]+)$`)

type vineExtractor func(url string) (vines []Vine, err error)

// DownloadVines downloads vines to files named after their shortIDs, eg
// bnmHnwVILKD.mp4.
func DownloadVines(vines []Vine) error {
	f := func(i interface{}) (err error) {
		vine := i.(Vine)
		file, err := os.Create(vine.VideoFilename())
		if err != nil {
			return err
		}
		defer func() {
			if cerr := file.Close(); err == nil && cerr != nil {
				err = cerr
			}
		}()
		err = vine.Download(file)
		if err != nil {
			log.Printf("get [%s] \"%.20s\": %s", vine.Uploader, vine.Title, err)
		} else if Verbose {
			log.Printf("got [%s] %s", vine.Uploader, vine.Title)
		}
		return err
	}

	// Convert []Vine to []interface{}
	jobs := make([]interface{}, len(vines))
	for i, v := range vines {
		jobs[i] = v
	}

	nerr := parallel(jobs, f, 4)
	if nerr > 0 {
		return fmt.Errorf("%d/%d failed", nerr, len(vines))
	}
	return nil
}

// ExtractVines gets vine metadata related to a url for a single vine, a user
// profile, or a user's likes. API requests are made as necessary to get all of
// a user's posts or likes.
func ExtractVines(url string) (vines []Vine, err error) {
	extractors := []vineExtractor{
		vineExtractor(vineURLToVines),
		vineExtractor(userURLToVines),
	}
	var errors []string
	for _, extractor := range extractors {
		vines, err := extractor(url)
		if err != nil {
			errors = append(errors, err.Error())
			continue
		}
		return vines, nil
	}
	return nil, fmt.Errorf("vine extraction: %s", strings.Join(errors, ", "))
}

// vineURLToVines GETs url, which is expected to the the url for a single vine,
// and extracts the vine's metadata from some JSON embedded in it.
func vineURLToVines(url string) (vines []Vine, err error) {
	matched, err := regexp.MatchString(`https?://(?:www\.)?vine\.co/(?:v|oembed)/(?P<id>\w+)`, url)
	if !matched {
		return nil, fmt.Errorf("url must be for an individual vine: %s", url)
	}
	if err != nil {
		return nil, err
	}
	// There doesn't seem to be an api endpoint for vines, so look in the html.
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return vineHTMLToVines(body)
}

func vineHTMLToVines(html []byte) (vines []Vine, err error) {
	jsonPattern := regexp.MustCompile(`window\.POST_DATA\s*=\s*({.+?});\s*</script>`)
	m := jsonPattern.FindSubmatch(html)
	if len(m) == 0 {
		return nil, fmt.Errorf("no vine metadata found in html")
	}
	var jm jsonMap
	err = json.Unmarshal(m[1], &jm)
	if err != nil {
		return nil, err
	}
	var jvine jsonVine
	for _, jvine = range jm { // Get first value from map.
		break
	}
	for _, jurl := range jvine.VideoURLs {
		if jurl.ID == "original" {
			created, err := time.Parse(vineDateFormat, jvine.Created)
			if err != nil {
				return nil, err
			}
			vines = append(vines, Vine{
				Title:      jvine.Description,
				Uploader:   jvine.Username,
				UploaderID: strconv.FormatInt(jvine.UserID, 10),
				URL:        jurl.VideoURL,
				UUID:       jvine.ShortID,
				Created:    created,
				Venue:      jvine.VenueName,
			})
			return vines, nil
		}
	}
	return nil, nil
}

func userURLToVines(url string) (vines []Vine, err error) {
	userID, err := userURLToUserID(url)
	if err != nil {
		return nil, err
	}
	m := userURLRE.FindStringSubmatch(url)
	if len(m[3]) > 0 {
		vines, err = likedVines(userID)
	} else {
		vines, err = postedVines(userID)
	}
	if err != nil {
		return nil, err
	}
	return vines, nil
}

func userURLToUserID(url string) (string, error) {
	m := userURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		return "", fmt.Errorf("unrecognized vine user url: %q", url)
	}
	isVanity := len(m[1]) == 0
	if isVanity {
		profileURL := fmt.Sprintf("%s/api/users/profiles/vanity/%s", baseURL, m[2])
		var ur userResult
		err := deserialize(profileURL, &ur)
		if err != nil {
			return "", err
		}
		return fmt.Sprint(ur.Data.UserID), nil
	} else {
		return m[2], nil
	}
}

func likedVines(userID string) (vines []Vine, err error) {
	url := fmt.Sprintf("%s/api/timelines/users/%s/likes", baseURL, userID)
	return timelineVines(url)
}

func postedVines(userID string) (vines []Vine, err error) {
	url := fmt.Sprintf("%s/api/timelines/users/%s", baseURL, userID)
	return timelineVines(url)
}

func timelineVines(url string) (vines []Vine, err error) {
	more := true
	for i := 1; more; i++ {
		urlWithParams := fmt.Sprintf("%s?page=%d&size=100", url, i)
		var pageVines []Vine
		pageVines, more, err = timelinePageVines(urlWithParams)
		if err != nil {
			return nil, err
		}
		vines = append(vines, pageVines...)
	}
	return vines, nil
}

func timelinePageVines(url string) (vines []Vine, more bool, err error) {
	var tr timelineResult
	err = deserialize(url, &tr)
	if err != nil {
		return nil, false, err
	}
	for _, tv := range tr.Data.Records {
		created, err := time.Parse(vineDateFormat, tv.Created)
		if err != nil {
			return nil, false, err
		}
		vines = append(vines, Vine{
			Title:      tv.Description,
			Uploader:   tv.Username,
			UploaderID: strconv.FormatInt(tv.UserID, 10),
			URL:        tv.VideoURL,
			UUID:       vineURLToUUID(tv.PermalinkURL),
			Created:    created,
			Venue:      tv.VenueName,
		})
	}
	more = tr.Data.NextPage > 0
	return vines, more, nil
}

// deserialize GETs a JSON API endpoint, unwraps the enveloping object and
// unmarshals the response.
func deserialize(url string, d interface{}) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var env jsonVineEnvelope
	err = json.Unmarshal(body, &env)
	if err != nil {
		return fmt.Errorf("unrecognized json: %s", body)
	}
	if !env.Success {
		return fmt.Errorf("GET %q: status %d: %s", url, resp.StatusCode, env.Error)
	}
	err = json.Unmarshal(body, &d)
	if err != nil {
		return err
	}
	return nil
}

func vineURLToUUID(url string) string {
	m := vineURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		uuidMutex.Lock()
		defer uuidMutex.Unlock()
		return fmt.Sprintf("fallbackID%d", fallbackUUIDCount)
	}
	return string(m[1])
}

func FilterOutReposts(vines []Vine, url string) ([]Vine, error) {
	userid, err := userURLToUserID(url)
	if err != nil {
		return vines, err
	}
	filtered := []Vine{}
	for _, vine := range vines {
		if vine.UploaderID != userid {
			continue
		}
		filtered = append(filtered, vine)
	}
	return filtered, nil
}

type jsonVineEnvelope struct {
	Success bool
	Error   string
}

// Single-Vine JSON structures
type jsonMap map[string]jsonVine
type jsonVine struct {
	Username    string
	UserID      int64
	Description string
	ShortID     string
	VideoURLs   []jsonVideoURL
	VenueName   string
	Created     string
}
type jsonVideoURL struct {
	VideoURL string
	ID       string
}

// Timeline (posts/likes) API JSON structures
type timelineResult struct {
	Data    timelineRecords
	Success bool
	Error   string
}
type timelineRecords struct {
	Records  []timelineVine
	NextPage int
}
type timelineVine struct {
	Username     string
	Description  string
	VideoURL     string
	PermalinkURL string
	VenueName    string
	Created      string
	UserID       int64
}

// User API JSON structures
type userResult struct {
	Data jsonUser
}
type jsonUser struct {
	UserID int64
}
<commit_msg>Use anchor when requesting timeline pages<commit_after>package creeperkeeper

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const baseURL = "https://vine.co"
const vineDateFormat = "2006-01-02T15:04:05.999999"

var fallbackUUIDCount = 0
var uuidMutex = sync.Mutex{}

var userURLRE = regexp.MustCompile(`(?:https?://)?vine\.co/(u/)?([^/]+)(/likes)?/?(\?.*)?$`)
var vineURLRE = regexp.MustCompile(`https://vine\.co/v/([a-zA-Z0-9]+)$`)

type vineExtractor func(url string) (vines []Vine, err error)

// DownloadVines downloads vines to files named after their shortIDs, eg
// bnmHnwVILKD.mp4.
func DownloadVines(vines []Vine) error {
	f := func(i interface{}) (err error) {
		vine := i.(Vine)
		file, err := os.Create(vine.VideoFilename())
		if err != nil {
			return err
		}
		defer func() {
			if cerr := file.Close(); err == nil && cerr != nil {
				err = cerr
			}
		}()
		err = vine.Download(file)
		if err != nil {
			log.Printf("get [%s] \"%.20s\": %s", vine.Uploader, vine.Title, err)
		} else if Verbose {
			log.Printf("got [%s] %s", vine.Uploader, vine.Title)
		}
		return err
	}

	// Convert []Vine to []interface{}
	jobs := make([]interface{}, len(vines))
	for i, v := range vines {
		jobs[i] = v
	}

	nerr := parallel(jobs, f, 4)
	if nerr > 0 {
		return fmt.Errorf("%d/%d failed", nerr, len(vines))
	}
	return nil
}

// ExtractVines gets vine metadata related to a url for a single vine, a user
// profile, or a user's likes. API requests are made as necessary to get all of
// a user's posts or likes.
func ExtractVines(url string) (vines []Vine, err error) {
	extractors := []vineExtractor{
		vineExtractor(vineURLToVines),
		vineExtractor(userURLToVines),
	}
	var errors []string
	for _, extractor := range extractors {
		vines, err := extractor(url)
		if err != nil {
			errors = append(errors, err.Error())
			continue
		}
		return vines, nil
	}
	return nil, fmt.Errorf("vine extraction: %s", strings.Join(errors, ", "))
}

// vineURLToVines GETs url, which is expected to the the url for a single vine,
// and extracts the vine's metadata from some JSON embedded in it.
func vineURLToVines(url string) (vines []Vine, err error) {
	matched, err := regexp.MatchString(`https?://(?:www\.)?vine\.co/(?:v|oembed)/(?P<id>\w+)`, url)
	if !matched {
		return nil, fmt.Errorf("url must be for an individual vine: %s", url)
	}
	if err != nil {
		return nil, err
	}
	// There doesn't seem to be an api endpoint for vines, so look in the html.
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return vineHTMLToVines(body)
}

func vineHTMLToVines(html []byte) (vines []Vine, err error) {
	jsonPattern := regexp.MustCompile(`window\.POST_DATA\s*=\s*({.+?});\s*</script>`)
	m := jsonPattern.FindSubmatch(html)
	if len(m) == 0 {
		return nil, fmt.Errorf("no vine metadata found in html")
	}
	var jm jsonMap
	err = json.Unmarshal(m[1], &jm)
	if err != nil {
		return nil, err
	}
	var jvine jsonVine
	for _, jvine = range jm { // Get first value from map.
		break
	}
	for _, jurl := range jvine.VideoURLs {
		if jurl.ID == "original" {
			created, err := time.Parse(vineDateFormat, jvine.Created)
			if err != nil {
				return nil, err
			}
			vines = append(vines, Vine{
				Title:      jvine.Description,
				Uploader:   jvine.Username,
				UploaderID: strconv.FormatInt(jvine.UserID, 10),
				URL:        jurl.VideoURL,
				UUID:       jvine.ShortID,
				Created:    created,
				Venue:      jvine.VenueName,
			})
			return vines, nil
		}
	}
	return nil, nil
}

func userURLToVines(url string) (vines []Vine, err error) {
	userID, err := userURLToUserID(url)
	if err != nil {
		return nil, err
	}
	m := userURLRE.FindStringSubmatch(url)
	if len(m[3]) > 0 {
		vines, err = likedVines(userID)
	} else {
		vines, err = postedVines(userID)
	}
	if err != nil {
		return nil, err
	}
	return vines, nil
}

func userURLToUserID(url string) (string, error) {
	m := userURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		return "", fmt.Errorf("unrecognized vine user url: %q", url)
	}
	isVanity := len(m[1]) == 0
	if isVanity {
		profileURL := fmt.Sprintf("%s/api/users/profiles/vanity/%s", baseURL, m[2])
		var ur userResult
		err := deserialize(profileURL, &ur)
		if err != nil {
			return "", err
		}
		return fmt.Sprint(ur.Data.UserID), nil
	} else {
		return m[2], nil
	}
}

func likedVines(userID string) (vines []Vine, err error) {
	url := fmt.Sprintf("%s/api/timelines/users/%s/likes", baseURL, userID)
	return timelineVines(url)
}

func postedVines(userID string) (vines []Vine, err error) {
	url := fmt.Sprintf("%s/api/timelines/users/%s", baseURL, userID)
	return timelineVines(url)
}

func timelineVines(url string) (vines []Vine, err error) {
	anchor := ""
	for i := 1; anchor != "" || i == 1; i++ {
		urlWithParams := fmt.Sprintf("%s?page=%d&anchor=%s&size=100", url, i, anchor)
		var pageVines []Vine
		pageVines, anchor, err = timelinePageVines(urlWithParams)
		if err != nil {
			return nil, err
		}
		if Verbose {
			log.Printf("page %d of metadata has %d vines", i, len(pageVines))
		}
		vines = append(vines, pageVines...)
	}
	if Verbose {
		log.Printf("got metadata for %d vines", len(vines))
	}
	return vines, nil
}

func timelinePageVines(url string) (vines []Vine, anchor string, err error) {
	var tr timelineResult
	err = deserialize(url, &tr)
	if err != nil {
		return nil, "", err
	}
	for _, tv := range tr.Data.Records {
		created, err := time.Parse(vineDateFormat, tv.Created)
		if err != nil {
			return nil, "", err
		}
		vines = append(vines, Vine{
			Title:      tv.Description,
			Uploader:   tv.Username,
			UploaderID: strconv.FormatInt(tv.UserID, 10),
			URL:        tv.VideoURL,
			UUID:       vineURLToUUID(tv.PermalinkURL),
			Created:    created,
			Venue:      tv.VenueName,
		})
	}
	return vines, tr.Data.AnchorStr, nil
}

// deserialize GETs a JSON API endpoint, unwraps the enveloping object and
// unmarshals the response.
func deserialize(url string, d interface{}) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var env jsonVineEnvelope
	err = json.Unmarshal(body, &env)
	if err != nil {
		return fmt.Errorf("unrecognized json: %s", body)
	}
	if !env.Success {
		return fmt.Errorf("GET %q: status %d: %s", url, resp.StatusCode, env.Error)
	}
	err = json.Unmarshal(body, &d)
	if err != nil {
		return err
	}
	return nil
}

func vineURLToUUID(url string) string {
	m := vineURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		uuidMutex.Lock()
		defer uuidMutex.Unlock()
		return fmt.Sprintf("fallbackID%d", fallbackUUIDCount)
	}
	return string(m[1])
}

func FilterOutReposts(vines []Vine, url string) ([]Vine, error) {
	userid, err := userURLToUserID(url)
	if err != nil {
		return vines, err
	}
	filtered := []Vine{}
	for _, vine := range vines {
		if vine.UploaderID != userid {
			continue
		}
		filtered = append(filtered, vine)
	}
	return filtered, nil
}

type jsonVineEnvelope struct {
	Success bool
	Error   string
}

// Single-Vine JSON structures
type jsonMap map[string]jsonVine
type jsonVine struct {
	Username    string
	UserID      int64
	Description string
	ShortID     string
	VideoURLs   []jsonVideoURL
	VenueName   string
	Created     string
}
type jsonVideoURL struct {
	VideoURL string
	ID       string
}

// Timeline (posts/likes) API JSON structures
type timelineResult struct {
	Data    timelineRecords
	Success bool
	Error   string
}
type timelineRecords struct {
	Records   []timelineVine
	AnchorStr string
}
type timelineVine struct {
	Username     string
	Description  string
	VideoURL     string
	PermalinkURL string
	VenueName    string
	Created      string
	UserID       int64
}

// User API JSON structures
type userResult struct {
	Data jsonUser
}
type jsonUser struct {
	UserID int64
}
<|endoftext|>


<commit_before>package device

import (
	"bufio"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/lxc/lxd/shared"
)

// NetworkSysctlGet retrieves the value of a sysctl file in /proc/sys/net.
func NetworkSysctlGet(path string) (string, error) {
	// Read the current content
	content, err := ioutil.ReadFile(fmt.Sprintf("/proc/sys/net/%s", path))
	if err != nil {
		return "", err
	}

	return string(content), nil
}

// NetworkSysctlSet writes a value to a sysctl file in /proc/sys/net.
func NetworkSysctlSet(path string, value string) error {
	// Get current value
	current, err := NetworkSysctlGet(path)
	if err == nil && current == value {
		// Nothing to update
		return nil
	}

	return ioutil.WriteFile(fmt.Sprintf("/proc/sys/net/%s", path), []byte(value), 0)
}

// NetworkGetDevMTU retrieves the current MTU setting for a named network device.
func NetworkGetDevMTU(devName string) (uint64, error) {
	content, err := ioutil.ReadFile(fmt.Sprintf("/sys/class/net/%s/mtu", devName))
	if err != nil {
		return 0, err
	}

	// Parse value
	mtu, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 32)
	if err != nil {
		return 0, err
	}

	return mtu, nil
}

// NetworkSetDevMTU sets the MTU setting for a named network device if different from current.
func NetworkSetDevMTU(devName string, mtu uint64) error {
	curMTU, err := NetworkGetDevMTU(devName)
	if err != nil {
		return err
	}

	// Only try and change the MTU if the requested mac is different to current one.
	if curMTU != mtu {
		_, err := shared.RunCommand("ip", "link", "set", "dev", devName, "mtu", fmt.Sprintf("%d", mtu))
		if err != nil {
			return err
		}
	}

	return nil
}

// NetworkGetDevMAC retrieves the current MAC setting for a named network device.
func NetworkGetDevMAC(devName string) (string, error) {
	content, err := ioutil.ReadFile(fmt.Sprintf("/sys/class/net/%s/address", devName))
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(fmt.Sprintf("%s", content)), nil
}

// NetworkSetDevMAC sets the MAC setting for a named network device if different from current.
func NetworkSetDevMAC(devName string, mac string) error {
	curMac, err := NetworkGetDevMAC(devName)
	if err != nil {
		return err
	}

	// Only try and change the MAC if the requested mac is different to current one.
	if curMac != mac {
		_, err := shared.RunCommand("ip", "link", "set", "dev", devName, "address", mac)
		if err != nil {
			return err
		}
	}

	return nil
}

// NetworkGetHostDevice figures out whether there is an existing interface for the supplied
// parent device and VLAN ID and returns it. Otherwise just returns the parent device name.
func NetworkGetHostDevice(parent string, vlan string) string {
	// If no VLAN, just use the raw device
	if vlan == "" {
		return parent
	}

	// If no VLANs are configured, use the default pattern
	defaultVlan := fmt.Sprintf("%s.%s", parent, vlan)
	if !shared.PathExists("/proc/net/vlan/config") {
		return defaultVlan
	}

	// Look for an existing VLAN
	f, err := os.Open("/proc/net/vlan/config")
	if err != nil {
		return defaultVlan
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		// Only grab the lines we're interested in
		s := strings.Split(scanner.Text(), "|")
		if len(s) != 3 {
			continue
		}

		vlanIface := strings.TrimSpace(s[0])
		vlanID := strings.TrimSpace(s[1])
		vlanParent := strings.TrimSpace(s[2])

		if vlanParent == parent && vlanID == vlan {
			return vlanIface
		}
	}

	// Return the default pattern
	return defaultVlan
}

// NetworkRemoveInterface removes a network interface by name.
func NetworkRemoveInterface(nic string) error {
	_, err := shared.RunCommand("ip", "link", "del", "dev", nic)
	return err
}

// NetworkCreateVlanDeviceIfNeeded creates a VLAN device if doesn't already exist.
func NetworkCreateVlanDeviceIfNeeded(parent string, vlanDevice string, vlanID string) (bool, error) {
	if vlanID != "" {
		if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", vlanDevice)) {
			// Bring the parent interface up so we can add a vlan to it.
			_, err := shared.RunCommand("ip", "link", "set", "dev", parent, "up")
			if err != nil {
				return false, fmt.Errorf("Failed to bring up parent %s: %v", parent, err)
			}

			// Add VLAN interface on top of parent.
			_, err = shared.RunCommand("ip", "link", "add", "link", parent, "name", vlanDevice, "up", "type", "vlan", "id", vlanID)
			if err != nil {
				return false, err
			}

			// Attempt to disable IPv6 router advertisement acceptance
			NetworkSysctlSet(fmt.Sprintf("ipv6/conf/%s/accept_ra", vlanDevice), "0")

			// We created a new vlan interface, return true
			return true, nil
		}
	}

	return false, nil
}

// networkSnapshotPhysicalNic records properties of the NIC to volatile so they can be restored later.
func networkSnapshotPhysicalNic(hostName string, volatile map[string]string) error {
	// Store current MTU for restoration on detach.
	mtu, err := NetworkGetDevMTU(hostName)
	if err != nil {
		return err
	}
	volatile["last_state.mtu"] = fmt.Sprintf("%d", mtu)

	// Store current MAC for restoration on detach
	mac, err := NetworkGetDevMAC(hostName)
	if err != nil {
		return err
	}
	volatile["last_state.hwaddr"] = mac
	return nil
}

// networkRestorePhysicalNic restores NIC properties from volatile to what they were before it was attached.
func networkRestorePhysicalNic(hostName string, volatile map[string]string) error {
	// If we created the "physical" device and then it should be removed.
	if shared.IsTrue(volatile["last_state.created"]) {
		return NetworkRemoveInterface(hostName)
	}

	// Bring the interface down, as this is sometimes needed to change settings on the nic.
	_, err := shared.RunCommand("ip", "link", "set", "dev", hostName, "down")
	if err != nil {
		return fmt.Errorf("Failed to bring down \"%s\": %v", hostName, err)
	}

	// If MTU value is specified then there is an original MTU that needs restoring.
	if volatile["last_state.mtu"] != "" {
		mtuInt, err := strconv.ParseUint(volatile["last_state.mtu"], 10, 32)
		if err != nil {
			return fmt.Errorf("Failed to convert mtu for \"%s\" mtu \"%s\": %v", hostName, volatile["last_state.mtu"], err)
		}

		err = NetworkSetDevMTU(hostName, mtuInt)
		if err != nil {
			return fmt.Errorf("Failed to restore physical dev \"%s\" mtu to \"%d\": %v", hostName, mtuInt, err)
		}
	}

	// If MAC value is specified then there is an original MAC that needs restoring.
	if volatile["last_state.hwaddr"] != "" {
		err := NetworkSetDevMAC(hostName, volatile["last_state.hwaddr"])
		if err != nil {
			return fmt.Errorf("Failed to restore physical dev \"%s\" mac to \"%s\": %v", hostName, volatile["last_state.hwaddr"], err)
		}
	}

	return nil
}

// NetworkRandomDevName returns a random device name with prefix.
// If the random string combined with the prefix exceeds 13 characters then empty string is returned.
// This is to ensure we support buggy dhclient applications: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=858580
func NetworkRandomDevName(prefix string) string {
	// Return a new random veth device name
	randBytes := make([]byte, 4)
	rand.Read(randBytes)
	iface := prefix + hex.EncodeToString(randBytes)
	if len(iface) > 13 {
		return ""
	}

	return iface
}

// NetworkAttachInterface attaches an interface to a bridge.
func NetworkAttachInterface(netName string, devName string) error {
	if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", netName)) {
		_, err := shared.RunCommand("ip", "link", "set", "dev", devName, "master", netName)
		if err != nil {
			return err
		}
	} else {
		_, err := shared.RunCommand("ovs-vsctl", "port-to-br", devName)
		if err != nil {
			_, err := shared.RunCommand("ovs-vsctl", "add-port", netName, devName)
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>device/utils: Adds veth management functions<commit_after>package device

import (
	"bufio"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/units"
)

// NetworkSysctlGet retrieves the value of a sysctl file in /proc/sys/net.
func NetworkSysctlGet(path string) (string, error) {
	// Read the current content
	content, err := ioutil.ReadFile(fmt.Sprintf("/proc/sys/net/%s", path))
	if err != nil {
		return "", err
	}

	return string(content), nil
}

// NetworkSysctlSet writes a value to a sysctl file in /proc/sys/net.
func NetworkSysctlSet(path string, value string) error {
	// Get current value
	current, err := NetworkSysctlGet(path)
	if err == nil && current == value {
		// Nothing to update
		return nil
	}

	return ioutil.WriteFile(fmt.Sprintf("/proc/sys/net/%s", path), []byte(value), 0)
}

// NetworkGetDevMTU retrieves the current MTU setting for a named network device.
func NetworkGetDevMTU(devName string) (uint64, error) {
	content, err := ioutil.ReadFile(fmt.Sprintf("/sys/class/net/%s/mtu", devName))
	if err != nil {
		return 0, err
	}

	// Parse value
	mtu, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 32)
	if err != nil {
		return 0, err
	}

	return mtu, nil
}

// NetworkSetDevMTU sets the MTU setting for a named network device if different from current.
func NetworkSetDevMTU(devName string, mtu uint64) error {
	curMTU, err := NetworkGetDevMTU(devName)
	if err != nil {
		return err
	}

	// Only try and change the MTU if the requested mac is different to current one.
	if curMTU != mtu {
		_, err := shared.RunCommand("ip", "link", "set", "dev", devName, "mtu", fmt.Sprintf("%d", mtu))
		if err != nil {
			return err
		}
	}

	return nil
}

// NetworkGetDevMAC retrieves the current MAC setting for a named network device.
func NetworkGetDevMAC(devName string) (string, error) {
	content, err := ioutil.ReadFile(fmt.Sprintf("/sys/class/net/%s/address", devName))
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(fmt.Sprintf("%s", content)), nil
}

// NetworkSetDevMAC sets the MAC setting for a named network device if different from current.
func NetworkSetDevMAC(devName string, mac string) error {
	curMac, err := NetworkGetDevMAC(devName)
	if err != nil {
		return err
	}

	// Only try and change the MAC if the requested mac is different to current one.
	if curMac != mac {
		_, err := shared.RunCommand("ip", "link", "set", "dev", devName, "address", mac)
		if err != nil {
			return err
		}
	}

	return nil
}

// NetworkGetHostDevice figures out whether there is an existing interface for the supplied
// parent device and VLAN ID and returns it. Otherwise just returns the parent device name.
func NetworkGetHostDevice(parent string, vlan string) string {
	// If no VLAN, just use the raw device
	if vlan == "" {
		return parent
	}

	// If no VLANs are configured, use the default pattern
	defaultVlan := fmt.Sprintf("%s.%s", parent, vlan)
	if !shared.PathExists("/proc/net/vlan/config") {
		return defaultVlan
	}

	// Look for an existing VLAN
	f, err := os.Open("/proc/net/vlan/config")
	if err != nil {
		return defaultVlan
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		// Only grab the lines we're interested in
		s := strings.Split(scanner.Text(), "|")
		if len(s) != 3 {
			continue
		}

		vlanIface := strings.TrimSpace(s[0])
		vlanID := strings.TrimSpace(s[1])
		vlanParent := strings.TrimSpace(s[2])

		if vlanParent == parent && vlanID == vlan {
			return vlanIface
		}
	}

	// Return the default pattern
	return defaultVlan
}

// NetworkRemoveInterface removes a network interface by name.
func NetworkRemoveInterface(nic string) error {
	_, err := shared.RunCommand("ip", "link", "del", "dev", nic)
	return err
}

// NetworkCreateVlanDeviceIfNeeded creates a VLAN device if doesn't already exist.
func NetworkCreateVlanDeviceIfNeeded(parent string, vlanDevice string, vlanID string) (bool, error) {
	if vlanID != "" {
		if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", vlanDevice)) {
			// Bring the parent interface up so we can add a vlan to it.
			_, err := shared.RunCommand("ip", "link", "set", "dev", parent, "up")
			if err != nil {
				return false, fmt.Errorf("Failed to bring up parent %s: %v", parent, err)
			}

			// Add VLAN interface on top of parent.
			_, err = shared.RunCommand("ip", "link", "add", "link", parent, "name", vlanDevice, "up", "type", "vlan", "id", vlanID)
			if err != nil {
				return false, err
			}

			// Attempt to disable IPv6 router advertisement acceptance
			NetworkSysctlSet(fmt.Sprintf("ipv6/conf/%s/accept_ra", vlanDevice), "0")

			// We created a new vlan interface, return true
			return true, nil
		}
	}

	return false, nil
}

// networkSnapshotPhysicalNic records properties of the NIC to volatile so they can be restored later.
func networkSnapshotPhysicalNic(hostName string, volatile map[string]string) error {
	// Store current MTU for restoration on detach.
	mtu, err := NetworkGetDevMTU(hostName)
	if err != nil {
		return err
	}
	volatile["last_state.mtu"] = fmt.Sprintf("%d", mtu)

	// Store current MAC for restoration on detach
	mac, err := NetworkGetDevMAC(hostName)
	if err != nil {
		return err
	}
	volatile["last_state.hwaddr"] = mac
	return nil
}

// networkRestorePhysicalNic restores NIC properties from volatile to what they were before it was attached.
func networkRestorePhysicalNic(hostName string, volatile map[string]string) error {
	// If we created the "physical" device and then it should be removed.
	if shared.IsTrue(volatile["last_state.created"]) {
		return NetworkRemoveInterface(hostName)
	}

	// Bring the interface down, as this is sometimes needed to change settings on the nic.
	_, err := shared.RunCommand("ip", "link", "set", "dev", hostName, "down")
	if err != nil {
		return fmt.Errorf("Failed to bring down \"%s\": %v", hostName, err)
	}

	// If MTU value is specified then there is an original MTU that needs restoring.
	if volatile["last_state.mtu"] != "" {
		mtuInt, err := strconv.ParseUint(volatile["last_state.mtu"], 10, 32)
		if err != nil {
			return fmt.Errorf("Failed to convert mtu for \"%s\" mtu \"%s\": %v", hostName, volatile["last_state.mtu"], err)
		}

		err = NetworkSetDevMTU(hostName, mtuInt)
		if err != nil {
			return fmt.Errorf("Failed to restore physical dev \"%s\" mtu to \"%d\": %v", hostName, mtuInt, err)
		}
	}

	// If MAC value is specified then there is an original MAC that needs restoring.
	if volatile["last_state.hwaddr"] != "" {
		err := NetworkSetDevMAC(hostName, volatile["last_state.hwaddr"])
		if err != nil {
			return fmt.Errorf("Failed to restore physical dev \"%s\" mac to \"%s\": %v", hostName, volatile["last_state.hwaddr"], err)
		}
	}

	return nil
}

// NetworkRandomDevName returns a random device name with prefix.
// If the random string combined with the prefix exceeds 13 characters then empty string is returned.
// This is to ensure we support buggy dhclient applications: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=858580
func NetworkRandomDevName(prefix string) string {
	// Return a new random veth device name
	randBytes := make([]byte, 4)
	rand.Read(randBytes)
	iface := prefix + hex.EncodeToString(randBytes)
	if len(iface) > 13 {
		return ""
	}

	return iface
}

// NetworkAttachInterface attaches an interface to a bridge.
func NetworkAttachInterface(netName string, devName string) error {
	if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", netName)) {
		_, err := shared.RunCommand("ip", "link", "set", "dev", devName, "master", netName)
		if err != nil {
			return err
		}
	} else {
		_, err := shared.RunCommand("ovs-vsctl", "port-to-br", devName)
		if err != nil {
			_, err := shared.RunCommand("ovs-vsctl", "add-port", netName, devName)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// networkCreateVethPair creates and configures a veth pair. It accepts the name of the host side
// interface as a parameter and returns the peer interface name.
func networkCreateVethPair(hostName string, m config.Device) (string, error) {
	peerName := NetworkRandomDevName("veth")

	_, err := shared.RunCommand("ip", "link", "add", "dev", hostName, "type", "veth", "peer", "name", peerName)
	if err != nil {
		return "", fmt.Errorf("Failed to create the veth interfaces %s and %s: %s", hostName, peerName, err)
	}

	_, err = shared.RunCommand("ip", "link", "set", "dev", hostName, "up")
	if err != nil {
		NetworkRemoveInterface(hostName)
		return "", fmt.Errorf("Failed to bring up the veth interface %s: %s", hostName, err)
	}

	// Set the MAC address on peer.
	if m["hwaddr"] != "" {
		_, err := shared.RunCommand("ip", "link", "set", "dev", peerName, "address", m["hwaddr"])
		if err != nil {
			NetworkRemoveInterface(peerName)
			return "", fmt.Errorf("Failed to set the MAC address: %s", err)
		}
	}

	// Set the MTU on peer.
	if m["mtu"] != "" {
		_, err := shared.RunCommand("ip", "link", "set", "dev", peerName, "mtu", m["mtu"])
		if err != nil {
			NetworkRemoveInterface(peerName)
			return "", fmt.Errorf("Failed to set the MTU: %s", err)
		}
	}

	return peerName, nil
}

// networkSetupHostVethDevice configures a nic device's host side veth settings.
func networkSetupHostVethDevice(device config.Device, oldDevice config.Device, v map[string]string) error {
	// If not configured, check if volatile data contains the most recently added host_name.
	if device["host_name"] == "" {
		device["host_name"] = v["host_name"]
	}

	// If not configured, check if volatile data contains the most recently added hwaddr.
	if device["hwaddr"] == "" {
		device["hwaddr"] = v["hwaddr"]
	}

	// Check whether host device resolution succeeded.
	if device["host_name"] == "" {
		return fmt.Errorf("Failed to find host side veth name for device \"%s\"", device["name"])
	}

	// Refresh tc limits.
	err := networkSetVethLimits(device)
	if err != nil {
		return err
	}

	// If oldDevice provided, remove old routes if any remain.
	if oldDevice != nil {
		// If not configured, copy the volatile host_name into old device to support live updates.
		if oldDevice["host_name"] == "" {
			oldDevice["host_name"] = v["host_name"]
		}

		// If not configured, copy the volatile host_name into old device to support live updates.
		if oldDevice["hwaddr"] == "" {
			oldDevice["hwaddr"] = v["hwaddr"]
		}

		networkRemoveVethRoutes(oldDevice)
	}

	// Setup static routes to container.
	err = networkSetVethRoutes(device)
	if err != nil {
		return err
	}

	return nil
}

// networkSetVethRoutes applies any static routes configured from the host to the container nic.
func networkSetVethRoutes(m config.Device) error {
	// Decide whether the route should point to the veth parent or the bridge parent.
	routeDev := m["host_name"]
	if m["nictype"] == "bridged" {
		routeDev = m["parent"]
	}

	if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", routeDev)) {
		return fmt.Errorf("Unknown or missing host side route interface: %s", routeDev)
	}

	// Add additional IPv4 routes (using boot proto to avoid conflicts with network static routes)
	if m["ipv4.routes"] != "" {
		for _, route := range strings.Split(m["ipv4.routes"], ",") {
			route = strings.TrimSpace(route)
			_, err := shared.RunCommand("ip", "-4", "route", "add", route, "dev", routeDev, "proto", "boot")
			if err != nil {
				return err
			}
		}
	}

	// Add additional IPv6 routes (using boot proto to avoid conflicts with network static routes)
	if m["ipv6.routes"] != "" {
		for _, route := range strings.Split(m["ipv6.routes"], ",") {
			route = strings.TrimSpace(route)
			_, err := shared.RunCommand("ip", "-6", "route", "add", route, "dev", routeDev, "proto", "boot")
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// networkRemoveVethRoutes removes any routes created for this device on the host that were first added
// with networkSetVethRoutes(). Expects to be passed the device config from the oldExpandedDevices.
func networkRemoveVethRoutes(m config.Device) {
	// Decide whether the route should point to the veth parent or the bridge parent
	routeDev := m["host_name"]
	if m["nictype"] == "bridged" {
		routeDev = m["parent"]
	}

	if m["ipv4.routes"] != "" || m["ipv6.routes"] != "" {
		if routeDev == "" {
			logger.Errorf("Failed to remove static routes as route dev isn't set")
			return
		}

		if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", routeDev)) {
			return //Routes will already be gone if device doesn't exist.
		}
	}

	// Remove IPv4 routes
	if m["ipv4.routes"] != "" {
		for _, route := range strings.Split(m["ipv4.routes"], ",") {
			route = strings.TrimSpace(route)
			_, err := shared.RunCommand("ip", "-4", "route", "flush", route, "dev", routeDev, "proto", "boot")
			if err != nil {
				logger.Errorf("Failed to remove static route: %s to %s: %s", route, routeDev, err)
			}
		}
	}

	// Remove IPv6 routes
	if m["ipv6.routes"] != "" {
		for _, route := range strings.Split(m["ipv6.routes"], ",") {
			route = strings.TrimSpace(route)
			_, err := shared.RunCommand("ip", "-6", "route", "flush", route, "dev", routeDev, "proto", "boot")
			if err != nil {
				logger.Errorf("Failed to remove static route: %s to %s: %s", route, routeDev, err)
			}
		}
	}
}

// networkSetVethLimits applies any network rate limits to the veth device specified in the config.
func networkSetVethLimits(m config.Device) error {
	var err error

	veth := m["host_name"]
	if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", veth)) {
		return fmt.Errorf("Unknown or missing host side veth: %s", veth)
	}

	// Apply max limit
	if m["limits.max"] != "" {
		m["limits.ingress"] = m["limits.max"]
		m["limits.egress"] = m["limits.max"]
	}

	// Parse the values
	var ingressInt int64
	if m["limits.ingress"] != "" {
		ingressInt, err = units.ParseBitSizeString(m["limits.ingress"])
		if err != nil {
			return err
		}
	}

	var egressInt int64
	if m["limits.egress"] != "" {
		egressInt, err = units.ParseBitSizeString(m["limits.egress"])
		if err != nil {
			return err
		}
	}

	// Clean any existing entry
	shared.RunCommand("tc", "qdisc", "del", "dev", veth, "root")
	shared.RunCommand("tc", "qdisc", "del", "dev", veth, "ingress")

	// Apply new limits
	if m["limits.ingress"] != "" {
		out, err := shared.RunCommand("tc", "qdisc", "add", "dev", veth, "root", "handle", "1:0", "htb", "default", "10")
		if err != nil {
			return fmt.Errorf("Failed to create root tc qdisc: %s", out)
		}

		out, err = shared.RunCommand("tc", "class", "add", "dev", veth, "parent", "1:0", "classid", "1:10", "htb", "rate", fmt.Sprintf("%dbit", ingressInt))
		if err != nil {
			return fmt.Errorf("Failed to create limit tc class: %s", out)
		}

		out, err = shared.RunCommand("tc", "filter", "add", "dev", veth, "parent", "1:0", "protocol", "all", "u32", "match", "u32", "0", "0", "flowid", "1:1")
		if err != nil {
			return fmt.Errorf("Failed to create tc filter: %s", out)
		}
	}

	if m["limits.egress"] != "" {
		out, err := shared.RunCommand("tc", "qdisc", "add", "dev", veth, "handle", "ffff:0", "ingress")
		if err != nil {
			return fmt.Errorf("Failed to create ingress tc qdisc: %s", out)
		}

		out, err = shared.RunCommand("tc", "filter", "add", "dev", veth, "parent", "ffff:0", "protocol", "all", "u32", "match", "u32", "0", "0", "police", "rate", fmt.Sprintf("%dbit", egressInt), "burst", "1024k", "mtu", "64kb", "drop")
		if err != nil {
			return fmt.Errorf("Failed to create ingress tc qdisc: %s", out)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package migration

import (
	"fmt"
	"io"
	"net/http"

	backupConfig "github.com/lxc/lxd/lxd/backup/config"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/units"
)

// Info represents the index frame sent if supported.
type Info struct {
	Config *backupConfig.Config `json:"config,omitempty" yaml:"config,omitempty"` // Equivalent of backup.yaml but embedded in index.
}

// InfoResponse represents the response to the index frame sent if supported.
// Right now this doesn't contain anything useful, its just used to indicate receipt of the index header.
// But in the future the itention is to use it allow the target to send back additional information to the source
// about which frames (such as snapshots) it needs for the migration after having inspected the Info index header.
type InfoResponse struct {
	StatusCode int
	Error      string
}

// Err returns the error of the response.
func (r *InfoResponse) Err() error {
	if r.StatusCode != http.StatusOK {
		return api.StatusErrorf(r.StatusCode, r.Error)
	}

	return nil
}

// Type represents the migration transport type. It indicates the method by which the migration can
// take place and what optional features are available.
type Type struct {
	FSType   MigrationFSType // Transport mode selected.
	Features []string        // Feature hints for selected FSType transport mode.
}

// VolumeSourceArgs represents the arguments needed to setup a volume migration source.
type VolumeSourceArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	MultiSync          bool
	FinalSync          bool
	Data               any // Optional store to persist storage driver state between MultiSync phases.
	ContentType        string
	AllowInconsistent  bool
	Refresh            bool
	Info               *Info
}

// VolumeTargetArgs represents the arguments needed to setup a volume migration sink.
type VolumeTargetArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Description        string
	Config             map[string]string
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	Refresh            bool
	Live               bool
	VolumeSize         int64
	ContentType        string
}

// TypesToHeader converts one or more Types to a MigrationHeader. It uses the first type argument
// supplied to indicate the preferred migration method and sets the MigrationHeader's Fs type
// to that. If the preferred type is ZFS then it will also set the header's optional ZfsFeatures.
// If the fallback Rsync type is present in any of the types even if it is not preferred, then its
// optional features are added to the header's RsyncFeatures, allowing for fallback negotiation to
// take place on the farside.
func TypesToHeader(types ...Type) *MigrationHeader {
	missingFeature := false
	hasFeature := true
	var preferredType Type

	if len(types) > 0 {
		preferredType = types[0]
	}

	header := MigrationHeader{Fs: &preferredType.FSType}

	// Add ZFS features if preferred type is ZFS.
	if preferredType.FSType == MigrationFSType_ZFS {
		features := ZfsFeatures{
			Compress: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == ZFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			}
		}

		header.ZfsFeatures = &features
	}

	// Add BTRFS features if preferred type is BTRFS.
	if preferredType.FSType == MigrationFSType_BTRFS {
		features := BtrfsFeatures{
			MigrationHeader:  &missingFeature,
			HeaderSubvolumes: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == BTRFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			} else if feature == BTRFSFeatureSubvolumes {
				features.HeaderSubvolumes = &hasFeature
			} else if feature == BTRFSFeatureSubvolumeUUIDs {
				features.HeaderSubvolumeUuids = &hasFeature
			}
		}

		header.BtrfsFeatures = &features
	}

	// Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list.
	for _, t := range types {
		if t.FSType != MigrationFSType_RSYNC && t.FSType != MigrationFSType_BLOCK_AND_RSYNC {
			continue
		}

		features := RsyncFeatures{
			Xattrs:        &missingFeature,
			Delete:        &missingFeature,
			Compress:      &missingFeature,
			Bidirectional: &missingFeature,
		}

		for _, feature := range t.Features {
			if feature == "xattrs" {
				features.Xattrs = &hasFeature
			} else if feature == "delete" {
				features.Delete = &hasFeature
			} else if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == "bidirectional" {
				features.Bidirectional = &hasFeature
			}
		}

		header.RsyncFeatures = &features
		break // Only use the first rsync transport type found to generate rsync features list.
	}

	return &header
}

// MatchTypes attempts to find matching migration transport types between an offered type sent from a remote
// source and the types supported by a local storage pool. If matches are found then one or more Types are
// returned containing the method and the matching optional features present in both. The function also takes a
// fallback type which is used as an additional offer type preference in case the preferred remote type is not
// compatible with the local type available. It is expected that both sides of the migration will support the
// fallback type for the volume's content type that is being migrated.
func MatchTypes(offer *MigrationHeader, fallbackType MigrationFSType, ourTypes []Type) ([]Type, error) {
	// Generate an offer types slice from the preferred type supplied from remote and the
	// fallback type supplied based on the content type of the transfer.
	offeredFSTypes := []MigrationFSType{offer.GetFs(), fallbackType}

	matchedTypes := []Type{}

	// Find first matching type.
	for _, ourType := range ourTypes {
		for _, offerFSType := range offeredFSTypes {
			if offerFSType != ourType.FSType {
				continue // Not a match, try the next one.
			}

			// We got a match, now extract the relevant offered features.
			var offeredFeatures []string
			if offerFSType == MigrationFSType_ZFS {
				offeredFeatures = offer.GetZfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_BTRFS {
				offeredFeatures = offer.GetBtrfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_RSYNC {
				offeredFeatures = offer.GetRsyncFeaturesSlice()
				if !shared.StringInSlice("bidirectional", offeredFeatures) {
					// If no bi-directional support, this means we are getting a response from
					// an old LXD server that doesn't support bidirectional negotiation, so
					// assume LXD 3.7 level. NOTE: Do NOT extend this list of arguments.
					offeredFeatures = []string{"xattrs", "delete", "compress"}
				}
			}

			// Find common features in both our type and offered type.
			commonFeatures := []string{}
			for _, ourFeature := range ourType.Features {
				if shared.StringInSlice(ourFeature, offeredFeatures) {
					commonFeatures = append(commonFeatures, ourFeature)
				}
			}

			if offer.Refresh != nil && *offer.Refresh == true {
				// Optimized refresh with zfs only works if ZfsFeatureMigrationHeader is available.
				if ourType.FSType == MigrationFSType_ZFS && !shared.StringInSlice(ZFSFeatureMigrationHeader, commonFeatures) {
					continue
				}

				// Optimized refresh with btrfs only works if BtrfsFeatureSubvolumeUUIDs is available.
				if ourType.FSType == MigrationFSType_BTRFS && !shared.StringInSlice(BTRFSFeatureSubvolumeUUIDs, commonFeatures) {
					continue
				}
			}

			// Append type with combined features.
			matchedTypes = append(matchedTypes, Type{
				FSType:   ourType.FSType,
				Features: commonFeatures,
			})
		}
	}

	if len(matchedTypes) < 1 {
		// No matching transport type found, generate an error with offered types and our types.
		offeredTypeStrings := make([]string, 0, len(offeredFSTypes))
		for _, offerFSType := range offeredFSTypes {
			offeredTypeStrings = append(offeredTypeStrings, offerFSType.String())
		}

		ourTypeStrings := make([]string, 0, len(ourTypes))
		for _, ourType := range ourTypes {
			ourTypeStrings = append(ourTypeStrings, ourType.FSType.String())
		}

		return matchedTypes, fmt.Errorf("No matching migration types found. Offered types: %v, our types: %v", offeredTypeStrings, ourTypeStrings)
	}

	return matchedTypes, nil
}

func progressWrapperRender(op *operations.Operation, key string, description string, progressInt int64, speedInt int64) {
	meta := op.Metadata()
	if meta == nil {
		meta = make(map[string]any)
	}

	progress := fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	if description != "" {
		progress = fmt.Sprintf("%s: %s (%s/s)", description, units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	}

	if meta[key] != progress {
		meta[key] = progress
		_ = op.UpdateMetadata(meta)
	}
}

// ProgressReader reports the read progress.
func ProgressReader(op *operations.Operation, key string, description string) func(io.ReadCloser) io.ReadCloser {
	return func(reader io.ReadCloser) io.ReadCloser {
		if op == nil {
			return reader
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		readPipe := &ioprogress.ProgressReader{
			ReadCloser: reader,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return readPipe
	}
}

// ProgressWriter reports the write progress.
func ProgressWriter(op *operations.Operation, key string, description string) func(io.WriteCloser) io.WriteCloser {
	return func(writer io.WriteCloser) io.WriteCloser {
		if op == nil {
			return writer
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		writePipe := &ioprogress.ProgressWriter{
			WriteCloser: writer,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return writePipe
	}
}

// ProgressTracker returns a migration I/O tracker
func ProgressTracker(op *operations.Operation, key string, description string) *ioprogress.ProgressTracker {
	progress := func(progressInt int64, speedInt int64) {
		progressWrapperRender(op, key, description, progressInt, speedInt)
	}

	tracker := &ioprogress.ProgressTracker{
		Handler: progress,
	}

	return tracker
}
<commit_msg>lxd/migration/migration/volumes: Clarify that Config field only used for custom volumes<commit_after>package migration

import (
	"fmt"
	"io"
	"net/http"

	backupConfig "github.com/lxc/lxd/lxd/backup/config"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/units"
)

// Info represents the index frame sent if supported.
type Info struct {
	Config *backupConfig.Config `json:"config,omitempty" yaml:"config,omitempty"` // Equivalent of backup.yaml but embedded in index.
}

// InfoResponse represents the response to the index frame sent if supported.
// Right now this doesn't contain anything useful, its just used to indicate receipt of the index header.
// But in the future the itention is to use it allow the target to send back additional information to the source
// about which frames (such as snapshots) it needs for the migration after having inspected the Info index header.
type InfoResponse struct {
	StatusCode int
	Error      string
}

// Err returns the error of the response.
func (r *InfoResponse) Err() error {
	if r.StatusCode != http.StatusOK {
		return api.StatusErrorf(r.StatusCode, r.Error)
	}

	return nil
}

// Type represents the migration transport type. It indicates the method by which the migration can
// take place and what optional features are available.
type Type struct {
	FSType   MigrationFSType // Transport mode selected.
	Features []string        // Feature hints for selected FSType transport mode.
}

// VolumeSourceArgs represents the arguments needed to setup a volume migration source.
type VolumeSourceArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	MultiSync          bool
	FinalSync          bool
	Data               any // Optional store to persist storage driver state between MultiSync phases.
	ContentType        string
	AllowInconsistent  bool
	Refresh            bool
	Info               *Info
}

// VolumeTargetArgs represents the arguments needed to setup a volume migration sink.
type VolumeTargetArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Description        string
	Config             map[string]string // Only used for custom volume migration.
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	Refresh            bool
	Live               bool
	VolumeSize         int64
	ContentType        string
}

// TypesToHeader converts one or more Types to a MigrationHeader. It uses the first type argument
// supplied to indicate the preferred migration method and sets the MigrationHeader's Fs type
// to that. If the preferred type is ZFS then it will also set the header's optional ZfsFeatures.
// If the fallback Rsync type is present in any of the types even if it is not preferred, then its
// optional features are added to the header's RsyncFeatures, allowing for fallback negotiation to
// take place on the farside.
func TypesToHeader(types ...Type) *MigrationHeader {
	missingFeature := false
	hasFeature := true
	var preferredType Type

	if len(types) > 0 {
		preferredType = types[0]
	}

	header := MigrationHeader{Fs: &preferredType.FSType}

	// Add ZFS features if preferred type is ZFS.
	if preferredType.FSType == MigrationFSType_ZFS {
		features := ZfsFeatures{
			Compress: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == ZFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			}
		}

		header.ZfsFeatures = &features
	}

	// Add BTRFS features if preferred type is BTRFS.
	if preferredType.FSType == MigrationFSType_BTRFS {
		features := BtrfsFeatures{
			MigrationHeader:  &missingFeature,
			HeaderSubvolumes: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == BTRFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			} else if feature == BTRFSFeatureSubvolumes {
				features.HeaderSubvolumes = &hasFeature
			} else if feature == BTRFSFeatureSubvolumeUUIDs {
				features.HeaderSubvolumeUuids = &hasFeature
			}
		}

		header.BtrfsFeatures = &features
	}

	// Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list.
	for _, t := range types {
		if t.FSType != MigrationFSType_RSYNC && t.FSType != MigrationFSType_BLOCK_AND_RSYNC {
			continue
		}

		features := RsyncFeatures{
			Xattrs:        &missingFeature,
			Delete:        &missingFeature,
			Compress:      &missingFeature,
			Bidirectional: &missingFeature,
		}

		for _, feature := range t.Features {
			if feature == "xattrs" {
				features.Xattrs = &hasFeature
			} else if feature == "delete" {
				features.Delete = &hasFeature
			} else if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == "bidirectional" {
				features.Bidirectional = &hasFeature
			}
		}

		header.RsyncFeatures = &features
		break // Only use the first rsync transport type found to generate rsync features list.
	}

	return &header
}

// MatchTypes attempts to find matching migration transport types between an offered type sent from a remote
// source and the types supported by a local storage pool. If matches are found then one or more Types are
// returned containing the method and the matching optional features present in both. The function also takes a
// fallback type which is used as an additional offer type preference in case the preferred remote type is not
// compatible with the local type available. It is expected that both sides of the migration will support the
// fallback type for the volume's content type that is being migrated.
func MatchTypes(offer *MigrationHeader, fallbackType MigrationFSType, ourTypes []Type) ([]Type, error) {
	// Generate an offer types slice from the preferred type supplied from remote and the
	// fallback type supplied based on the content type of the transfer.
	offeredFSTypes := []MigrationFSType{offer.GetFs(), fallbackType}

	matchedTypes := []Type{}

	// Find first matching type.
	for _, ourType := range ourTypes {
		for _, offerFSType := range offeredFSTypes {
			if offerFSType != ourType.FSType {
				continue // Not a match, try the next one.
			}

			// We got a match, now extract the relevant offered features.
			var offeredFeatures []string
			if offerFSType == MigrationFSType_ZFS {
				offeredFeatures = offer.GetZfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_BTRFS {
				offeredFeatures = offer.GetBtrfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_RSYNC {
				offeredFeatures = offer.GetRsyncFeaturesSlice()
				if !shared.StringInSlice("bidirectional", offeredFeatures) {
					// If no bi-directional support, this means we are getting a response from
					// an old LXD server that doesn't support bidirectional negotiation, so
					// assume LXD 3.7 level. NOTE: Do NOT extend this list of arguments.
					offeredFeatures = []string{"xattrs", "delete", "compress"}
				}
			}

			// Find common features in both our type and offered type.
			commonFeatures := []string{}
			for _, ourFeature := range ourType.Features {
				if shared.StringInSlice(ourFeature, offeredFeatures) {
					commonFeatures = append(commonFeatures, ourFeature)
				}
			}

			if offer.Refresh != nil && *offer.Refresh == true {
				// Optimized refresh with zfs only works if ZfsFeatureMigrationHeader is available.
				if ourType.FSType == MigrationFSType_ZFS && !shared.StringInSlice(ZFSFeatureMigrationHeader, commonFeatures) {
					continue
				}

				// Optimized refresh with btrfs only works if BtrfsFeatureSubvolumeUUIDs is available.
				if ourType.FSType == MigrationFSType_BTRFS && !shared.StringInSlice(BTRFSFeatureSubvolumeUUIDs, commonFeatures) {
					continue
				}
			}

			// Append type with combined features.
			matchedTypes = append(matchedTypes, Type{
				FSType:   ourType.FSType,
				Features: commonFeatures,
			})
		}
	}

	if len(matchedTypes) < 1 {
		// No matching transport type found, generate an error with offered types and our types.
		offeredTypeStrings := make([]string, 0, len(offeredFSTypes))
		for _, offerFSType := range offeredFSTypes {
			offeredTypeStrings = append(offeredTypeStrings, offerFSType.String())
		}

		ourTypeStrings := make([]string, 0, len(ourTypes))
		for _, ourType := range ourTypes {
			ourTypeStrings = append(ourTypeStrings, ourType.FSType.String())
		}

		return matchedTypes, fmt.Errorf("No matching migration types found. Offered types: %v, our types: %v", offeredTypeStrings, ourTypeStrings)
	}

	return matchedTypes, nil
}

func progressWrapperRender(op *operations.Operation, key string, description string, progressInt int64, speedInt int64) {
	meta := op.Metadata()
	if meta == nil {
		meta = make(map[string]any)
	}

	progress := fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	if description != "" {
		progress = fmt.Sprintf("%s: %s (%s/s)", description, units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	}

	if meta[key] != progress {
		meta[key] = progress
		_ = op.UpdateMetadata(meta)
	}
}

// ProgressReader reports the read progress.
func ProgressReader(op *operations.Operation, key string, description string) func(io.ReadCloser) io.ReadCloser {
	return func(reader io.ReadCloser) io.ReadCloser {
		if op == nil {
			return reader
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		readPipe := &ioprogress.ProgressReader{
			ReadCloser: reader,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return readPipe
	}
}

// ProgressWriter reports the write progress.
func ProgressWriter(op *operations.Operation, key string, description string) func(io.WriteCloser) io.WriteCloser {
	return func(writer io.WriteCloser) io.WriteCloser {
		if op == nil {
			return writer
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		writePipe := &ioprogress.ProgressWriter{
			WriteCloser: writer,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return writePipe
	}
}

// ProgressTracker returns a migration I/O tracker
func ProgressTracker(op *operations.Operation, key string, description string) *ioprogress.ProgressTracker {
	progress := func(progressInt int64, speedInt int64) {
		progressWrapperRender(op, key, description, progressInt, speedInt)
	}

	tracker := &ioprogress.ProgressTracker{
		Handler: progress,
	}

	return tracker
}
<|endoftext|>