<commit_before>package tlj

import (
//	"net"
	"reflect"
//	"errors"
	"encoding/json"
	"encoding/binary"
	"testing"
)

type Thingy struct {
	Name	string
	ID		int
}

func BuildThingy(data []byte) interface{} {
		thing := &Thingy{}
		err := json.Unmarshal(data, &thing)
		if err != nil { return nil }
		return thing
}

func TestTypeStoreIsCorrectType(t *testing.T) {
	type_store := NewTypeStore()
	if reflect.TypeOf(type_store) != reflect.TypeOf(TypeStore{}) {
		t.Errorf("return value of NewTypeStore() != tlj.TypeStore")
	}
}

func TestTypeStoreHasCapsuleBuilder(t *testing.T) {
	type_store := NewTypeStore()
	cap := Capsule {
		RequestID:	1,
		Type:		1,
		Data:		"test",
	}
	cap_bytes, _ := json.Marshal(cap)
	iface := type_store.BuildType(0, cap_bytes)
	if restored, ok := iface.(*Capsule); ok {
		if restored.RequestID != cap.RequestID {
			t.Errorf("capsule builder did not restore RequestID")
		}
		if restored.Type != cap.Type {
			t.Errorf("capsule builder did not restore Type")
		}
		if restored.Data != cap.Data {
			t.Errorf("capsule builder did not restore Data")
		}
	} else {
		t.Errorf("could not assert *Capsule type on restored interface")
	}
}

func TestTypeStoreCanAddType(t *testing.T) {
	type_store := NewTypeStore()
	thingy_type := reflect.TypeOf(Thingy{})
	type_store.AddType(thingy_type, BuildThingy)
	if type_store.TypeCodes[thingy_type] != 1 {
		t.Errorf("call to AddType on new TypeStore did not create type_id of 1")
	}
}

func TestTypeStoreCanLookupCode(t *testing.T) {
	type_store := NewTypeStore()
	code, present := type_store.LookupCode(reflect.TypeOf(Capsule{}))
	if code != 0 || !present {
		t.Errorf("unable to lookup type_code for Capsule")
	}
}

func TestTypeStoreWontLookupBadCode(t *testing.T) {
	type_store := NewTypeStore()
	_, present := type_store.LookupCode(reflect.TypeOf(Thingy{}))
	if present {
		t.Errorf("nonexistent type returns a code")
	}
}

func TestTypeStoreCanBuildType(t *testing.T) {
	type_store := NewTypeStore()
	thingy_type := reflect.TypeOf(Thingy{})
	type_store.AddType(thingy_type, BuildThingy)
	if type_store.TypeCodes[thingy_type] != 1 {
		t.Errorf("call to AddType on new TypeStore did not create type_id of 1")
	}
	thingy := Thingy {
		Name:	"test",
		ID:		1,
	}
	marshalled, err := json.Marshal(thingy)
	if err != nil {
		t.Errorf("marshalling thingy returned an error")
	}
	iface := type_store.BuildType(1, marshalled)
	if restored, ok := iface.(*Thingy); ok {
		if restored.Name != thingy.Name {
			t.Errorf("string not presevered when building from marshalled struct")
		}
		if restored.ID != thingy.ID {
			t.Errorf("int not presevered when building from marshalled struct")
		}
	} else {
		t.Errorf("could not assert *Thingy type on restored interface")
	}
}

func TestTypeStoreWontBuildBadType(t *testing.T) {
	type_store := NewTypeStore()
	iface := type_store.BuildType(1, make([]byte, 0))
	if iface != nil {
		t.Errorf("type_store built something with a nonexistent id")
	}
}

func TestTypeStoreWontBuildUnformattedData(t *testing.T) {
	type_store := NewTypeStore()
	iface := type_store.BuildType(0, []byte("notjson"))
	if iface != nil {
		t.Errorf("type_store built something when bad data was supplied")
	}
}

func TestFormat(t *testing.T) {
	type_store := NewTypeStore()
	type_store.AddType(reflect.TypeOf(Thingy{}), BuildThingy)
	thing := Thingy {
		Name:	"test",
		ID:		1,
	}
	bytes, err := format(thing, &type_store)
	if err != nil {
		t.Errorf("error formatting valid struct: %s", err)
	}
	type_bytes := bytes[:2]
	size_bytes := bytes[2:6]
	json_data := bytes[6:]
	type_int := binary.LittleEndian.Uint16(type_bytes)
	size_int := binary.LittleEndian.Uint32(size_bytes)
	if type_int != 1 {
		t.Errorf("format didn't use the correct type ID")
	}
	if int(size_int) != len(json_data) {
		t.Errorf("format didn't set the correct length")
	}
	restored_thing := &Thingy{}
	err = json.Unmarshal(json_data, &restored_thing)
	if err != nil {
		t.Errorf("error unmarahalling format data: %s", err)
	}

}

func TestCantFormatUnknownType(t *testing.T) {
	type_store := NewTypeStore()
	thing := Thingy {
		Name:	"test",
		ID:		1,
	}
	_, err := format(thing, &type_store)
	if err == nil {
		t.Errorf("format didn't return error when unknown type was passed in")
	}

}
<commit_msg>next test stubs<commit_after>package tlj

import (
//	"net"
	"reflect"
//	"errors"
	"encoding/json"
	"encoding/binary"
	"testing"
)

type Thingy struct {
	Name	string
	ID		int
}

func BuildThingy(data []byte) interface{} {
		thing := &Thingy{}
		err := json.Unmarshal(data, &thing)
		if err != nil { return nil }
		return thing
}

func TestTypeStoreIsCorrectType(t *testing.T) {
	type_store := NewTypeStore()
	if reflect.TypeOf(type_store) != reflect.TypeOf(TypeStore{}) {
		t.Errorf("return value of NewTypeStore() != tlj.TypeStore")
	}
}

func TestTypeStoreHasCapsuleBuilder(t *testing.T) {
	type_store := NewTypeStore()
	cap := Capsule {
		RequestID:	1,
		Type:		1,
		Data:		"test",
	}
	cap_bytes, _ := json.Marshal(cap)
	iface := type_store.BuildType(0, cap_bytes)
	if restored, ok := iface.(*Capsule); ok {
		if restored.RequestID != cap.RequestID {
			t.Errorf("capsule builder did not restore RequestID")
		}
		if restored.Type != cap.Type {
			t.Errorf("capsule builder did not restore Type")
		}
		if restored.Data != cap.Data {
			t.Errorf("capsule builder did not restore Data")
		}
	} else {
		t.Errorf("could not assert *Capsule type on restored interface")
	}
}

func TestTypeStoreCanAddType(t *testing.T) {
	type_store := NewTypeStore()
	thingy_type := reflect.TypeOf(Thingy{})
	type_store.AddType(thingy_type, BuildThingy)
	if type_store.TypeCodes[thingy_type] != 1 {
		t.Errorf("call to AddType on new TypeStore did not create type_id of 1")
	}
}

func TestTypeStoreCanLookupCode(t *testing.T) {
	type_store := NewTypeStore()
	code, present := type_store.LookupCode(reflect.TypeOf(Capsule{}))
	if code != 0 || !present {
		t.Errorf("unable to lookup type_code for Capsule")
	}
}

func TestTypeStoreWontLookupBadCode(t *testing.T) {
	type_store := NewTypeStore()
	_, present := type_store.LookupCode(reflect.TypeOf(Thingy{}))
	if present {
		t.Errorf("nonexistent type returns a code")
	}
}

func TestTypeStoreCanBuildType(t *testing.T) {
	type_store := NewTypeStore()
	thingy_type := reflect.TypeOf(Thingy{})
	type_store.AddType(thingy_type, BuildThingy)
	if type_store.TypeCodes[thingy_type] != 1 {
		t.Errorf("call to AddType on new TypeStore did not create type_id of 1")
	}
	thingy := Thingy {
		Name:	"test",
		ID:		1,
	}
	marshalled, err := json.Marshal(thingy)
	if err != nil {
		t.Errorf("marshalling thingy returned an error")
	}
	iface := type_store.BuildType(1, marshalled)
	if restored, ok := iface.(*Thingy); ok {
		if restored.Name != thingy.Name {
			t.Errorf("string not presevered when building from marshalled struct")
		}
		if restored.ID != thingy.ID {
			t.Errorf("int not presevered when building from marshalled struct")
		}
	} else {
		t.Errorf("could not assert *Thingy type on restored interface")
	}
}

func TestTypeStoreWontBuildBadType(t *testing.T) {
	type_store := NewTypeStore()
	iface := type_store.BuildType(1, make([]byte, 0))
	if iface != nil {
		t.Errorf("type_store built something with a nonexistent id")
	}
}

func TestTypeStoreWontBuildUnformattedData(t *testing.T) {
	type_store := NewTypeStore()
	iface := type_store.BuildType(0, []byte("notjson"))
	if iface != nil {
		t.Errorf("type_store built something when bad data was supplied")
	}
}

func TestFormat(t *testing.T) {
	type_store := NewTypeStore()
	type_store.AddType(reflect.TypeOf(Thingy{}), BuildThingy)
	thing := Thingy {
		Name:	"test",
		ID:		1,
	}
	bytes, err := format(thing, &type_store)
	if err != nil {
		t.Errorf("error formatting valid struct: %s", err)
	}
	type_bytes := bytes[:2]
	size_bytes := bytes[2:6]
	json_data := bytes[6:]
	type_int := binary.LittleEndian.Uint16(type_bytes)
	size_int := binary.LittleEndian.Uint32(size_bytes)
	if type_int != 1 {
		t.Errorf("format didn't use the correct type ID")
	}
	if int(size_int) != len(json_data) {
		t.Errorf("format didn't set the correct length")
	}
	restored_thing := &Thingy{}
	err = json.Unmarshal(json_data, &restored_thing)
	if err != nil {
		t.Errorf("error unmarahalling format data: %s", err)
	}
}

func TestCantFormatUnknownType(t *testing.T) {
	type_store := NewTypeStore()
	thing := Thingy {
		Name:	"test",
		ID:		1,
	}
	_, err := format(thing, &type_store)
	if err == nil {
		t.Errorf("format didn't return error when unknown type was passed in")
	}
}

/*
func TestFormatCapsule(t *testing.T) {

}

func TestCantFormatCapsuleWithUnknownType(t *testing.T) {

}

func TestNextStruct(t *testing.T) {

}

func TestNextStructErrorWithBrokenSocket(t *testing.T) {

}

func TestNextStructNilWhenMissing(t *testing.T) {

}

func TestNextStructNilWhenHeaderTooSmall(t *testing.T) {

}
*/
<|endoftext|>


<commit_before>package warc_test

import (
	"crypto/sha1"
	"fmt"
	"github.com/slyrz/warc"
	"io"
	"io/ioutil"
	"os"
	"testing"
)

func testFileHash(t *testing.T, path string, mode warc.Mode) {
	t.Logf("testFileHash %q, mode %v", path, mode)

	file, err := os.Open(path)
	if err != nil {
		t.Fatalf("failed to open %q: %v", path, err)
	}
	defer file.Close()

	reader, err := warc.NewReader(file, mode)
	if err != nil {
		t.Fatalf("warc.NewReader failed for %q: %v", path, err)
	}
	defer reader.Close()

	for {
		record, err := reader.ReadRecord()
		if err != nil {
			if err != io.EOF {
				t.Fatalf("failed to read all record content: %v", err)
			}
			break
		}
		content, err := ioutil.ReadAll(record.Content)
		if err != nil {
			t.Fatalf("failed to read all record content: %v", err)
		}
		hash := fmt.Sprintf("sha1:%x", sha1.Sum(content))
		if hash != record.Header["warc-block-digest"] {
			t.Fatalf("expected %q, got %q", record.Header["warc-block-digest"], hash)
		}
	}
}

func testFileScan(t *testing.T, path string, mode warc.Mode) {
	file, err := os.Open(path)
	if err != nil {
		t.Fatalf("failed to open %q: %v", path, err)
	}
	defer file.Close()

	reader, err := warc.NewReader(file, mode)
	if err != nil {
		t.Fatalf("warc.NewReader failed for %q: %v", path, err)
	}
	defer reader.Close()

	total := 0
	for {
		if _, err := reader.ReadRecord(); err != nil {
			break
		}
		total++
	}
	if total != 50 {
		t.Fatalf("expected 50 records, got %v", total)
	}
}

func TestReader(t *testing.T) {
	var paths = []string{
		"testdata/test.warc",
		"testdata/test.warc.gz",
		"testdata/test.warc.bz2",
	}
	for _, path := range paths {
		testFileHash(t, path, warc.SequentialMode)
		testFileHash(t, path, warc.AsynchronousMode)
		testFileScan(t, path, warc.SequentialMode)
		testFileScan(t, path, warc.AsynchronousMode)
	}
}
<commit_msg>provide example functions in godoc<commit_after>package warc_test

import (
	"crypto/sha1"
	"fmt"
	"github.com/slyrz/warc"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func testFileHash(t *testing.T, path string, mode warc.Mode) {
	t.Logf("testFileHash %q, mode %v", path, mode)

	file, err := os.Open(path)
	if err != nil {
		t.Fatalf("failed to open %q: %v", path, err)
	}
	defer file.Close()

	reader, err := warc.NewReader(file, mode)
	if err != nil {
		t.Fatalf("warc.NewReader failed for %q: %v", path, err)
	}
	defer reader.Close()

	for {
		record, err := reader.ReadRecord()
		if err != nil {
			if err != io.EOF {
				t.Fatalf("failed to read all record content: %v", err)
			}
			break
		}
		content, err := ioutil.ReadAll(record.Content)
		if err != nil {
			t.Fatalf("failed to read all record content: %v", err)
		}
		hash := fmt.Sprintf("sha1:%x", sha1.Sum(content))
		if hash != record.Header["warc-block-digest"] {
			t.Fatalf("expected %q, got %q", record.Header["warc-block-digest"], hash)
		}
	}
}

func testFileScan(t *testing.T, path string, mode warc.Mode) {
	file, err := os.Open(path)
	if err != nil {
		t.Fatalf("failed to open %q: %v", path, err)
	}
	defer file.Close()

	reader, err := warc.NewReader(file, mode)
	if err != nil {
		t.Fatalf("warc.NewReader failed for %q: %v", path, err)
	}
	defer reader.Close()

	total := 0
	for {
		if _, err := reader.ReadRecord(); err != nil {
			break
		}
		total++
	}
	if total != 50 {
		t.Fatalf("expected 50 records, got %v", total)
	}
}

func TestReader(t *testing.T) {
	var paths = []string{
		"testdata/test.warc",
		"testdata/test.warc.gz",
		"testdata/test.warc.bz2",
	}
	for _, path := range paths {
		testFileHash(t, path, warc.SequentialMode)
		testFileHash(t, path, warc.AsynchronousMode)
		testFileScan(t, path, warc.SequentialMode)
		testFileScan(t, path, warc.AsynchronousMode)
	}
}

func ExampleReader() {
	// Read WARC file from os.Stdin.
	reader, err := warc.NewReader(os.Stdin, warc.SequentialMode)
	if err != nil {
		panic(err)
	}
	defer reader.Close()
	// Iterate over records.
	for {
		record, err := reader.ReadRecord()
		if err != nil {
			break
		}
		fmt.Printf("Record of size %d:\n", record.Length)
		for key, value := range record.Header {
			fmt.Printf("\t%v = %v\n", key, value)
		}
	}
}

func ExampleWriter() {
	// Write WARC records to os.Stdout.
	writer := warc.NewWriter(os.Stdout)
	// Create a new WARC record.
	record := warc.NewRecord()
	// Store metadata in the header. Key should be all lowercase.
	record.Header["warc-type"] = "resource"
	record.Header["content-type"] = "plain/text"
	// Assign the content to the record.
	record.Content = strings.NewReader("Hello, World!")
	// Write the record to os.Stdout.
	if err := writer.WriteRecord(record); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package async

func Waterfall(routines []Routine, callbacks ...Done) {
  l := New()
  l.Multiple(routines...)

  l.RunWaterfall(callbacks...)
}

func (l *List) RunWaterfall(callbacks ...Done) {
  fall := fall(l, callbacks...)
  next := next(l, callbacks...)

  l.Wait.Add(l.Len())

  fall(next)
}

func fall(l *List, callbacks ...Done) func(Done, ...interface{}) {
  return func(next Done, args ...interface{}) {
    e := l.Front()
    _, r := l.Remove(e)

    // Run the first waterfall routine and give it the next function, and
    // any arguments that were provided
    go r(next, args...)
    l.Wait.Wait()
  }
}

func next(l *List, callbacks ...Done) Done {
  fall := fall(l, callbacks...)

  return func(err error, args ...interface{}) {
    next := next(l, callbacks...)

    l.Wait.Done()
    if err != nil || l.Len() == 0 {
      // Just in case it's an error, let's make sure we've cleared
      // all of the sync.WaitGroup waits that we initiated.
      for i := 0; i < l.Len(); i++ {
        l.Wait.Done()
      }

      // Send the results to the callbacks
      for i := 0; i < len(callbacks); i++ {
        callbacks[i](err, args...)
      }
      return
    }

    // Run the next waterfall routine with any arguments that were provided
    fall(next, args...)
    return
  }
}
<commit_msg>Remove a commented import that isn't used<commit_after>package async

// import "fmt"

func Waterfall(routines []Routine, callbacks ...Done) {
  l := New()
  l.Multiple(routines...)

  l.RunWaterfall(callbacks...)
}

func (l *List) RunWaterfall(callbacks ...Done) {
  fall := fall(l, callbacks...)
  next := next(l, callbacks...)

  l.Wait.Add(l.Len())

  fall(next)
}

func fall(l *List, callbacks ...Done) func(Done, ...interface{}) {
  return func(next Done, args ...interface{}) {
    e := l.Front()
    _, r := l.Remove(e)

    // Run the first waterfall routine and give it the next function, and
    // any arguments that were provided
    go r(next, args...)
    l.Wait.Wait()
  }
}

func next(l *List, callbacks ...Done) Done {
  fall := fall(l, callbacks...)

  return func(err error, args ...interface{}) {
    next := next(l, callbacks...)

    l.Wait.Done()
    if err != nil || l.Len() == 0 {
      // Just in case it's an error, let's make sure we've cleared
      // all of the sync.WaitGroup waits that we initiated.
      for i := 0; i < l.Len(); i++ {
        l.Wait.Done()
      }

      // Send the results to the callbacks
      for i := 0; i < len(callbacks); i++ {
        callbacks[i](err, args...)
      }
      return
    }

    // Run the next waterfall routine with any arguments that were provided
    fall(next, args...)
    return
  }
}
<|endoftext|>


<commit_before>package waveguide

import (
	"fmt"
	"html"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/urlfetch"
)

func init() {
	http.HandleFunc("/", handle(root))
	http.HandleFunc("/update_all", handle(updateAll))
	http.HandleFunc("/update_one", handle(updateOne))
	http.HandleFunc("/show", handle(show))
	http.HandleFunc("/clear", handle(clear))
}

func handle(f func(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := appengine.NewContext(r)
		status, err := f(ctx, w, r)
		if err != nil {
			log.Errorf(ctx, "%v", err)
			w.WriteHeader(status)
			fmt.Fprintf(w, "%v", err)
		}
	}
}

// root shows the main page.
func root(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if r.URL.Path != "/" {
		return http.StatusNotFound, fmt.Errorf("%s not found", r.URL.Path)
	}
	q := datastore.NewQuery("Spot")
	var spots []Spot
	log.Infof(ctx, "root: Before GetAll")
	_, err := q.GetAll(ctx, &spots)
	log.Infof(ctx, "root: After GetAll")
	if err != nil {
		return http.StatusInternalServerError, fmt.Errorf("root: fetching spots: %v", err)
	}
	sort.Sort(ByRating(spots))
	data := struct {
		Head  template.HTML
		Spots []Spot
	}{
		Head:  head,
		Spots: spots,
	}
	err = rootTmpl.Execute(w, data)
	if err != nil {
		return http.StatusInternalServerError, fmt.Errorf("root: template: %v", err)
	}
	return http.StatusOK, nil
}

// updateAll fetches the site map from msw, saves Spots based on it, and adds tasks to update all the Spots.
func updateAll(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	// Get form arg n: number of spots to update, or -1 for all of them
	if err := r.ParseForm(); err != nil {
		return http.StatusInternalServerError, err
	}
	sn := r.FormValue("n")
	if sn == "" {
		sn = "-1"
	}
	n, err := strconv.Atoi(sn)

	// Download magicseaweed.com/site-map.php
	client := urlfetch.Client(ctx)
	body, err := get(client, "http://magicseaweed.com/site-map.php")
	if err != nil {
		return http.StatusInternalServerError, err
	}
	reportPaths := reportRx.FindAll(body, n)

	// For each report url:
	for _, rp := range reportPaths {
		// Add a task to the queue to download the report and store its stars and wave
		// height to the db.
		srp := string(rp)
		su := "http://magicseaweed.com" + srp
		t := taskqueue.NewPOSTTask("/update_one", map[string][]string{"url": {su}})
		if _, err := taskqueue.Add(ctx, t, ""); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return http.StatusInternalServerError, err
		}
	}

	fmt.Fprintf(w, "Processed %d paths.\n", len(reportPaths))
	fmt.Fprintf(w, "ok")
	return http.StatusOK, nil
}

// updateOne updates the conditions for a single surfing spot, given the
// path to the surf report for it.
func updateOne(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if err := r.ParseForm(); err != nil {
		return http.StatusInternalServerError, err
	}
	su := r.FormValue("url")
	log.Infof(ctx, "/update_one: url=%q", su)
	u, err := url.Parse(su)
	if err != nil {
		return http.StatusBadRequest, err
	}
	path := u.Path
	qual, err := fetchSpotQuality(ctx, su)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	AddQualityToSpot(ctx, path, qual)
	w.Write([]byte("ok"))
	return http.StatusOK, nil
}

// show shows the data for a single spot given its url, for debugging.
func show(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if err := r.ParseForm(); err != nil {
		return http.StatusInternalServerError, err
	}
	log.Infof(ctx, "/show. form: %v", r.Form)
	su := r.FormValue("url")
	u, err := url.Parse(su)
	if err != nil {
		return http.StatusBadRequest, err
	}
	path := u.Path
	key := SpotKey(ctx, path)
	var s Spot
	if err := datastore.Get(ctx, key, &s); err != nil {
		return http.StatusInternalServerError, err
	}
	fmt.Fprintf(w, "%+v", s)
	return http.StatusOK, nil
}

func clear(ctx context.Context, w http.ResponseWriter, _ *http.Request) (int, error) {
	// TODO: This may time out. Rewrite it to queue up a bunch of tasks to delete individual spots or batches of
	// them.
	q := datastore.NewQuery("Spot")
	var spots []Spot
	_, err := q.GetAll(ctx, &spots)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	keys := make([]*datastore.Key, len(spots))
	for i, s := range spots {
		keys[i] = SpotKey(ctx, s.MswPath)
	}
	err = datastore.DeleteMulti(ctx, keys)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	fmt.Fprintf(w, "ok")
	return http.StatusOK, nil
}

func get(client *http.Client, url string) ([]byte, error) {
	res, err := client.Get(url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to read body of %s", url)
	}
	return body, nil
}

var starSectionRx = regexp.MustCompile(`<ul class="rating rating-large clearfix">.*?</ul>`)
var starRx = regexp.MustCompile(`<li class="active"> *<i class="glyphicon glyphicon-star"></i> *</li>`)
var heightRx = regexp.MustCompile(`(\d+(?:-\d+)?)<small>ft`)
var reportRx = regexp.MustCompile(`/[^"/]+-Surf-Report/\d+/`)
var srpTailRx = regexp.MustCompile(`-Surf-Report/\d+/`)

func surfReportPathToName(srp string) (string, error) {
	s := srpTailRx.ReplaceAllString(srp, "")
	// TODO: Use PathUnescape once GAE supports Go 1.8
	s, err := url.QueryUnescape(s)
	if err != nil {
		return "", fmt.Errorf("Failed to unescape path in surfReportPathToName(%q). %v", srp, err)
	}
	s = html.UnescapeString(s)
	s = s[1:] // Remove leading /
	s = strings.Replace(s, "-", " ", -1)
	return s, nil
}

// fetchSpotQuality scrapes the surf report at the given url and returns a Quality struct
// summarizing the conditions.
func fetchSpotQuality(ctx context.Context, url string) (*Quality, error) {
	log.Infof(ctx, "Fetching %s", url)
	client := urlfetch.Client(ctx)
	body, err := get(client, url)
	if err != nil {
		return nil, err
	}
	rating := countTopStars(body)
	hMatch := heightRx.FindSubmatch(body)
	if len(hMatch) != 2 {
		return nil, fmt.Errorf("Wave height regex failed.")
	}
	height := fmt.Sprintf("%s ft", hMatch[1])
	q := &Quality{
		Rating:     rating,
		WaveHeight: height,
		TimeUnix:   time.Now().Unix(),
	}
	return q, nil
}

// countTopStars returns the number of stars in the first rating section on the page.
func countTopStars(body []byte) int {
	starSection := starSectionRx.Find(body)
	foundStars := starRx.FindAll(starSection, -1)
	return len(foundStars)
}
<commit_msg>Add /delete_one, make /clear queue tasks to call it.<commit_after>package waveguide

import (
	"fmt"
	"html"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/urlfetch"
)

func init() {
	http.HandleFunc("/", handle(root))
	http.HandleFunc("/update_all", handle(updateAll))
	http.HandleFunc("/update_one", handle(updateOne))
	http.HandleFunc("/show", handle(show))
	http.HandleFunc("/clear", handle(clear))
	http.HandleFunc("/delete_one", handle(deleteOne))
}

func handle(f func(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := appengine.NewContext(r)
		status, err := f(ctx, w, r)
		if err != nil {
			log.Errorf(ctx, "%v", err)
			w.WriteHeader(status)
			fmt.Fprintf(w, "%v", err)
		}
	}
}

// root shows the main page.
func root(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if r.URL.Path != "/" {
		return http.StatusNotFound, fmt.Errorf("%s not found", r.URL.Path)
	}
	q := datastore.NewQuery("Spot")
	var spots []Spot
	log.Infof(ctx, "root: Before GetAll")
	_, err := q.GetAll(ctx, &spots)
	log.Infof(ctx, "root: After GetAll")
	if err != nil {
		return http.StatusInternalServerError, fmt.Errorf("root: fetching spots: %v", err)
	}
	sort.Sort(ByRating(spots))
	data := struct {
		Head  template.HTML
		Spots []Spot
	}{
		Head:  head,
		Spots: spots,
	}
	err = rootTmpl.Execute(w, data)
	if err != nil {
		return http.StatusInternalServerError, fmt.Errorf("root: template: %v", err)
	}
	return http.StatusOK, nil
}

// updateAll fetches the site map from msw, saves Spots based on it, and adds tasks to update all the Spots.
func updateAll(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	// Get form arg n: number of spots to update, or -1 for all of them
	if err := r.ParseForm(); err != nil {
		return http.StatusInternalServerError, err
	}
	sn := r.FormValue("n")
	if sn == "" {
		sn = "-1"
	}
	n, err := strconv.Atoi(sn)

	// Download magicseaweed.com/site-map.php
	client := urlfetch.Client(ctx)
	body, err := get(client, "http://magicseaweed.com/site-map.php")
	if err != nil {
		return http.StatusInternalServerError, err
	}
	reportPaths := reportRx.FindAll(body, n)

	// For each report url:
	for _, rp := range reportPaths {
		// Add a task to the queue to download the report and store its stars and wave
		// height to the db.
		srp := string(rp)
		su := "http://magicseaweed.com" + srp
		t := taskqueue.NewPOSTTask("/update_one", map[string][]string{"url": {su}})
		if _, err := taskqueue.Add(ctx, t, ""); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return http.StatusInternalServerError, err
		}
	}

	fmt.Fprintf(w, "Processed %d paths.\n", len(reportPaths))
	fmt.Fprintf(w, "ok")
	return http.StatusOK, nil
}

// updateOne updates the conditions for a single surfing spot, given the
// path to the surf report for it.
func updateOne(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if err := r.ParseForm(); err != nil {
		return http.StatusInternalServerError, err
	}
	su := r.FormValue("url")
	log.Infof(ctx, "/update_one: url=%q", su)
	u, err := url.Parse(su)
	if err != nil {
		return http.StatusBadRequest, err
	}
	path := u.Path
	qual, err := fetchSpotQuality(ctx, su)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	AddQualityToSpot(ctx, path, qual)
	w.Write([]byte("ok"))
	return http.StatusOK, nil
}

// show shows the data for a single spot given its url, for debugging.
func show(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if err := r.ParseForm(); err != nil {
		return http.StatusInternalServerError, err
	}
	su := r.FormValue("url")
	u, err := url.Parse(su)
	if err != nil {
		return http.StatusBadRequest, err
	}
	path := u.Path
	key := SpotKey(ctx, path)
	var s Spot
	if err := datastore.Get(ctx, key, &s); err != nil {
		return http.StatusInternalServerError, err
	}
	fmt.Fprintf(w, "%+v", s)
	return http.StatusOK, nil
}

// clear removes all the Spots from datastore. They can easily be brought back with updateAll.
func clear(ctx context.Context, w http.ResponseWriter, _ *http.Request) (int, error) {
	// TODO: This may time out. Rewrite it to queue up a bunch of tasks to delete individual spots or batches of
	// them.
	q := datastore.NewQuery("Spot")
	var spots []Spot
	_, err := q.GetAll(ctx, &spots)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	keys := make([]*datastore.Key, len(spots))
	for i, s := range spots {
		keys[i] = SpotKey(ctx, s.MswPath)
		t := taskqueue.NewPOSTTask("/delete_one", map[string][]string{"path": {s.MswPath}})
		_, err = taskqueue.Add(ctx, t, "")
		if err != nil {
			return http.StatusInternalServerError, err
		}
	}
	fmt.Fprintf(w, "Queued %d spots for deletion.", len(spots))
	fmt.Fprintf(w, "ok")
	return http.StatusOK, nil
}

func deleteOne(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if err := r.ParseForm(); err != nil {
		return http.StatusInternalServerError, err
	}
	p := r.FormValue("path")
	key := SpotKey(ctx, p)
	err := datastore.Delete(ctx, key)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	fmt.Fprintf(w, "Deleted Spot %q\n", p)
	fmt.Fprintf(w, "ok")
	return http.StatusOK, nil
}

func get(client *http.Client, url string) ([]byte, error) {
	res, err := client.Get(url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to read body of %s", url)
	}
	return body, nil
}

var starSectionRx = regexp.MustCompile(`<ul class="rating rating-large clearfix">.*?</ul>`)
var starRx = regexp.MustCompile(`<li class="active"> *<i class="glyphicon glyphicon-star"></i> *</li>`)
var heightRx = regexp.MustCompile(`(\d+(?:-\d+)?)<small>ft`)
var reportRx = regexp.MustCompile(`/[^"/]+-Surf-Report/\d+/`)
var srpTailRx = regexp.MustCompile(`-Surf-Report/\d+/`)

func surfReportPathToName(srp string) (string, error) {
	s := srpTailRx.ReplaceAllString(srp, "")
	// TODO: Use PathUnescape once GAE supports Go 1.8
	s, err := url.QueryUnescape(s)
	if err != nil {
		return "", fmt.Errorf("Failed to unescape path in surfReportPathToName(%q). %v", srp, err)
	}
	s = html.UnescapeString(s)
	s = s[1:] // Remove leading /
	s = strings.Replace(s, "-", " ", -1)
	return s, nil
}

// fetchSpotQuality scrapes the surf report at the given url and returns a Quality struct
// summarizing the conditions.
func fetchSpotQuality(ctx context.Context, url string) (*Quality, error) {
	log.Infof(ctx, "Fetching %s", url)
	client := urlfetch.Client(ctx)
	body, err := get(client, url)
	if err != nil {
		return nil, err
	}
	rating := countTopStars(body)
	hMatch := heightRx.FindSubmatch(body)
	if len(hMatch) != 2 {
		return nil, fmt.Errorf("Wave height regex failed.")
	}
	height := fmt.Sprintf("%s ft", hMatch[1])
	q := &Quality{
		Rating:     rating,
		WaveHeight: height,
		TimeUnix:   time.Now().Unix(),
	}
	return q, nil
}

// countTopStars returns the number of stars in the first rating section on the page.
func countTopStars(body []byte) int {
	starSection := starSectionRx.Find(body)
	foundStars := starRx.FindAll(starSection, -1)
	return len(foundStars)
}
<|endoftext|>


<commit_before>package main

/*
webexconv is a program used to convert .txt webex backups into a csv capable of being imported by hyperoffice
Created by:	Lyle Cleveland
Date:		14Oct2014
*/

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"
)

var filename string //Holds filename from command line argument

func main() {
	args := os.Args[1:]
	if len(args) == 0 {
		fmt.Println("Enter the filename to convert or drop the file onto webexconv icon. Ex. webconv test.txt")
		return
	}
	filename = args[0]
	filename = filename[:len(filename)-4]
	start := time.Now()
	parseWebex()
	elapsed := time.Since(start)
	fmt.Print("Execution Time: ")
	fmt.Println(elapsed)
}

//parseWebex takes a webex txt backup and returns an array containing the headers and a seperate array containg the fields
func parseWebex() {
	r := regexp.MustCompile("^\\W$") //Regexp for stripping special characters out of the header
	var headers []string
	var fields []string
	scan, error := ioutil.ReadFile(filename + ".txt")
	if error != nil {
		panic(error)
	}
	var buffer bytes.Buffer
	headerEnd := 0

	//Loop for parsing header fields
	for i, b := range scan {
		//Only parse first line
		if b == '\n' {
			headers = strings.Split(buffer.String(), "\t")
			headerEnd = i
			buffer.Reset()
			break
		}
		//Remove spaces and special characters
		if b == ' ' || r.MatchString(string(b)) && b != '\t' {
			continue
		} else {
			buffer.WriteByte(b)
		}
	}

	inString := false //Variable used to determine whether within a string so as to not parse tabs or new lines

	//Main parsing loop
	for i := headerEnd + 1; i < len(scan); i++ {
		if scan[i] == '"' && scan[i-1] == '\t' && inString == false {
			//Found the beginning of a string field
			buffer.WriteByte(scan[i])
			inString = true
		} else if scan[i] == '"' && scan[i+1] == '\t' && inString == true {
			//Found the end of the string field
			buffer.WriteByte(scan[i])
			fields = append(fields, buffer.String())
			buffer.Reset()
			inString = false
			i += 1
		} else if inString {
			//We don't want to parse tabs or new lines if we are in a string
			buffer.WriteByte(scan[i])
		} else if scan[i] == '\n' && inString == false {
			//End of a line and not in a string so we are at the end of a field
			fields = append(fields, buffer.String())
			buffer.Reset()
		} else if scan[i] != '\t' {
			buffer.WriteByte(scan[i])
		} else {
			//Hit a tab so we found the end of a field
			fields = append(fields, buffer.String())
			buffer.Reset()
		}
	}
	writeCSV(headers, fields)
	return
}

//writeCSV is used to generate the parsed csv file for importing
func writeCSV(headers []string, fields []string) {
	file, error := os.Create(filename + ".csv")
	if error != nil {
		panic(error)
	}
	defer file.Close()
	w := csv.NewWriter(file)
	w.UseCRLF = true
	w.Write(headers)
	w.Flush()
	count := 0 //Used to know when we should make a new line of records after we have wrote fields equal to the amount of header fields
	var f []string
	for i := 0; i < len(fields); i++ {
		f = append(f, fields[i])
		count++
		if count%len(headers) == 0 {
			w.Write(f)
			f = nil
			count = 0
		}
	}
	w.Flush()

	//Reparses file to remove quotes from header and blank quoted fields as hyperoffice won't import them
	scan, error := ioutil.ReadFile(filename + ".csv")
	if error != nil {
		panic(error)
	}
	var temp []byte
	inHeader := true
	for i := 0; i < len(scan); i++ {
		if scan[i] == '"' && inHeader == true {
			//Remove quotes from header
			continue
		}
		if scan[i] == '"' && scan[i+1] == '"' && scan[i+2] == ',' {
			//Remove blank quoted fields
			temp = append(temp, ',')
			i += 2
		} else {
			temp = append(temp, scan[i])
		}
		if scan[i] == '\n' && inHeader == true {
			inHeader = false
		}
	}
	ioutil.WriteFile(filename+".csv", temp, 0644)
}
<commit_msg>Updated comment to be more clear as to why the re-parsing is done after writing file<commit_after>package main

/*
webexconv is a program used to convert .txt webex backups into a csv capable of being imported by hyperoffice
Created by:	Lyle Cleveland
Date:		14Oct2014
*/

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"
)

var filename string //Holds filename from command line argument

func main() {
	args := os.Args[1:]
	if len(args) == 0 {
		fmt.Println("Enter the filename to convert or drop the file onto webexconv icon. Ex. webconv test.txt")
		return
	}
	filename = args[0]
	filename = filename[:len(filename)-4]
	start := time.Now()
	parseWebex()
	elapsed := time.Since(start)
	fmt.Print("Execution Time: ")
	fmt.Println(elapsed)
}

//parseWebex takes a webex txt backup and returns an array containing the headers and a seperate array containg the fields
func parseWebex() {
	r := regexp.MustCompile("^\\W$") //Regexp for stripping special characters out of the header
	var headers []string
	var fields []string
	scan, error := ioutil.ReadFile(filename + ".txt")
	if error != nil {
		panic(error)
	}
	var buffer bytes.Buffer
	headerEnd := 0

	//Loop for parsing header fields
	for i, b := range scan {
		//Only parse first line
		if b == '\n' {
			headers = strings.Split(buffer.String(), "\t")
			headerEnd = i
			buffer.Reset()
			break
		}
		//Remove spaces and special characters
		if b == ' ' || r.MatchString(string(b)) && b != '\t' {
			continue
		} else {
			buffer.WriteByte(b)
		}
	}

	inString := false //Variable used to determine whether within a string so as to not parse tabs or new lines

	//Main parsing loop
	for i := headerEnd + 1; i < len(scan); i++ {
		if scan[i] == '"' && scan[i-1] == '\t' && inString == false {
			//Found the beginning of a string field
			buffer.WriteByte(scan[i])
			inString = true
		} else if scan[i] == '"' && scan[i+1] == '\t' && inString == true {
			//Found the end of the string field
			buffer.WriteByte(scan[i])
			fields = append(fields, buffer.String())
			buffer.Reset()
			inString = false
			i += 1
		} else if inString {
			//We don't want to parse tabs or new lines if we are in a string
			buffer.WriteByte(scan[i])
		} else if scan[i] == '\n' && inString == false {
			//End of a line and not in a string so we are at the end of a field
			fields = append(fields, buffer.String())
			buffer.Reset()
		} else if scan[i] != '\t' {
			buffer.WriteByte(scan[i])
		} else {
			//Hit a tab so we found the end of a field
			fields = append(fields, buffer.String())
			buffer.Reset()
		}
	}
	writeCSV(headers, fields)
	return
}

//writeCSV is used to generate the parsed csv file for importing
func writeCSV(headers []string, fields []string) {
	file, error := os.Create(filename + ".csv")
	if error != nil {
		panic(error)
	}
	defer file.Close()
	w := csv.NewWriter(file)
	w.UseCRLF = true
	w.Write(headers)
	w.Flush()
	count := 0 //Used to know when we should make a new line of records after we have wrote fields equal to the amount of header fields
	var f []string
	for i := 0; i < len(fields); i++ {
		f = append(f, fields[i])
		count++
		if count%len(headers) == 0 {
			w.Write(f)
			f = nil
			count = 0
		}
	}
	w.Flush()

	//Reparses file to remove quotes from header and blank quoted fields as hyperoffice won't import them
	//This is done to fix noncompatabilities introduced by golang's csv writer
	scan, error := ioutil.ReadFile(filename + ".csv")
	if error != nil {
		panic(error)
	}
	var temp []byte
	inHeader := true
	for i := 0; i < len(scan); i++ {
		if scan[i] == '"' && inHeader == true {
			//Remove quotes from header
			continue
		}
		if scan[i] == '"' && scan[i+1] == '"' && scan[i+2] == ',' {
			//Remove blank quoted fields
			temp = append(temp, ',')
			i += 2
		} else {
			temp = append(temp, scan[i])
		}
		if scan[i] == '\n' && inHeader == true {
			inHeader = false
		}
	}
	ioutil.WriteFile(filename+".csv", temp, 0644)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package yarpcerrors

import (
	"bytes"
	"fmt"
)

// Newf returns a new Status.
//
// The Code should never be CodeOK, if it is, this will return nil.
func Newf(code Code, format string, args ...interface{}) *Status {
	if code == CodeOK {
		return nil
	}
	return &Status{
		code:    code,
		message: sprintf(format, args...),
	}
}

// FromError returns the Status for the error.
//
// If the error is nil, ut returns nil
// If the error is a Status, it returns err.
// If the error is not a Status, it returns a new error with the Code
// CodeUnknown and the message err.Error()
func FromError(err error) *Status {
	if err == nil {
		return nil
	}
	if status, ok := err.(*Status); ok {
		return status
	}
	return &Status{
		code:    CodeUnknown,
		message: err.Error(),
	}
}

// IsStatus returns true if err is a Status.
func IsStatus(err error) bool {
	_, ok := err.(*Status)
	return ok
}

// Status represents a YARPC error.
type Status struct {
	code    Code
	name    string
	message string
}

// WithName returns a new Status with the given name.
//
// If s is nil, this will still return nil.
//
// This should be used for user-defined errors.
//
// The name must only contain lowercase letters from a-z and dashes (-), and
// cannot start or end in a dash. If the name is something else, an error with
// code CodeInternal will be returned.
//
// Deprecated: Only use Codes to represent the type of the error. We plan to add
// a WithDetails method to add semantic metadata to Statuses soon.
func (s *Status) WithName(name string) *Status {
	if s == nil {
		return nil
	}
	if err := validateName(name); err != nil {
		return err.(*Status)
	}
	return &Status{
		code:    s.code,
		name:    name,
		message: s.message,
	}
}

// Code returns the code for the Status.
func (s *Status) Code() Code {
	if s == nil {
		return CodeOK
	}
	return s.code
}

// Name returns the name for the Status.
func (s *Status) Name() string {
	if s == nil {
		return ""
	}
	return s.name
}

// Message returns the message for the Status.
func (s *Status) Message() string {
	if s == nil {
		return ""
	}
	return s.message
}

// Error implements the error interface.
func (s *Status) Error() string {
	buffer := bytes.NewBuffer(nil)
	_, _ = buffer.WriteString(`code:`)
	_, _ = buffer.WriteString(s.code.String())
	if s.name != "" {
		_, _ = buffer.WriteString(` name:`)
		_, _ = buffer.WriteString(s.name)
	}
	if s.message != "" {
		_, _ = buffer.WriteString(` message:`)
		_, _ = buffer.WriteString(s.message)
	}
	return buffer.String()
}

// CancelledErrorf returns a new Status with code CodeCancelled
// by calling Newf(CodeCancelled, format, args...).
func CancelledErrorf(format string, args ...interface{}) error {
	return Newf(CodeCancelled, format, args...)
}

// UnknownErrorf returns a new Status with code CodeUnknown
// by calling Newf(CodeUnknown, format, args...).
func UnknownErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnknown, format, args...)
}

// InvalidArgumentErrorf returns a new Status with code CodeInvalidArgument
// by calling Newf(CodeInvalidArgument, format, args...).
func InvalidArgumentErrorf(format string, args ...interface{}) error {
	return Newf(CodeInvalidArgument, format, args...)
}

// DeadlineExceededErrorf returns a new Status with code CodeDeadlineExceeded
// by calling Newf(CodeDeadlineExceeded, format, args...).
func DeadlineExceededErrorf(format string, args ...interface{}) error {
	return Newf(CodeDeadlineExceeded, format, args...)
}

// NotFoundErrorf returns a new Status with code CodeNotFound
// by calling Newf(CodeNotFound, format, args...).
func NotFoundErrorf(format string, args ...interface{}) error {
	return Newf(CodeNotFound, format, args...)
}

// AlreadyExistsErrorf returns a new Status with code CodeAlreadyExists
// by calling Newf(CodeAlreadyExists, format, args...).
func AlreadyExistsErrorf(format string, args ...interface{}) error {
	return Newf(CodeAlreadyExists, format, args...)
}

// PermissionDeniedErrorf returns a new Status with code CodePermissionDenied
// by calling Newf(CodePermissionDenied, format, args...).
func PermissionDeniedErrorf(format string, args ...interface{}) error {
	return Newf(CodePermissionDenied, format, args...)
}

// ResourceExhaustedErrorf returns a new Status with code CodeResourceExhausted
// by calling Newf(CodeResourceExhausted, format, args...).
func ResourceExhaustedErrorf(format string, args ...interface{}) error {
	return Newf(CodeResourceExhausted, format, args...)
}

// FailedPreconditionErrorf returns a new Status with code CodeFailedPrecondition
// by calling Newf(CodeFailedPrecondition, format, args...).
func FailedPreconditionErrorf(format string, args ...interface{}) error {
	return Newf(CodeFailedPrecondition, format, args...)
}

// AbortedErrorf returns a new Status with code CodeAborted
// by calling Newf(CodeAborted, format, args...).
func AbortedErrorf(format string, args ...interface{}) error {
	return Newf(CodeAborted, format, args...)
}

// OutOfRangeErrorf returns a new Status with code CodeOutOfRange
// by calling Newf(CodeOutOfRange, format, args...).
func OutOfRangeErrorf(format string, args ...interface{}) error {
	return Newf(CodeOutOfRange, format, args...)
}

// UnimplementedErrorf returns a new Status with code CodeUnimplemented
// by calling Newf(CodeUnimplemented, format, args...).
func UnimplementedErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnimplemented, format, args...)
}

// InternalErrorf returns a new Status with code CodeInternal
// by calling Newf(CodeInternal, format, args...).
func InternalErrorf(format string, args ...interface{}) error {
	return Newf(CodeInternal, format, args...)
}

// UnavailableErrorf returns a new Status with code CodeUnavailable
// by calling Newf(CodeUnavailable, format, args...).
func UnavailableErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnavailable, format, args...)
}

// DataLossErrorf returns a new Status with code CodeDataLoss
// by calling Newf(CodeDataLoss, format, args...).
func DataLossErrorf(format string, args ...interface{}) error {
	return Newf(CodeDataLoss, format, args...)
}

// UnauthenticatedErrorf returns a new Status with code CodeUnauthenticated
// by calling Newf(CodeUnauthenticated, format, args...).
func UnauthenticatedErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnauthenticated, format, args...)
}

// IsCancelled returns true if FromError(err).Code() == CodeCancelled.
func IsCancelled(err error) bool {
	return FromError(err).Code() == CodeCancelled
}

// IsUnknown returns true if FromError(err).Code() == CodeUnknown.
func IsUnknown(err error) bool {
	return FromError(err).Code() == CodeUnknown
}

// IsInvalidArgument returns true if FromError(err).Code() == CodeInvalidArgument.
func IsInvalidArgument(err error) bool {
	return FromError(err).Code() == CodeInvalidArgument
}

// IsDeadlineExceeded returns true if FromError(err).Code() == CodeDeadlineExceeded.
func IsDeadlineExceeded(err error) bool {
	return FromError(err).Code() == CodeDeadlineExceeded
}

// IsNotFound returns true if FromError(err).Code() == CodeNotFound.
func IsNotFound(err error) bool {
	return FromError(err).Code() == CodeNotFound
}

// IsAlreadyExists returns true if FromError(err).Code() == CodeAlreadyExists.
func IsAlreadyExists(err error) bool {
	return FromError(err).Code() == CodeAlreadyExists
}

// IsPermissionDenied returns true if FromError(err).Code() == CodePermissionDenied.
func IsPermissionDenied(err error) bool {
	return FromError(err).Code() == CodePermissionDenied
}

// IsResourceExhausted returns true if FromError(err).Code() == CodeResourceExhausted.
func IsResourceExhausted(err error) bool {
	return FromError(err).Code() == CodeResourceExhausted
}

// IsFailedPrecondition returns true if FromError(err).Code() == CodeFailedPrecondition.
func IsFailedPrecondition(err error) bool {
	return FromError(err).Code() == CodeFailedPrecondition
}

// IsAborted returns true if FromError(err).Code() == CodeAborted.
func IsAborted(err error) bool {
	return FromError(err).Code() == CodeAborted
}

// IsOutOfRange returns true if FromError(err).Code() == CodeOutOfRange.
func IsOutOfRange(err error) bool {
	return FromError(err).Code() == CodeOutOfRange
}

// IsUnimplemented returns true if FromError(err).Code() == CodeUnimplemented.
func IsUnimplemented(err error) bool {
	return FromError(err).Code() == CodeUnimplemented
}

// IsInternal returns true if FromError(err).Code() == CodeInternal.
func IsInternal(err error) bool {
	return FromError(err).Code() == CodeInternal
}

// IsUnavailable returns true if FromError(err).Code() == CodeUnavailable.
func IsUnavailable(err error) bool {
	return FromError(err).Code() == CodeUnavailable
}

// IsDataLoss returns true if FromError(err).Code() == CodeDataLoss.
func IsDataLoss(err error) bool {
	return FromError(err).Code() == CodeDataLoss
}

// IsUnauthenticated returns true if FromError(err).Code() == CodeUnauthenticated.
func IsUnauthenticated(err error) bool {
	return FromError(err).Code() == CodeUnauthenticated
}

// IsYARPCError is a convenience function that returns true if the given error
// is a non-nil YARPC error.
//
// Deprecated: use IsStatus instead.
func IsYARPCError(err error) bool {
	return IsStatus(err)
}

// ErrorCode returns the Code for the given error, CodeOK if the error is nil,
// or CodeUnknown if the given error is not a YARPC error.
//
// Deprecated: Use FromError and Code instead.
func ErrorCode(err error) Code {
	return FromError(err).Code()
}

// ErrorName returns the name for the given error, or "" if the given
// error is not a YARPC error created with NamedErrorf that has a non-empty name.
//
// Deprecated: Use FromError and Name instead.
func ErrorName(err error) string {
	return FromError(err).Name()
}

// ErrorMessage returns the message for the given error, or "" if the given
// error is nil, or err.Error() if the given error is not a YARPC error or
// the YARPC error had no message.
//
// Deprecated: Use FromError and Message instead.
func ErrorMessage(err error) string {
	return FromError(err).Message()
}

// NamedErrorf returns a new Status with code CodeUnknown and the given name.
//
// This should be used for user-defined errors.
//
// The name must only contain lowercase letters from a-z and dashes (-), and
// cannot start or end in a dash. If the name is something else, an error with
// code CodeInternal will be returned.
//
// Deprecated: Use Newf and WithName instead.
func NamedErrorf(name string, format string, args ...interface{}) error {
	return Newf(CodeUnknown, format, args...).WithName(name)
}

// FromHeaders returns a new Status from headers transmitted from the server side.
//
// If the specified code is CodeOK, this will return nil.
//
// The name must only contain lowercase letters from a-z and dashes (-), and
// cannot start or end in a dash. If the name is something else, an error with
// code CodeInternal will be returned.
//
// This function should not be used by server implementations, use the individual
// error constructors instead. This should only be used by transport implementations.
//
// Deprecated: Use Newf and WithName instead.
func FromHeaders(code Code, name string, message string) error {
	return Newf(code, message).WithName(name)
}

func sprintf(format string, args ...interface{}) string {
	if len(args) == 0 {
		return format
	}
	return fmt.Sprintf(format, args...)
}
<commit_msg>yarpcerrors: Clear up godocs (#1323)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package yarpcerrors

import (
	"bytes"
	"fmt"
)

// Newf returns a new Status.
//
// The Code should never be CodeOK, if it is, this will return nil.
func Newf(code Code, format string, args ...interface{}) *Status {
	if code == CodeOK {
		return nil
	}
	return &Status{
		code:    code,
		message: sprintf(format, args...),
	}
}

// FromError returns the Status for the provided error. If the provided error
// is not a Status, a new error with code CodeUnknown is returned.
//
// Returns nil if the provided error is nil.
func FromError(err error) *Status {
	if err == nil {
		return nil
	}
	if status, ok := err.(*Status); ok {
		return status
	}
	return &Status{
		code:    CodeUnknown,
		message: err.Error(),
	}
}

// IsStatus returns whether the provided error is a YARPC error.
//
// This is always false if the error is nil.
func IsStatus(err error) bool {
	_, ok := err.(*Status)
	return ok
}

// Status represents a YARPC error.
type Status struct {
	code    Code
	name    string
	message string
}

// WithName returns a new Status with the given name.
//
// This should be used for user-defined errors.
//
// The name must only contain lowercase letters from a-z and dashes (-), and
// cannot start or end in a dash. If the name is something else, an error with
// code CodeInternal will be returned.
//
// Deprecated: Use only error codes to represent the type of the error.
func (s *Status) WithName(name string) *Status {
	// TODO: We plan to add a WithDetails method to add semantic metadata to
	// Statuses soon.
	if s == nil {
		return nil
	}
	if err := validateName(name); err != nil {
		return err.(*Status)
	}
	return &Status{
		code:    s.code,
		name:    name,
		message: s.message,
	}
}

// Code returns the error code for this Status.
func (s *Status) Code() Code {
	if s == nil {
		return CodeOK
	}
	return s.code
}

// Name returns the name of the error for this Status.
//
// This is an empty string for all built-in YARPC errors. It may be customized
// by using WithName.
func (s *Status) Name() string {
	if s == nil {
		return ""
	}
	return s.name
}

// Message returns the error message for this Status.
func (s *Status) Message() string {
	if s == nil {
		return ""
	}
	return s.message
}

// Error implements the error interface.
func (s *Status) Error() string {
	buffer := bytes.NewBuffer(nil)
	_, _ = buffer.WriteString(`code:`)
	_, _ = buffer.WriteString(s.code.String())
	if s.name != "" {
		_, _ = buffer.WriteString(` name:`)
		_, _ = buffer.WriteString(s.name)
	}
	if s.message != "" {
		_, _ = buffer.WriteString(` message:`)
		_, _ = buffer.WriteString(s.message)
	}
	return buffer.String()
}

// CancelledErrorf returns a new Status with code CodeCancelled
// by calling Newf(CodeCancelled, format, args...).
func CancelledErrorf(format string, args ...interface{}) error {
	return Newf(CodeCancelled, format, args...)
}

// UnknownErrorf returns a new Status with code CodeUnknown
// by calling Newf(CodeUnknown, format, args...).
func UnknownErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnknown, format, args...)
}

// InvalidArgumentErrorf returns a new Status with code CodeInvalidArgument
// by calling Newf(CodeInvalidArgument, format, args...).
func InvalidArgumentErrorf(format string, args ...interface{}) error {
	return Newf(CodeInvalidArgument, format, args...)
}

// DeadlineExceededErrorf returns a new Status with code CodeDeadlineExceeded
// by calling Newf(CodeDeadlineExceeded, format, args...).
func DeadlineExceededErrorf(format string, args ...interface{}) error {
	return Newf(CodeDeadlineExceeded, format, args...)
}

// NotFoundErrorf returns a new Status with code CodeNotFound
// by calling Newf(CodeNotFound, format, args...).
func NotFoundErrorf(format string, args ...interface{}) error {
	return Newf(CodeNotFound, format, args...)
}

// AlreadyExistsErrorf returns a new Status with code CodeAlreadyExists
// by calling Newf(CodeAlreadyExists, format, args...).
func AlreadyExistsErrorf(format string, args ...interface{}) error {
	return Newf(CodeAlreadyExists, format, args...)
}

// PermissionDeniedErrorf returns a new Status with code CodePermissionDenied
// by calling Newf(CodePermissionDenied, format, args...).
func PermissionDeniedErrorf(format string, args ...interface{}) error {
	return Newf(CodePermissionDenied, format, args...)
}

// ResourceExhaustedErrorf returns a new Status with code CodeResourceExhausted
// by calling Newf(CodeResourceExhausted, format, args...).
func ResourceExhaustedErrorf(format string, args ...interface{}) error {
	return Newf(CodeResourceExhausted, format, args...)
}

// FailedPreconditionErrorf returns a new Status with code CodeFailedPrecondition
// by calling Newf(CodeFailedPrecondition, format, args...).
func FailedPreconditionErrorf(format string, args ...interface{}) error {
	return Newf(CodeFailedPrecondition, format, args...)
}

// AbortedErrorf returns a new Status with code CodeAborted
// by calling Newf(CodeAborted, format, args...).
func AbortedErrorf(format string, args ...interface{}) error {
	return Newf(CodeAborted, format, args...)
}

// OutOfRangeErrorf returns a new Status with code CodeOutOfRange
// by calling Newf(CodeOutOfRange, format, args...).
func OutOfRangeErrorf(format string, args ...interface{}) error {
	return Newf(CodeOutOfRange, format, args...)
}

// UnimplementedErrorf returns a new Status with code CodeUnimplemented
// by calling Newf(CodeUnimplemented, format, args...).
func UnimplementedErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnimplemented, format, args...)
}

// InternalErrorf returns a new Status with code CodeInternal
// by calling Newf(CodeInternal, format, args...).
func InternalErrorf(format string, args ...interface{}) error {
	return Newf(CodeInternal, format, args...)
}

// UnavailableErrorf returns a new Status with code CodeUnavailable
// by calling Newf(CodeUnavailable, format, args...).
func UnavailableErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnavailable, format, args...)
}

// DataLossErrorf returns a new Status with code CodeDataLoss
// by calling Newf(CodeDataLoss, format, args...).
func DataLossErrorf(format string, args ...interface{}) error {
	return Newf(CodeDataLoss, format, args...)
}

// UnauthenticatedErrorf returns a new Status with code CodeUnauthenticated
// by calling Newf(CodeUnauthenticated, format, args...).
func UnauthenticatedErrorf(format string, args ...interface{}) error {
	return Newf(CodeUnauthenticated, format, args...)
}

// IsCancelled returns true if FromError(err).Code() == CodeCancelled.
func IsCancelled(err error) bool {
	return FromError(err).Code() == CodeCancelled
}

// IsUnknown returns true if FromError(err).Code() == CodeUnknown.
func IsUnknown(err error) bool {
	return FromError(err).Code() == CodeUnknown
}

// IsInvalidArgument returns true if FromError(err).Code() == CodeInvalidArgument.
func IsInvalidArgument(err error) bool {
	return FromError(err).Code() == CodeInvalidArgument
}

// IsDeadlineExceeded returns true if FromError(err).Code() == CodeDeadlineExceeded.
func IsDeadlineExceeded(err error) bool {
	return FromError(err).Code() == CodeDeadlineExceeded
}

// IsNotFound returns true if FromError(err).Code() == CodeNotFound.
func IsNotFound(err error) bool {
	return FromError(err).Code() == CodeNotFound
}

// IsAlreadyExists returns true if FromError(err).Code() == CodeAlreadyExists.
func IsAlreadyExists(err error) bool {
	return FromError(err).Code() == CodeAlreadyExists
}

// IsPermissionDenied returns true if FromError(err).Code() == CodePermissionDenied.
func IsPermissionDenied(err error) bool {
	return FromError(err).Code() == CodePermissionDenied
}

// IsResourceExhausted returns true if FromError(err).Code() == CodeResourceExhausted.
func IsResourceExhausted(err error) bool {
	return FromError(err).Code() == CodeResourceExhausted
}

// IsFailedPrecondition returns true if FromError(err).Code() == CodeFailedPrecondition.
func IsFailedPrecondition(err error) bool {
	return FromError(err).Code() == CodeFailedPrecondition
}

// IsAborted returns true if FromError(err).Code() == CodeAborted.
func IsAborted(err error) bool {
	return FromError(err).Code() == CodeAborted
}

// IsOutOfRange returns true if FromError(err).Code() == CodeOutOfRange.
func IsOutOfRange(err error) bool {
	return FromError(err).Code() == CodeOutOfRange
}

// IsUnimplemented returns true if FromError(err).Code() == CodeUnimplemented.
func IsUnimplemented(err error) bool {
	return FromError(err).Code() == CodeUnimplemented
}

// IsInternal returns true if FromError(err).Code() == CodeInternal.
func IsInternal(err error) bool {
	return FromError(err).Code() == CodeInternal
}

// IsUnavailable returns true if FromError(err).Code() == CodeUnavailable.
func IsUnavailable(err error) bool {
	return FromError(err).Code() == CodeUnavailable
}

// IsDataLoss returns true if FromError(err).Code() == CodeDataLoss.
func IsDataLoss(err error) bool {
	return FromError(err).Code() == CodeDataLoss
}

// IsUnauthenticated returns true if FromError(err).Code() == CodeUnauthenticated.
func IsUnauthenticated(err error) bool {
	return FromError(err).Code() == CodeUnauthenticated
}

// IsYARPCError returns whether the provided error is a YARPC error.
//
// This is always false if the error is nil.
//
// Deprecated: use IsStatus instead.
func IsYARPCError(err error) bool {
	return IsStatus(err)
}

// ErrorCode returns the Code for the given error, CodeOK if the error is nil,
// or CodeUnknown if the given error is not a YARPC error.
//
// Deprecated: Use FromError and Code instead.
func ErrorCode(err error) Code {
	return FromError(err).Code()
}

// ErrorName returns the name for the given error, or "" if the given
// error is not a YARPC error created with NamedErrorf that has a non-empty name.
//
// Deprecated: Use FromError and Name instead.
func ErrorName(err error) string {
	return FromError(err).Name()
}

// ErrorMessage returns the message for the given error, or "" if the given
// error is nil, or err.Error() if the given error is not a YARPC error or
// the YARPC error had no message.
//
// Deprecated: Use FromError and Message instead.
func ErrorMessage(err error) string {
	return FromError(err).Message()
}

// NamedErrorf returns a new Status with code CodeUnknown and the given name.
//
// This should be used for user-defined errors.
//
// The name must only contain lowercase letters from a-z and dashes (-), and
// cannot start or end in a dash. If the name is something else, an error with
// code CodeInternal will be returned.
//
// Deprecated: Use Newf and WithName instead.
func NamedErrorf(name string, format string, args ...interface{}) error {
	return Newf(CodeUnknown, format, args...).WithName(name)
}

// FromHeaders returns a new Status from headers transmitted from the server side.
//
// If the specified code is CodeOK, this will return nil.
//
// The name must only contain lowercase letters from a-z and dashes (-), and
// cannot start or end in a dash. If the name is something else, an error with
// code CodeInternal will be returned.
//
// This function should not be used by server implementations, use the individual
// error constructors instead. This should only be used by transport implementations.
//
// Deprecated: Use Newf and WithName instead.
func FromHeaders(code Code, name string, message string) error {
	return Newf(code, message).WithName(name)
}

func sprintf(format string, args ...interface{}) string {
	if len(args) == 0 {
		return format
	}
	return fmt.Sprintf(format, args...)
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package mutators

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"strings"

	admissionv1 "k8s.io/api/admission/v1"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/log"
	"kubevirt.io/kubevirt/pkg/util"
	utiltypes "kubevirt.io/kubevirt/pkg/util/types"
	webhookutils "kubevirt.io/kubevirt/pkg/util/webhooks"
	"kubevirt.io/kubevirt/pkg/virt-api/webhooks"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
)

type VMIsMutator struct {
	ClusterConfig *virtconfig.ClusterConfig
}

func (mutator *VMIsMutator) Mutate(ar *admissionv1.AdmissionReview) *admissionv1.AdmissionResponse {
	if !webhookutils.ValidateRequestResource(ar.Request.Resource, webhooks.VirtualMachineInstanceGroupVersionResource.Group, webhooks.VirtualMachineInstanceGroupVersionResource.Resource) {
		err := fmt.Errorf("expect resource to be '%s'", webhooks.VirtualMachineInstanceGroupVersionResource.Resource)
		return webhookutils.ToAdmissionResponseError(err)
	}

	if resp := webhookutils.ValidateSchema(v1.VirtualMachineInstanceGroupVersionKind, ar.Request.Object.Raw); resp != nil {
		return resp
	}
	// Get new VMI from admission response
	newVMI, oldVMI, err := webhookutils.GetVMIFromAdmissionReview(ar)
	if err != nil {
		return webhookutils.ToAdmissionResponseError(err)
	}

	var patch []utiltypes.PatchOperation

	// Patch the spec, metadata and status with defaults if we deal with a create operation
	if ar.Request.Operation == admissionv1.Create {
		informers := webhooks.GetInformers()

		// Apply presets
		err = applyPresets(newVMI, informers.VMIPresetInformer)
		if err != nil {
			return &admissionv1.AdmissionResponse{
				Result: &metav1.Status{
					Message: err.Error(),
					Code:    http.StatusUnprocessableEntity,
				},
			}
		}

		// Apply namespace limits
		applyNamespaceLimitRangeValues(newVMI, informers.NamespaceLimitsInformer)

		// Set VMI defaults
		log.Log.Object(newVMI).V(4).Info("Apply defaults")
		mutator.setDefaultCPUModel(newVMI)
		mutator.setDefaultMachineType(newVMI)
		mutator.setDefaultResourceRequests(newVMI)
		mutator.setDefaultGuestCPUTopology(newVMI)
		mutator.setDefaultPullPoliciesOnContainerDisks(newVMI)
		err = mutator.setDefaultNetworkInterface(newVMI)
		if err != nil {
			return webhookutils.ToAdmissionResponseError(err)
		}
		v1.SetObjectDefaults_VirtualMachineInstance(newVMI)

		// In a future, yet undecided, release either libvirt or QEMU are going to check the hyperv dependencies, so we can get rid of this code.
		// Until that time, we need to handle the hyperv deps to avoid obscure rejections from QEMU later on
		log.Log.V(4).Info("Set HyperV dependencies")
		err = webhooks.SetVirtualMachineInstanceHypervFeatureDependencies(newVMI)
		if err != nil {
			// HyperV is a special case. If our best-effort attempt fails, we should leave
			// rejection to be performed later on in the validating webhook, and continue here.
			// Please note this means that partial changes may have been performed.
			// This is OK since each dependency must be atomic and independent (in ACID sense),
			// so the VMI configuration is still legal.
			log.Log.V(2).Infof("Failed to set HyperV dependencies: %s", err)
		}

		// Do some specific setting for Arm64 Arch. It should put before SetObjectDefaults_VirtualMachineInstance
		if webhooks.IsARM64() {
			log.Log.V(4).Info("Apply Arm64 specific setting")
			err = webhooks.SetVirtualMachineInstanceArm64Defaults(newVMI)
			if err != nil {
				// if SetVirtualMachineInstanceArm64Defaults fails, it's due to a validation error, which will get caught in the validation webhook after mutation finishes.
				log.Log.V(2).Infof("Failed to setting for Arm64: %s", err)
			}
		}
		// Add foreground finalizer
		newVMI.Finalizers = append(newVMI.Finalizers, v1.VirtualMachineInstanceFinalizer)

		// Set the phase to pending to avoid blank status
		newVMI.Status.Phase = v1.Pending

		if mutator.ClusterConfig.NonRootEnabled() {
			if canBeNonRoot(newVMI) {
				if newVMI.ObjectMeta.Annotations == nil {
					newVMI.ObjectMeta.Annotations = make(map[string]string)
				}
				newVMI.ObjectMeta.Annotations[v1.NonRootVMIAnnotation] = ""
			}
		}

		var value interface{}
		value = newVMI.Spec
		patch = append(patch, utiltypes.PatchOperation{
			Op:    "replace",
			Path:  "/spec",
			Value: value,
		})

		value = newVMI.ObjectMeta
		patch = append(patch, utiltypes.PatchOperation{
			Op:    "replace",
			Path:  "/metadata",
			Value: value,
		})

		value = newVMI.Status
		patch = append(patch, utiltypes.PatchOperation{
			Op:    "replace",
			Path:  "/status",
			Value: value,
		})

	} else if ar.Request.Operation == admissionv1.Update {
		// Ignore status updates if they are not coming from our service accounts
		// TODO: As soon as CRDs support field selectors we can remove this and just enable
		// the status subresource. Until then we need to update Status and Metadata labels in parallel for e.g. Migrations.
		if !reflect.DeepEqual(newVMI.Status, oldVMI.Status) {
			if !webhooks.IsKubeVirtServiceAccount(ar.Request.UserInfo.Username) {
				patch = append(patch, utiltypes.PatchOperation{
					Op:    "replace",
					Path:  "/status",
					Value: oldVMI.Status,
				})
			}
		}

	}

	patchBytes, err := json.Marshal(patch)
	if err != nil {
		return webhookutils.ToAdmissionResponseError(err)
	}

	jsonPatchType := admissionv1.PatchTypeJSONPatch
	return &admissionv1.AdmissionResponse{
		Allowed:   true,
		Patch:     patchBytes,
		PatchType: &jsonPatchType,
	}
}

func (mutator *VMIsMutator) setDefaultNetworkInterface(obj *v1.VirtualMachineInstance) error {
	autoAttach := obj.Spec.Domain.Devices.AutoattachPodInterface
	if autoAttach != nil && *autoAttach == false {
		return nil
	}

	// Override only when nothing is specified
	if len(obj.Spec.Networks) == 0 && len(obj.Spec.Domain.Devices.Interfaces) == 0 {
		iface := v1.NetworkInterfaceType(mutator.ClusterConfig.GetDefaultNetworkInterface())
		switch iface {
		case v1.BridgeInterface:
			if !mutator.ClusterConfig.IsBridgeInterfaceOnPodNetworkEnabled() {
				return fmt.Errorf("Bridge interface is not enabled in kubevirt-config")
			}
			obj.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultBridgeNetworkInterface()}
		case v1.MasqueradeInterface:
			obj.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultMasqueradeNetworkInterface()}
		case v1.SlirpInterface:
			if !mutator.ClusterConfig.IsSlirpInterfaceEnabled() {
				return fmt.Errorf("Slirp interface is not enabled in kubevirt-config")
			}
			defaultIface := v1.DefaultSlirpNetworkInterface()
			obj.Spec.Domain.Devices.Interfaces = []v1.Interface{*defaultIface}
		}

		obj.Spec.Networks = []v1.Network{*v1.DefaultPodNetwork()}
	}
	return nil
}

func (mutator *VMIsMutator) setDefaultCPUModel(vmi *v1.VirtualMachineInstance) {
	//if vmi doesn't have cpu topology or cpu model set
	if vmi.Spec.Domain.CPU == nil || vmi.Spec.Domain.CPU.Model == "" {
		if defaultCPUModel := mutator.ClusterConfig.GetCPUModel(); defaultCPUModel != "" {
			// create cpu topology struct
			if vmi.Spec.Domain.CPU == nil {
				vmi.Spec.Domain.CPU = &v1.CPU{}
			}
			//set is as vmi cpu model
			vmi.Spec.Domain.CPU.Model = defaultCPUModel
		}
	}
}

func (mutator *VMIsMutator) setDefaultGuestCPUTopology(vmi *v1.VirtualMachineInstance) {
	cores := uint32(1)
	threads := uint32(1)
	sockets := uint32(1)
	vmiCPU := vmi.Spec.Domain.CPU
	if vmiCPU == nil || (vmiCPU.Cores == 0 && vmiCPU.Sockets == 0 && vmiCPU.Threads == 0) {
		// create cpu topology struct
		if vmi.Spec.Domain.CPU == nil {
			vmi.Spec.Domain.CPU = &v1.CPU{}
		}
		//if cores, sockets, threads are not set, take value from domain resources request or limits and
		//set value into sockets, which have best performance (https://bugzilla.redhat.com/show_bug.cgi?id=1653453)
		resources := vmi.Spec.Domain.Resources
		if cpuLimit, ok := resources.Limits[k8sv1.ResourceCPU]; ok {
			sockets = uint32(cpuLimit.Value())
		} else if cpuRequests, ok := resources.Requests[k8sv1.ResourceCPU]; ok {
			sockets = uint32(cpuRequests.Value())
		}

		vmi.Spec.Domain.CPU.Sockets = sockets
		vmi.Spec.Domain.CPU.Cores = cores
		vmi.Spec.Domain.CPU.Threads = threads
	}
}

func (mutator *VMIsMutator) setDefaultMachineType(vmi *v1.VirtualMachineInstance) {
	machineType := mutator.ClusterConfig.GetMachineType()

	if machine := vmi.Spec.Domain.Machine; machine != nil {
		if machine.Type == "" {
			machine.Type = machineType
		}
	} else {
		vmi.Spec.Domain.Machine = &v1.Machine{Type: machineType}
	}
}

func (mutator *VMIsMutator) setDefaultPullPoliciesOnContainerDisks(vmi *v1.VirtualMachineInstance) {
	for _, volume := range vmi.Spec.Volumes {
		if volume.ContainerDisk != nil && volume.ContainerDisk.ImagePullPolicy == "" {
			if strings.HasSuffix(volume.ContainerDisk.Image, ":latest") || !strings.ContainsAny(volume.ContainerDisk.Image, ":@") {
				volume.ContainerDisk.ImagePullPolicy = k8sv1.PullAlways
			} else {
				volume.ContainerDisk.ImagePullPolicy = k8sv1.PullIfNotPresent
			}
		}
	}
}

func (mutator *VMIsMutator) setDefaultResourceRequests(vmi *v1.VirtualMachineInstance) {

	resources := &vmi.Spec.Domain.Resources

	if !resources.Limits.Cpu().IsZero() && resources.Requests.Cpu().IsZero() {
		if resources.Requests == nil {
			resources.Requests = k8sv1.ResourceList{}
		}
		resources.Requests[k8sv1.ResourceCPU] = resources.Limits[k8sv1.ResourceCPU]
	}

	if !resources.Limits.Memory().IsZero() && resources.Requests.Memory().IsZero() {
		if resources.Requests == nil {
			resources.Requests = k8sv1.ResourceList{}
		}
		resources.Requests[k8sv1.ResourceMemory] = resources.Limits[k8sv1.ResourceMemory]
	}

	if _, exists := resources.Requests[k8sv1.ResourceMemory]; !exists {
		var memory *resource.Quantity
		if vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Guest != nil {
			memory = vmi.Spec.Domain.Memory.Guest
		}
		if memory == nil && vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Hugepages != nil {
			if hugepagesSize, err := resource.ParseQuantity(vmi.Spec.Domain.Memory.Hugepages.PageSize); err == nil {
				memory = &hugepagesSize
			}
		}
		if memory != nil && memory.Value() > 0 {
			if resources.Requests == nil {
				resources.Requests = k8sv1.ResourceList{}
			}
			overcommit := mutator.ClusterConfig.GetMemoryOvercommit()
			if overcommit == 100 {
				resources.Requests[k8sv1.ResourceMemory] = *memory
			} else {
				value := (memory.Value() * int64(100)) / int64(overcommit)
				resources.Requests[k8sv1.ResourceMemory] = *resource.NewQuantity(value, memory.Format)
			}
			memoryRequest := resources.Requests[k8sv1.ResourceMemory]
			log.Log.Object(vmi).V(4).Infof("Set memory-request to %s as a result of memory-overcommit = %v%%", memoryRequest.String(), overcommit)
		}
	}

	if _, exists := resources.Requests[k8sv1.ResourceCPU]; !exists {
		if vmi.Spec.Domain.CPU != nil && vmi.Spec.Domain.CPU.DedicatedCPUPlacement {
			return
		}
		if resources.Requests == nil {
			resources.Requests = k8sv1.ResourceList{}
		}
		resources.Requests[k8sv1.ResourceCPU] = *mutator.ClusterConfig.GetCPURequest()
	}
}

func canBeNonRoot(vmi *v1.VirtualMachineInstance) bool {
	// VirtioFS doesn't work with session mode
	return !util.IsVMIVirtiofsEnabled(vmi)
}
<commit_msg>Run SRIOV as root<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package mutators

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"strings"

	admissionv1 "k8s.io/api/admission/v1"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/log"
	"kubevirt.io/kubevirt/pkg/util"
	utiltypes "kubevirt.io/kubevirt/pkg/util/types"
	webhookutils "kubevirt.io/kubevirt/pkg/util/webhooks"
	"kubevirt.io/kubevirt/pkg/virt-api/webhooks"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
)

type VMIsMutator struct {
	ClusterConfig *virtconfig.ClusterConfig
}

func (mutator *VMIsMutator) Mutate(ar *admissionv1.AdmissionReview) *admissionv1.AdmissionResponse {
	if !webhookutils.ValidateRequestResource(ar.Request.Resource, webhooks.VirtualMachineInstanceGroupVersionResource.Group, webhooks.VirtualMachineInstanceGroupVersionResource.Resource) {
		err := fmt.Errorf("expect resource to be '%s'", webhooks.VirtualMachineInstanceGroupVersionResource.Resource)
		return webhookutils.ToAdmissionResponseError(err)
	}

	if resp := webhookutils.ValidateSchema(v1.VirtualMachineInstanceGroupVersionKind, ar.Request.Object.Raw); resp != nil {
		return resp
	}
	// Get new VMI from admission response
	newVMI, oldVMI, err := webhookutils.GetVMIFromAdmissionReview(ar)
	if err != nil {
		return webhookutils.ToAdmissionResponseError(err)
	}

	var patch []utiltypes.PatchOperation

	// Patch the spec, metadata and status with defaults if we deal with a create operation
	if ar.Request.Operation == admissionv1.Create {
		informers := webhooks.GetInformers()

		// Apply presets
		err = applyPresets(newVMI, informers.VMIPresetInformer)
		if err != nil {
			return &admissionv1.AdmissionResponse{
				Result: &metav1.Status{
					Message: err.Error(),
					Code:    http.StatusUnprocessableEntity,
				},
			}
		}

		// Apply namespace limits
		applyNamespaceLimitRangeValues(newVMI, informers.NamespaceLimitsInformer)

		// Set VMI defaults
		log.Log.Object(newVMI).V(4).Info("Apply defaults")
		mutator.setDefaultCPUModel(newVMI)
		mutator.setDefaultMachineType(newVMI)
		mutator.setDefaultResourceRequests(newVMI)
		mutator.setDefaultGuestCPUTopology(newVMI)
		mutator.setDefaultPullPoliciesOnContainerDisks(newVMI)
		err = mutator.setDefaultNetworkInterface(newVMI)
		if err != nil {
			return webhookutils.ToAdmissionResponseError(err)
		}
		v1.SetObjectDefaults_VirtualMachineInstance(newVMI)

		// In a future, yet undecided, release either libvirt or QEMU are going to check the hyperv dependencies, so we can get rid of this code.
		// Until that time, we need to handle the hyperv deps to avoid obscure rejections from QEMU later on
		log.Log.V(4).Info("Set HyperV dependencies")
		err = webhooks.SetVirtualMachineInstanceHypervFeatureDependencies(newVMI)
		if err != nil {
			// HyperV is a special case. If our best-effort attempt fails, we should leave
			// rejection to be performed later on in the validating webhook, and continue here.
			// Please note this means that partial changes may have been performed.
			// This is OK since each dependency must be atomic and independent (in ACID sense),
			// so the VMI configuration is still legal.
			log.Log.V(2).Infof("Failed to set HyperV dependencies: %s", err)
		}

		// Do some specific setting for Arm64 Arch. It should put before SetObjectDefaults_VirtualMachineInstance
		if webhooks.IsARM64() {
			log.Log.V(4).Info("Apply Arm64 specific setting")
			err = webhooks.SetVirtualMachineInstanceArm64Defaults(newVMI)
			if err != nil {
				// if SetVirtualMachineInstanceArm64Defaults fails, it's due to a validation error, which will get caught in the validation webhook after mutation finishes.
				log.Log.V(2).Infof("Failed to setting for Arm64: %s", err)
			}
		}
		// Add foreground finalizer
		newVMI.Finalizers = append(newVMI.Finalizers, v1.VirtualMachineInstanceFinalizer)

		// Set the phase to pending to avoid blank status
		newVMI.Status.Phase = v1.Pending

		if mutator.ClusterConfig.NonRootEnabled() {
			if canBeNonRoot(newVMI) {
				if newVMI.ObjectMeta.Annotations == nil {
					newVMI.ObjectMeta.Annotations = make(map[string]string)
				}
				newVMI.ObjectMeta.Annotations[v1.NonRootVMIAnnotation] = ""
			}
		}

		var value interface{}
		value = newVMI.Spec
		patch = append(patch, utiltypes.PatchOperation{
			Op:    "replace",
			Path:  "/spec",
			Value: value,
		})

		value = newVMI.ObjectMeta
		patch = append(patch, utiltypes.PatchOperation{
			Op:    "replace",
			Path:  "/metadata",
			Value: value,
		})

		value = newVMI.Status
		patch = append(patch, utiltypes.PatchOperation{
			Op:    "replace",
			Path:  "/status",
			Value: value,
		})

	} else if ar.Request.Operation == admissionv1.Update {
		// Ignore status updates if they are not coming from our service accounts
		// TODO: As soon as CRDs support field selectors we can remove this and just enable
		// the status subresource. Until then we need to update Status and Metadata labels in parallel for e.g. Migrations.
		if !reflect.DeepEqual(newVMI.Status, oldVMI.Status) {
			if !webhooks.IsKubeVirtServiceAccount(ar.Request.UserInfo.Username) {
				patch = append(patch, utiltypes.PatchOperation{
					Op:    "replace",
					Path:  "/status",
					Value: oldVMI.Status,
				})
			}
		}

	}

	patchBytes, err := json.Marshal(patch)
	if err != nil {
		return webhookutils.ToAdmissionResponseError(err)
	}

	jsonPatchType := admissionv1.PatchTypeJSONPatch
	return &admissionv1.AdmissionResponse{
		Allowed:   true,
		Patch:     patchBytes,
		PatchType: &jsonPatchType,
	}
}

func (mutator *VMIsMutator) setDefaultNetworkInterface(obj *v1.VirtualMachineInstance) error {
	autoAttach := obj.Spec.Domain.Devices.AutoattachPodInterface
	if autoAttach != nil && *autoAttach == false {
		return nil
	}

	// Override only when nothing is specified
	if len(obj.Spec.Networks) == 0 && len(obj.Spec.Domain.Devices.Interfaces) == 0 {
		iface := v1.NetworkInterfaceType(mutator.ClusterConfig.GetDefaultNetworkInterface())
		switch iface {
		case v1.BridgeInterface:
			if !mutator.ClusterConfig.IsBridgeInterfaceOnPodNetworkEnabled() {
				return fmt.Errorf("Bridge interface is not enabled in kubevirt-config")
			}
			obj.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultBridgeNetworkInterface()}
		case v1.MasqueradeInterface:
			obj.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultMasqueradeNetworkInterface()}
		case v1.SlirpInterface:
			if !mutator.ClusterConfig.IsSlirpInterfaceEnabled() {
				return fmt.Errorf("Slirp interface is not enabled in kubevirt-config")
			}
			defaultIface := v1.DefaultSlirpNetworkInterface()
			obj.Spec.Domain.Devices.Interfaces = []v1.Interface{*defaultIface}
		}

		obj.Spec.Networks = []v1.Network{*v1.DefaultPodNetwork()}
	}
	return nil
}

func (mutator *VMIsMutator) setDefaultCPUModel(vmi *v1.VirtualMachineInstance) {
	//if vmi doesn't have cpu topology or cpu model set
	if vmi.Spec.Domain.CPU == nil || vmi.Spec.Domain.CPU.Model == "" {
		if defaultCPUModel := mutator.ClusterConfig.GetCPUModel(); defaultCPUModel != "" {
			// create cpu topology struct
			if vmi.Spec.Domain.CPU == nil {
				vmi.Spec.Domain.CPU = &v1.CPU{}
			}
			//set is as vmi cpu model
			vmi.Spec.Domain.CPU.Model = defaultCPUModel
		}
	}
}

func (mutator *VMIsMutator) setDefaultGuestCPUTopology(vmi *v1.VirtualMachineInstance) {
	cores := uint32(1)
	threads := uint32(1)
	sockets := uint32(1)
	vmiCPU := vmi.Spec.Domain.CPU
	if vmiCPU == nil || (vmiCPU.Cores == 0 && vmiCPU.Sockets == 0 && vmiCPU.Threads == 0) {
		// create cpu topology struct
		if vmi.Spec.Domain.CPU == nil {
			vmi.Spec.Domain.CPU = &v1.CPU{}
		}
		//if cores, sockets, threads are not set, take value from domain resources request or limits and
		//set value into sockets, which have best performance (https://bugzilla.redhat.com/show_bug.cgi?id=1653453)
		resources := vmi.Spec.Domain.Resources
		if cpuLimit, ok := resources.Limits[k8sv1.ResourceCPU]; ok {
			sockets = uint32(cpuLimit.Value())
		} else if cpuRequests, ok := resources.Requests[k8sv1.ResourceCPU]; ok {
			sockets = uint32(cpuRequests.Value())
		}

		vmi.Spec.Domain.CPU.Sockets = sockets
		vmi.Spec.Domain.CPU.Cores = cores
		vmi.Spec.Domain.CPU.Threads = threads
	}
}

func (mutator *VMIsMutator) setDefaultMachineType(vmi *v1.VirtualMachineInstance) {
	machineType := mutator.ClusterConfig.GetMachineType()

	if machine := vmi.Spec.Domain.Machine; machine != nil {
		if machine.Type == "" {
			machine.Type = machineType
		}
	} else {
		vmi.Spec.Domain.Machine = &v1.Machine{Type: machineType}
	}
}

func (mutator *VMIsMutator) setDefaultPullPoliciesOnContainerDisks(vmi *v1.VirtualMachineInstance) {
	for _, volume := range vmi.Spec.Volumes {
		if volume.ContainerDisk != nil && volume.ContainerDisk.ImagePullPolicy == "" {
			if strings.HasSuffix(volume.ContainerDisk.Image, ":latest") || !strings.ContainsAny(volume.ContainerDisk.Image, ":@") {
				volume.ContainerDisk.ImagePullPolicy = k8sv1.PullAlways
			} else {
				volume.ContainerDisk.ImagePullPolicy = k8sv1.PullIfNotPresent
			}
		}
	}
}

func (mutator *VMIsMutator) setDefaultResourceRequests(vmi *v1.VirtualMachineInstance) {

	resources := &vmi.Spec.Domain.Resources

	if !resources.Limits.Cpu().IsZero() && resources.Requests.Cpu().IsZero() {
		if resources.Requests == nil {
			resources.Requests = k8sv1.ResourceList{}
		}
		resources.Requests[k8sv1.ResourceCPU] = resources.Limits[k8sv1.ResourceCPU]
	}

	if !resources.Limits.Memory().IsZero() && resources.Requests.Memory().IsZero() {
		if resources.Requests == nil {
			resources.Requests = k8sv1.ResourceList{}
		}
		resources.Requests[k8sv1.ResourceMemory] = resources.Limits[k8sv1.ResourceMemory]
	}

	if _, exists := resources.Requests[k8sv1.ResourceMemory]; !exists {
		var memory *resource.Quantity
		if vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Guest != nil {
			memory = vmi.Spec.Domain.Memory.Guest
		}
		if memory == nil && vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Hugepages != nil {
			if hugepagesSize, err := resource.ParseQuantity(vmi.Spec.Domain.Memory.Hugepages.PageSize); err == nil {
				memory = &hugepagesSize
			}
		}
		if memory != nil && memory.Value() > 0 {
			if resources.Requests == nil {
				resources.Requests = k8sv1.ResourceList{}
			}
			overcommit := mutator.ClusterConfig.GetMemoryOvercommit()
			if overcommit == 100 {
				resources.Requests[k8sv1.ResourceMemory] = *memory
			} else {
				value := (memory.Value() * int64(100)) / int64(overcommit)
				resources.Requests[k8sv1.ResourceMemory] = *resource.NewQuantity(value, memory.Format)
			}
			memoryRequest := resources.Requests[k8sv1.ResourceMemory]
			log.Log.Object(vmi).V(4).Infof("Set memory-request to %s as a result of memory-overcommit = %v%%", memoryRequest.String(), overcommit)
		}
	}

	if _, exists := resources.Requests[k8sv1.ResourceCPU]; !exists {
		if vmi.Spec.Domain.CPU != nil && vmi.Spec.Domain.CPU.DedicatedCPUPlacement {
			return
		}
		if resources.Requests == nil {
			resources.Requests = k8sv1.ResourceList{}
		}
		resources.Requests[k8sv1.ResourceCPU] = *mutator.ClusterConfig.GetCPURequest()
	}
}

func canBeNonRoot(vmi *v1.VirtualMachineInstance) bool {
	// VirtioFS doesn't work with session mode
	return !util.IsVMIVirtiofsEnabled(vmi) && !util.IsSRIOVVmi(vmi)
}
<|endoftext|>