<commit_before>package auth

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
)

const (
	tokenCookieName = "access_token"
)

func respondError(w http.ResponseWriter, message string, status int) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	encoder := json.NewEncoder(w)
	encoder.Encode(map[string]string{"message": message})
}

func AddRoutes(r *mux.Router, service Service) {
	// Do not serve these routes over http.
	r.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		decoder := json.NewDecoder(r.Body)
		req := LoginRequest{}
		err := decoder.Decode(&req)
		if err != nil {
			respondError(w, fmt.Sprintf("failed to decode login request from request body: %v", err), 400)
			return
		}

		token, err := service.Login(req)
		if err != nil {
			// Explicitly do not pass up the reason for login failure.
			respondError(w, "Invalid username or password.", 403)
			return
		}

		signedString, err := service.Sign(token)
		if err != nil {
			respondError(w, fmt.Sprintf("failed to issue token: %v", err), 503)
			return
		}

		// Return token as a cookie.
		cookie := &http.Cookie{
			Name:     tokenCookieName,
			Value:    signedString,
			Path:     "/api",
			Secure:   true,
			HttpOnly: true,
		}
		http.SetCookie(w, cookie)

		w.WriteHeader(http.StatusNoContent)
	}).Methods("POST")

	r.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
		claims, err := validateRequest(r, service)
		if err != nil {
			respondError(w, err.Error(), 403)
			return
		}

		info := UserInfo{
			Username: claims.Subject,
		}

		encoder := json.NewEncoder(w)
		err = encoder.Encode(info)
		if err != nil {
			log.Printf("failed to encode user info: %v", err)
			respondError(w, "failed to return user info", 503)
			return
		}
	}).Methods("GET")

	r.HandleFunc("/logout", func(w http.ResponseWriter, r *http.Request) {
		// Delete login cookie by setting one with 0 life.
		cookie := &http.Cookie{
			Name:     tokenCookieName,
			Value:    "",
			Path:     "/api",
			Secure:   true,
			HttpOnly: true,
			Expires:  time.Unix(0, 0),
			MaxAge:   -1,
		}
		http.SetCookie(w, cookie)

		w.WriteHeader(http.StatusNoContent)
	}).Methods("POST")
}

func AuthenticateFunc(service Service, f http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		_, err := validateRequest(r, service)
		if err != nil {
			respondError(w, err.Error(), 403)
		}

		f(w, r)
	}
}

func validateRequest(r *http.Request, service Service) (*Claims, error) {
	cookie, err := r.Cookie(tokenCookieName)
	if err != nil {
		log.Printf("no token provided: %v", err)
		return nil, fmt.Errorf("no access token provided")
	}

	ss := cookie.Value
	token, err := service.Parse(ss)
	if err != nil {
		log.Printf("invalid token: %v", err)
		return nil, fmt.Errorf("invalid token")
	}

	claims, ok := token.Claims.(*Claims)
	if !ok {
		log.Printf("invalid token claims")
		return nil, fmt.Errorf("invalid token")
	}

	return claims, nil
}
<commit_msg>Fix bug where failure to authenticate didn't prevent requests being processed.  (wow)<commit_after>package auth

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
)

const (
	tokenCookieName = "access_token"
)

func respondError(w http.ResponseWriter, message string, status int) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	encoder := json.NewEncoder(w)
	encoder.Encode(map[string]string{"message": message})
}

func AddRoutes(r *mux.Router, service Service) {
	// Do not serve these routes over http.
	r.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		decoder := json.NewDecoder(r.Body)
		req := LoginRequest{}
		err := decoder.Decode(&req)
		if err != nil {
			respondError(w, fmt.Sprintf("failed to decode login request from request body: %v", err), 400)
			return
		}

		token, err := service.Login(req)
		if err != nil {
			// Explicitly do not pass up the reason for login failure.
			respondError(w, "Invalid username or password.", 403)
			return
		}

		signedString, err := service.Sign(token)
		if err != nil {
			respondError(w, fmt.Sprintf("failed to issue token: %v", err), 503)
			return
		}

		// Return token as a cookie.
		cookie := &http.Cookie{
			Name:     tokenCookieName,
			Value:    signedString,
			Path:     "/api",
			Secure:   true,
			HttpOnly: true,
		}
		http.SetCookie(w, cookie)

		w.WriteHeader(http.StatusNoContent)
	}).Methods("POST")

	r.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
		claims, err := validateRequest(r, service)
		if err != nil {
			respondError(w, err.Error(), 403)
			return
		}

		info := UserInfo{
			Username: claims.Subject,
		}

		encoder := json.NewEncoder(w)
		err = encoder.Encode(info)
		if err != nil {
			log.Printf("failed to encode user info: %v", err)
			respondError(w, "failed to return user info", 503)
			return
		}
	}).Methods("GET")

	r.HandleFunc("/logout", func(w http.ResponseWriter, r *http.Request) {
		// Delete login cookie by setting one with 0 life.
		cookie := &http.Cookie{
			Name:     tokenCookieName,
			Value:    "",
			Path:     "/api",
			Secure:   true,
			HttpOnly: true,
			Expires:  time.Unix(0, 0),
			MaxAge:   -1,
		}
		http.SetCookie(w, cookie)

		w.WriteHeader(http.StatusNoContent)
	}).Methods("POST")
}

func AuthenticateFunc(service Service, f http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		_, err := validateRequest(r, service)
		if err != nil {
			respondError(w, err.Error(), 403)
			return
		}

		f(w, r)
	}
}

func validateRequest(r *http.Request, service Service) (*Claims, error) {
	cookie, err := r.Cookie(tokenCookieName)
	if err != nil {
		log.Printf("no token provided: %v", err)
		return nil, fmt.Errorf("no access token provided")
	}

	ss := cookie.Value
	token, err := service.Parse(ss)
	if err != nil {
		log.Printf("invalid token: %v", err)
		return nil, fmt.Errorf("invalid token")
	}

	claims, ok := token.Claims.(*Claims)
	if !ok {
		log.Printf("invalid token claims")
		return nil, fmt.Errorf("invalid token")
	}

	return claims, nil
}
<|endoftext|>


<commit_before>package content

import (
	"encoding/json"
	"errors"
	"fmt"
)

type KimonoContent struct {
	Url string
}

func (kimono KimonoContent) callAPI() ([]Content, error) {
	content, err := getWebserviceContent(kimono.Url)
	if err != nil {
		fmt.Println("Error while calling API")
		return nil, err
	} else {
		// Fill the record with the data from the JSON
		var jsonObj map[string]interface{}
		err = json.Unmarshal(content, &jsonObj)
		if err != nil {
			fmt.Println("An error occurred while converting our JSON to an object")
			return nil, err
		}

		contents := make([]Content, 0)

		if val, ok := jsonObj["results"]; ok {

			if results, ok := val.(map[string]interface{}); ok {

				if val, ok := results["collection1"]; ok {

					if collection, ok := val.([]interface{}); ok {

						for _, item := range collection {

							if property, ok := item.(map[string]interface{}); ok {

								if property1, ok := property["property1"]; ok {

									if contentData, ok := property1.(map[string]interface{}); ok {

										if text, ok := contentData["text"].(string); ok {

											if href, ok := contentData["href"].(string); ok {

												contents = append(contents, Content{Text: text, Url: href})

											} else {
												fmt.Println("Error mapping href as string")
												return nil, errors.New("json mapping error")
											}

										} else {
											fmt.Println("Error mapping text as string")
											return nil, errors.New("json mapping error")
										}

									} else {
										fmt.Println("Error mapping property1 as json obj")
										return nil, errors.New("json mapping error")
									}

								} else {
									fmt.Println("Missing mappings property1")
									return nil, errors.New("json mapping error")
								}

							} else {
								fmt.Println("Error mappings property1")
								return nil, errors.New("json mapping error")
							}

						}

					} else {
						fmt.Println("Error mappings collection1 as array")
						return nil, errors.New("json mapping error")
					}

				} else {
					fmt.Println("Error mappings collection1")
					return nil, errors.New("json mapping error")
				}

			} else {
				fmt.Println("Error mappings results as json obj")
				return nil, errors.New("json mapping error")
			}

			return contents, nil

		} else {
			fmt.Println("No field results in json")
			return nil, errors.New("json mapping error")
		}
	}
}
<commit_msg>Update kimono content to new version<commit_after>package content

import (
	"encoding/json"
	"errors"
	"fmt"
)

type KimonoContent struct {
	Url string
}

func (kimono KimonoContent) callAPI() ([]Content, error) {
	content, err := getWebserviceContent(kimono.Url)
	if err != nil {
		fmt.Println("Error while calling API")
		return nil, err
	} else {
		// Fill the record with the data from the JSON
		var jsonObj map[string]interface{}
		err = json.Unmarshal(content, &jsonObj)
		if err != nil {
			fmt.Println("An error occurred while converting our JSON to an object")
			return nil, err
		}

		contents := make([]Content, 0)

		if val, ok := jsonObj["results"]; ok {

			if results, ok := val.(map[string]interface{}); ok {

				if val, ok := results["collection1"]; ok {

					if collection, ok := val.([]interface{}); ok {

						for _, item := range collection {

							if property, ok := item.(map[string]interface{}); ok {

								if property1, ok := property["property1"]; ok {

									if contentData, ok := property1.(map[string]interface{}); ok {

										if text, ok := contentData["text"].(string); ok {

											if property2, ok := property["property2"]; ok {

												if contentData, ok := property2.(map[string]interface{}); ok {

													if href, ok := contentData["href"].(string); ok {

														contents = append(contents, Content{Text: text, Url: href})

													} else {
														fmt.Println("Error mapping href as string")
														return nil, errors.New("json mapping error")
													}
												} else {
													fmt.Println("Error mapping property2 as json obj")
													return nil, errors.New("json mapping error")
												}
											} else {
												fmt.Println("Missing mappings property2")
												return nil, errors.New("json mapping error")
											}

										} else {
											fmt.Println("Error mapping text as string")
											return nil, errors.New("json mapping error")
										}

									} else {
										fmt.Println("Error mapping property1 as json obj")
										return nil, errors.New("json mapping error")
									}

								} else {
									fmt.Println("Missing mappings property1")
									return nil, errors.New("json mapping error")
								}

							} else {
								fmt.Println("Error mappings property1")
								return nil, errors.New("json mapping error")
							}

						}

					} else {
						fmt.Println("Error mappings collection1 as array")
						return nil, errors.New("json mapping error")
					}

				} else {
					fmt.Println("Error mappings collection1")
					return nil, errors.New("json mapping error")
				}

			} else {
				fmt.Println("Error mappings results as json obj")
				return nil, errors.New("json mapping error")
			}

			return contents, nil

		} else {
			fmt.Println("No field results in json")
			return nil, errors.New("json mapping error")
		}
	}
}
<|endoftext|>


<commit_before>package reform_test

import (
	"errors"
	"time"

	"github.com/AlekSi/pointer"
	"github.com/enodata/faker"

	"gopkg.in/reform.v1"
	"gopkg.in/reform.v1/dialects/postgresql"
	. "gopkg.in/reform.v1/internal/test/models"
)

func (s *ReformSuite) TestInsert() {
	newEmail := faker.Internet().Email()
	person := &Person{Email: &newEmail}
	err := s.q.Insert(person)
	s.NoError(err)
	s.NotEqual(int32(0), person.ID)
	s.Equal("", person.Name)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(time.Now(), person.CreatedAt, 2*time.Second)
	s.Nil(person.UpdatedAt)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertWithValues() {
	t := time.Now()
	newEmail := faker.Internet().Email()
	person := &Person{Email: &newEmail, CreatedAt: t, UpdatedAt: &t}
	err := s.q.Insert(person)
	s.NoError(err)
	s.NotEqual(int32(0), person.ID)
	s.Equal("", person.Name)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(t, person.CreatedAt, 2*time.Second)
	s.WithinDuration(t, *person.UpdatedAt, 2*time.Second)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertWithPrimaryKey() {
	setIdentityInsert(s.T(), s.q, "people", true)

	newEmail := faker.Internet().Email()
	person := &Person{ID: 50, Email: &newEmail}
	err := s.q.Insert(person)
	s.NoError(err)
	s.Equal(int32(50), person.ID)
	s.Equal("", person.Name)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(time.Now(), person.CreatedAt, 2*time.Second)
	s.Nil(person.UpdatedAt)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertWithStringPrimaryKey() {
	if s.q.Dialect.LastInsertIdMethod() == reform.LastInsertId {
		s.T().Skip("%s uses LastInsertId", s.q.Dialect)
	}

	project := &Project{ID: "new", End: pointer.ToTime(time.Now().Truncate(24 * time.Hour))}
	err := s.q.Insert(project)
	s.NoError(err)
	s.Equal("new", project.ID)

	project2, err := s.q.FindByPrimaryKeyFrom(ProjectTable, project.ID)
	s.NoError(err)
	s.Equal(project, project2)

	err = s.q.Insert(project)
	s.Error(err)
}

func (s *ReformSuite) TestInsertIntoView() {
	pp := &PersonProject{PersonID: 1, ProjectID: "baron"}
	err := s.q.Insert(pp)
	s.NoError(err)

	err = s.q.Insert(pp)
	s.Error(err)

	s.RestartTransaction()

	pp = &PersonProject{PersonID: 1, ProjectID: "no_such_project"}
	err = s.q.Insert(pp)
	s.Error(err)
}

func (s *ReformSuite) TestInsertColumns() {
	t := time.Now()
	newEmail := faker.Internet().Email()
	person := &Person{Email: &newEmail, CreatedAt: t, UpdatedAt: &t}
	err := s.q.InsertColumns(person, "name", "email", "created_at", "updated_at")
	s.NoError(err)
	s.NotEqual(int32(0), person.ID)
	s.Equal("", person.Name)
	s.Equal((*int32)(nil), person.GroupID)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(t, person.CreatedAt, 2*time.Second)
	s.WithinDuration(t, *person.UpdatedAt, 2*time.Second)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	person.GroupID = pointer.ToInt32(65534)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertColumnsIntoView() {
	pp := &PersonProject{PersonID: 1, ProjectID: "baron"}
	err := s.q.InsertColumns(pp, "person_id", "project_id")
	s.NoError(err)

	err = s.q.InsertColumns(pp, "person_id", "project_id")
	s.Error(err)

	s.RestartTransaction()

	pp = &PersonProject{PersonID: 1, ProjectID: "no_such_project"}
	err = s.q.InsertColumns(pp, "person_id", "project_id")
	s.Error(err)
}

func (s *ReformSuite) TestInsertMulti() {
	newEmail := faker.Internet().Email()
	newName := faker.Name().Name()
	person1, person2 := &Person{Email: &newEmail}, &Person{Name: newName}
	err := s.q.InsertMulti(person1, person2)
	s.NoError(err)

	s.Equal(int32(0), person1.ID)
	s.Equal("", person1.Name)
	s.Equal(&newEmail, person1.Email)
	s.WithinDuration(time.Now(), person1.CreatedAt, 2*time.Second)
	s.Nil(person1.UpdatedAt)

	s.Equal(int32(0), person2.ID)
	s.Equal(newName, person2.Name)
	s.Nil(person2.Email)
	s.WithinDuration(time.Now(), person2.CreatedAt, 2*time.Second)
	s.Nil(person2.UpdatedAt)
}

func (s *ReformSuite) TestInsertMultiWithPrimaryKeys() {
	setIdentityInsert(s.T(), s.q, "people", true)

	newEmail := faker.Internet().Email()
	newName := faker.Name().Name()
	person1, person2 := &Person{ID: 50, Email: &newEmail}, &Person{ID: 51, Name: newName}
	err := s.q.InsertMulti(person1, person2)
	s.NoError(err)

	s.Equal(int32(50), person1.ID)
	s.Equal("", person1.Name)
	s.Equal(&newEmail, person1.Email)
	s.WithinDuration(time.Now(), person1.CreatedAt, 2*time.Second)
	s.Nil(person1.UpdatedAt)

	s.Equal(int32(51), person2.ID)
	s.Equal(newName, person2.Name)
	s.Nil(person2.Email)
	s.WithinDuration(time.Now(), person2.CreatedAt, 2*time.Second)
	s.Nil(person2.UpdatedAt)

	person, err := s.q.FindByPrimaryKeyFrom(PersonTable, person1.ID)
	s.NoError(err)
	s.Equal(person1, person)

	person, err = s.q.FindByPrimaryKeyFrom(PersonTable, person2.ID)
	s.NoError(err)
	s.Equal(person2, person)
}

func (s *ReformSuite) TestInsertMultiMixes() {
	err := s.q.InsertMulti()
	s.NoError(err)

	err = s.q.InsertMulti(&Person{}, &Project{})
	s.Error(err)

	err = s.q.InsertMulti(&Person{ID: 1}, &Person{})
	s.Error(err)
}

func (s *ReformSuite) TestInsertIDOnly() {
	id := &IDOnly{}
	err := s.q.Insert(id)
	s.NoError(err)
	s.Equal(int32(1), id.ID)
}

func (s *ReformSuite) TestUpdate() {
	var person Person
	err := s.q.Update(&person)
	s.Equal(reform.ErrNoPK, err)

	person.ID = 99
	err = s.q.Update(&person)
	s.Equal(reform.ErrNoRows, err)

	err = s.q.FindByPrimaryKeyTo(&person, 102)
	s.NoError(err)

	person.Email = pointer.ToString(faker.Internet().Email())
	err = s.q.Update(&person)
	s.NoError(err)
	s.Equal(personCreated, person.CreatedAt)
	s.Require().NotNil(person.UpdatedAt)
	s.WithinDuration(time.Now(), *person.UpdatedAt, 2*time.Second)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(&person, person2)
}

func (s *ReformSuite) TestUpdateOverwrite() {
	newEmail := faker.Internet().Email()
	person := Person{ID: 102, Email: pointer.ToString(newEmail)}
	err := s.q.Update(&person)
	s.NoError(err)

	var person2 Person
	err = s.q.FindByPrimaryKeyTo(&person2, person.ID)
	s.NoError(err)
	s.Equal("", person2.Name)
	s.Equal(&newEmail, person2.Email)
	s.WithinDuration(time.Now(), person2.CreatedAt, 2*time.Second)
	s.Require().NotNil(person2.UpdatedAt)
	s.WithinDuration(time.Now(), *person2.UpdatedAt, 2*time.Second)
}

func (s *ReformSuite) TestUpdateColumns() {
	newName := faker.Name().Name()
	newEmail := faker.Internet().Email()

	for p, columns := range map[*Person][]string{
		&Person{Name: "Elfrieda Abbott", Email: &newEmail}:                             {"email", "updated_at"},
		&Person{Name: newName, Email: pointer.ToString("elfrieda_abbott@example.org")}: {"name", "name", "updated_at"},
		&Person{Name: newName, Email: &newEmail}:                                       {"name", "email", "updated_at"},
	} {
		var person Person
		err := s.q.FindByPrimaryKeyTo(&person, 102)
		s.NoError(err)

		person.Name = p.Name
		person.Email = p.Email
		err = s.q.UpdateColumns(&person, columns...)
		s.NoError(err)
		s.Equal(personCreated, person.CreatedAt)
		s.Require().NotNil(person.UpdatedAt)
		s.WithinDuration(time.Now(), *person.UpdatedAt, 2*time.Second)

		person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
		s.NoError(err)
		s.Equal(&person, person2)

		s.RestartTransaction()
	}

	person := &Person{ID: 102, Name: newName, Email: &newEmail, CreatedAt: personCreated}
	for e, columns := range map[error][]string{
		errors.New("reform: unexpected columns: [foo]"):     {"foo"},
		errors.New("reform: will not update PK column: id"): {"id"},
		errors.New("reform: nothing to update"):             {},
	} {
		err := s.q.UpdateColumns(person, columns...)
		s.Error(err)
		s.Equal(e, err)
	}
}

func (s *ReformSuite) TestSave() {
	newName := faker.Name().Name()
	person := &Person{Name: newName}
	err := s.q.Save(person)
	s.NoError(err)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.Require().NoError(err)
	s.Equal(newName, person2.(*Person).Name)
	s.Nil(person2.(*Person).Email)
	s.Equal(person, person2)

	newEmail := faker.Internet().Email()
	person.Email = &newEmail
	err = s.q.Save(person)
	s.NoError(err)

	person2, err = s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.Require().NoError(err)
	s.Equal(newName, person2.(*Person).Name)
	s.Equal(&newEmail, person2.(*Person).Email)
	s.Equal(person, person2)
}

func (s *ReformSuite) TestDelete() {
	person := &Person{ID: 1}
	err := s.q.Delete(person)
	s.NoError(err)
	err = s.q.Reload(person)
	s.Equal(reform.ErrNoRows, err)

	project := &Project{ID: "baron"}
	err = s.q.Delete(project)
	s.NoError(err)
	err = s.q.Reload(project)
	s.Equal(reform.ErrNoRows, err)

	project = &Project{}
	err = s.q.Delete(project)
	s.Equal(reform.ErrNoPK, err)

	project = &Project{ID: "no_such_project"}
	err = s.q.Delete(project)
	s.Equal(reform.ErrNoRows, err)
}

func (s *ReformSuite) TestDeleteFrom() {
	ra, err := s.q.DeleteFrom(PersonTable, "WHERE email IS NULL")
	s.NoError(err)
	s.Equal(uint(3), ra)

	ra, err = s.q.DeleteFrom(PersonTable, "WHERE email IS NULL")
	s.NoError(err)
	s.Equal(uint(0), ra)

	// -1 second for SQLite3, otherwise it also deletes queen itself ¯\_(ツ)_/¯
	ra, err = s.q.DeleteFrom(ProjectTable, "WHERE start < "+s.q.Placeholder(1), queenStart.Add(-time.Second))
	s.NoError(err)
	s.Equal(uint(3), ra)

	ra, err = s.q.DeleteFrom(ProjectTable, "")
	s.NoError(err)
	s.Equal(uint(2), ra)

	ra, err = s.q.DeleteFrom(ProjectTable, "WHERE invalid_tail")
	s.Error(err)
	s.Equal(uint(0), ra)
}

func (s *ReformSuite) TestCommandsSchema() {
	if s.q.Dialect != postgresql.Dialect {
		s.T().Skip("only PostgreSQL supports schemas")
	}

	legacyPerson := &LegacyPerson{Name: pointer.ToString(faker.Name().Name())}
	err := s.q.Save(legacyPerson)
	s.NoError(err)
	err = s.q.Save(legacyPerson)
	s.NoError(err)
	err = s.q.Delete(legacyPerson)
	s.NoError(err)
}
<commit_msg>Use Skipf.<commit_after>package reform_test

import (
	"errors"
	"time"

	"github.com/AlekSi/pointer"
	"github.com/enodata/faker"

	"gopkg.in/reform.v1"
	"gopkg.in/reform.v1/dialects/postgresql"
	. "gopkg.in/reform.v1/internal/test/models"
)

func (s *ReformSuite) TestInsert() {
	newEmail := faker.Internet().Email()
	person := &Person{Email: &newEmail}
	err := s.q.Insert(person)
	s.NoError(err)
	s.NotEqual(int32(0), person.ID)
	s.Equal("", person.Name)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(time.Now(), person.CreatedAt, 2*time.Second)
	s.Nil(person.UpdatedAt)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertWithValues() {
	t := time.Now()
	newEmail := faker.Internet().Email()
	person := &Person{Email: &newEmail, CreatedAt: t, UpdatedAt: &t}
	err := s.q.Insert(person)
	s.NoError(err)
	s.NotEqual(int32(0), person.ID)
	s.Equal("", person.Name)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(t, person.CreatedAt, 2*time.Second)
	s.WithinDuration(t, *person.UpdatedAt, 2*time.Second)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertWithPrimaryKey() {
	setIdentityInsert(s.T(), s.q, "people", true)

	newEmail := faker.Internet().Email()
	person := &Person{ID: 50, Email: &newEmail}
	err := s.q.Insert(person)
	s.NoError(err)
	s.Equal(int32(50), person.ID)
	s.Equal("", person.Name)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(time.Now(), person.CreatedAt, 2*time.Second)
	s.Nil(person.UpdatedAt)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertWithStringPrimaryKey() {
	if s.q.Dialect.LastInsertIdMethod() == reform.LastInsertId {
		s.T().Skipf("%s uses LastInsertId", s.q.Dialect)
	}

	project := &Project{ID: "new", End: pointer.ToTime(time.Now().Truncate(24 * time.Hour))}
	err := s.q.Insert(project)
	s.NoError(err)
	s.Equal("new", project.ID)

	project2, err := s.q.FindByPrimaryKeyFrom(ProjectTable, project.ID)
	s.NoError(err)
	s.Equal(project, project2)

	err = s.q.Insert(project)
	s.Error(err)
}

func (s *ReformSuite) TestInsertIntoView() {
	pp := &PersonProject{PersonID: 1, ProjectID: "baron"}
	err := s.q.Insert(pp)
	s.NoError(err)

	err = s.q.Insert(pp)
	s.Error(err)

	s.RestartTransaction()

	pp = &PersonProject{PersonID: 1, ProjectID: "no_such_project"}
	err = s.q.Insert(pp)
	s.Error(err)
}

func (s *ReformSuite) TestInsertColumns() {
	t := time.Now()
	newEmail := faker.Internet().Email()
	person := &Person{Email: &newEmail, CreatedAt: t, UpdatedAt: &t}
	err := s.q.InsertColumns(person, "name", "email", "created_at", "updated_at")
	s.NoError(err)
	s.NotEqual(int32(0), person.ID)
	s.Equal("", person.Name)
	s.Equal((*int32)(nil), person.GroupID)
	s.Equal(&newEmail, person.Email)
	s.WithinDuration(t, person.CreatedAt, 2*time.Second)
	s.WithinDuration(t, *person.UpdatedAt, 2*time.Second)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	person.GroupID = pointer.ToInt32(65534)
	s.Equal(person, person2)

	err = s.q.Insert(person)
	s.Error(err)
}

func (s *ReformSuite) TestInsertColumnsIntoView() {
	pp := &PersonProject{PersonID: 1, ProjectID: "baron"}
	err := s.q.InsertColumns(pp, "person_id", "project_id")
	s.NoError(err)

	err = s.q.InsertColumns(pp, "person_id", "project_id")
	s.Error(err)

	s.RestartTransaction()

	pp = &PersonProject{PersonID: 1, ProjectID: "no_such_project"}
	err = s.q.InsertColumns(pp, "person_id", "project_id")
	s.Error(err)
}

func (s *ReformSuite) TestInsertMulti() {
	newEmail := faker.Internet().Email()
	newName := faker.Name().Name()
	person1, person2 := &Person{Email: &newEmail}, &Person{Name: newName}
	err := s.q.InsertMulti(person1, person2)
	s.NoError(err)

	s.Equal(int32(0), person1.ID)
	s.Equal("", person1.Name)
	s.Equal(&newEmail, person1.Email)
	s.WithinDuration(time.Now(), person1.CreatedAt, 2*time.Second)
	s.Nil(person1.UpdatedAt)

	s.Equal(int32(0), person2.ID)
	s.Equal(newName, person2.Name)
	s.Nil(person2.Email)
	s.WithinDuration(time.Now(), person2.CreatedAt, 2*time.Second)
	s.Nil(person2.UpdatedAt)
}

func (s *ReformSuite) TestInsertMultiWithPrimaryKeys() {
	setIdentityInsert(s.T(), s.q, "people", true)

	newEmail := faker.Internet().Email()
	newName := faker.Name().Name()
	person1, person2 := &Person{ID: 50, Email: &newEmail}, &Person{ID: 51, Name: newName}
	err := s.q.InsertMulti(person1, person2)
	s.NoError(err)

	s.Equal(int32(50), person1.ID)
	s.Equal("", person1.Name)
	s.Equal(&newEmail, person1.Email)
	s.WithinDuration(time.Now(), person1.CreatedAt, 2*time.Second)
	s.Nil(person1.UpdatedAt)

	s.Equal(int32(51), person2.ID)
	s.Equal(newName, person2.Name)
	s.Nil(person2.Email)
	s.WithinDuration(time.Now(), person2.CreatedAt, 2*time.Second)
	s.Nil(person2.UpdatedAt)

	person, err := s.q.FindByPrimaryKeyFrom(PersonTable, person1.ID)
	s.NoError(err)
	s.Equal(person1, person)

	person, err = s.q.FindByPrimaryKeyFrom(PersonTable, person2.ID)
	s.NoError(err)
	s.Equal(person2, person)
}

func (s *ReformSuite) TestInsertMultiMixes() {
	err := s.q.InsertMulti()
	s.NoError(err)

	err = s.q.InsertMulti(&Person{}, &Project{})
	s.Error(err)

	err = s.q.InsertMulti(&Person{ID: 1}, &Person{})
	s.Error(err)
}

func (s *ReformSuite) TestInsertIDOnly() {
	id := &IDOnly{}
	err := s.q.Insert(id)
	s.NoError(err)
	s.Equal(int32(1), id.ID)
}

func (s *ReformSuite) TestUpdate() {
	var person Person
	err := s.q.Update(&person)
	s.Equal(reform.ErrNoPK, err)

	person.ID = 99
	err = s.q.Update(&person)
	s.Equal(reform.ErrNoRows, err)

	err = s.q.FindByPrimaryKeyTo(&person, 102)
	s.NoError(err)

	person.Email = pointer.ToString(faker.Internet().Email())
	err = s.q.Update(&person)
	s.NoError(err)
	s.Equal(personCreated, person.CreatedAt)
	s.Require().NotNil(person.UpdatedAt)
	s.WithinDuration(time.Now(), *person.UpdatedAt, 2*time.Second)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.NoError(err)
	s.Equal(&person, person2)
}

func (s *ReformSuite) TestUpdateOverwrite() {
	newEmail := faker.Internet().Email()
	person := Person{ID: 102, Email: pointer.ToString(newEmail)}
	err := s.q.Update(&person)
	s.NoError(err)

	var person2 Person
	err = s.q.FindByPrimaryKeyTo(&person2, person.ID)
	s.NoError(err)
	s.Equal("", person2.Name)
	s.Equal(&newEmail, person2.Email)
	s.WithinDuration(time.Now(), person2.CreatedAt, 2*time.Second)
	s.Require().NotNil(person2.UpdatedAt)
	s.WithinDuration(time.Now(), *person2.UpdatedAt, 2*time.Second)
}

func (s *ReformSuite) TestUpdateColumns() {
	newName := faker.Name().Name()
	newEmail := faker.Internet().Email()

	for p, columns := range map[*Person][]string{
		&Person{Name: "Elfrieda Abbott", Email: &newEmail}:                             {"email", "updated_at"},
		&Person{Name: newName, Email: pointer.ToString("elfrieda_abbott@example.org")}: {"name", "name", "updated_at"},
		&Person{Name: newName, Email: &newEmail}:                                       {"name", "email", "updated_at"},
	} {
		var person Person
		err := s.q.FindByPrimaryKeyTo(&person, 102)
		s.NoError(err)

		person.Name = p.Name
		person.Email = p.Email
		err = s.q.UpdateColumns(&person, columns...)
		s.NoError(err)
		s.Equal(personCreated, person.CreatedAt)
		s.Require().NotNil(person.UpdatedAt)
		s.WithinDuration(time.Now(), *person.UpdatedAt, 2*time.Second)

		person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
		s.NoError(err)
		s.Equal(&person, person2)

		s.RestartTransaction()
	}

	person := &Person{ID: 102, Name: newName, Email: &newEmail, CreatedAt: personCreated}
	for e, columns := range map[error][]string{
		errors.New("reform: unexpected columns: [foo]"):     {"foo"},
		errors.New("reform: will not update PK column: id"): {"id"},
		errors.New("reform: nothing to update"):             {},
	} {
		err := s.q.UpdateColumns(person, columns...)
		s.Error(err)
		s.Equal(e, err)
	}
}

func (s *ReformSuite) TestSave() {
	newName := faker.Name().Name()
	person := &Person{Name: newName}
	err := s.q.Save(person)
	s.NoError(err)

	person2, err := s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.Require().NoError(err)
	s.Equal(newName, person2.(*Person).Name)
	s.Nil(person2.(*Person).Email)
	s.Equal(person, person2)

	newEmail := faker.Internet().Email()
	person.Email = &newEmail
	err = s.q.Save(person)
	s.NoError(err)

	person2, err = s.q.FindByPrimaryKeyFrom(PersonTable, person.ID)
	s.Require().NoError(err)
	s.Equal(newName, person2.(*Person).Name)
	s.Equal(&newEmail, person2.(*Person).Email)
	s.Equal(person, person2)
}

func (s *ReformSuite) TestDelete() {
	person := &Person{ID: 1}
	err := s.q.Delete(person)
	s.NoError(err)
	err = s.q.Reload(person)
	s.Equal(reform.ErrNoRows, err)

	project := &Project{ID: "baron"}
	err = s.q.Delete(project)
	s.NoError(err)
	err = s.q.Reload(project)
	s.Equal(reform.ErrNoRows, err)

	project = &Project{}
	err = s.q.Delete(project)
	s.Equal(reform.ErrNoPK, err)

	project = &Project{ID: "no_such_project"}
	err = s.q.Delete(project)
	s.Equal(reform.ErrNoRows, err)
}

func (s *ReformSuite) TestDeleteFrom() {
	ra, err := s.q.DeleteFrom(PersonTable, "WHERE email IS NULL")
	s.NoError(err)
	s.Equal(uint(3), ra)

	ra, err = s.q.DeleteFrom(PersonTable, "WHERE email IS NULL")
	s.NoError(err)
	s.Equal(uint(0), ra)

	// -1 second for SQLite3, otherwise it also deletes queen itself ¯\_(ツ)_/¯
	ra, err = s.q.DeleteFrom(ProjectTable, "WHERE start < "+s.q.Placeholder(1), queenStart.Add(-time.Second))
	s.NoError(err)
	s.Equal(uint(3), ra)

	ra, err = s.q.DeleteFrom(ProjectTable, "")
	s.NoError(err)
	s.Equal(uint(2), ra)

	ra, err = s.q.DeleteFrom(ProjectTable, "WHERE invalid_tail")
	s.Error(err)
	s.Equal(uint(0), ra)
}

func (s *ReformSuite) TestCommandsSchema() {
	if s.q.Dialect != postgresql.Dialect {
		s.T().Skip("only PostgreSQL supports schemas")
	}

	legacyPerson := &LegacyPerson{Name: pointer.ToString(faker.Name().Name())}
	err := s.q.Save(legacyPerson)
	s.NoError(err)
	err = s.q.Save(legacyPerson)
	s.NoError(err)
	err = s.q.Delete(legacyPerson)
	s.NoError(err)
}
<|endoftext|>


<commit_before>// Package mcrouter gathers Redis related data from a host.
package mcrouter

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Base struct {
	HostAndPort string
	ConfigFile  string
}

func (mcr *Base) Stats() (map[string]interface{}, error) {
	stats, err := mcr.StatsFromNetcat()
	if err != nil {
		return nil, err
	}

	statsFromFile, err := mcr.StatsFromFile()
	if err != nil {
		return nil, err
	}

	for key, value := range statsFromFile {
		trimmedKey := strings.Replace(key, "libmcrouter.mcrouter.5000.", "", -1)
		stats[trimmedKey] = value
	}

	hostname, err := os.Hostname()
	if err == nil {
		stats["hostname"] = hostname
	}

	return stats, nil
}

func (mcr *Base) StatsFromNetcat() (map[string]interface{}, error) {
	addrParts := strings.Split(mcr.HostAndPort, ":")
	host := addrParts[0]
	port := addrParts[1]

	if host == "" {
		host = "localhost"
	}

	c1 := exec.Command("echo", "stats")
	c2 := exec.Command("nc", host, port)

	var c2Output bytes.Buffer

	c2.Stdin, _ = c1.StdoutPipe()
	c2.Stdout = &c2Output

	c2.Start()

	err := c1.Run()
	if err != nil {
		return nil, err
	}

	err = c2.Wait()
	if err != nil {
		return nil, err
	}

	return NewStatsFromNetcat(c2Output.Bytes()), nil
}

func (mcr *Base) StatsFromFile() (map[string]interface{}, error) {
	addrParts := strings.Split(mcr.HostAndPort, ":")
	port := addrParts[1]

	statsFile := fmt.Sprintf("/var/mcrouter/stats/libmcrouter.mcrouter.%v.stats", port)

	statsJson, err := ioutil.ReadFile(statsFile)
	if err != nil {
		return nil, err
	}

	var stats map[string]interface{}
	err = json.Unmarshal(statsJson, &stats)
	if err != nil {
		return nil, err
	}

	return stats, nil
}

func NewStatsFromNetcat(data []byte) map[string]interface{} {
	stats := make(map[string]interface{})

	scanner := bufio.NewScanner(bytes.NewReader(data))
	for scanner.Scan() {
		parts := strings.Fields(scanner.Text())
		command := parts[0]

		if command != "STAT" {
			continue
		}

		statsKey := parts[1]
		value := strings.Join(parts[2:], " ")

		stats[statsKey] = value

		if statsKey == "pid" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "parent_pid" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "time" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "uptime" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "num_servers") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "num_clients" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "num_suspect_servers" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "mcc_txbuf_reqs" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "mcc_waiting_replies" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "destination_batch_size" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "asynclog_requests" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "proxy_reqs_processing" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "proxy_reqs_waiting" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "client_queue_notify_period" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "rusage") {
			valueFloat64, err := strconv.ParseFloat(value, 64)
			if err == nil {
				stats[statsKey] = valueFloat64
			}
		}
		if statsKey == "ps_num_minor_faults" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "ps_num_major_faults" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "ps_user_time_sec" {
			valueFloat64, err := strconv.ParseFloat(value, 64)
			if err == nil {
				stats[statsKey] = valueFloat64
			}
		}
		if statsKey == "ps_system_time_sec" {
			valueFloat64, err := strconv.ParseFloat(value, 64)
			if err == nil {
				stats[statsKey] = valueFloat64
			}
		}
		if statsKey == "ps_vsize" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "ps_rss" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "successful_client_connections" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "fibers") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_cas_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_delete_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_get_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_gets_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_set_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_other_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
	}

	return stats
}
<commit_msg>remove hostname on mcrouter<commit_after>// Package mcrouter gathers Redis related data from a host.
package mcrouter

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strconv"
	"strings"
)

type Base struct {
	HostAndPort string
	ConfigFile  string
}

func (mcr *Base) Stats() (map[string]interface{}, error) {
	stats, err := mcr.StatsFromNetcat()
	if err != nil {
		return nil, err
	}

	statsFromFile, err := mcr.StatsFromFile()
	if err != nil {
		return nil, err
	}

	for key, value := range statsFromFile {
		trimmedKey := strings.Replace(key, "libmcrouter.mcrouter.5000.", "", -1)
		stats[trimmedKey] = value
	}

	return stats, nil
}

func (mcr *Base) StatsFromNetcat() (map[string]interface{}, error) {
	addrParts := strings.Split(mcr.HostAndPort, ":")
	host := addrParts[0]
	port := addrParts[1]

	if host == "" {
		host = "localhost"
	}

	c1 := exec.Command("echo", "stats")
	c2 := exec.Command("nc", host, port)

	var c2Output bytes.Buffer

	c2.Stdin, _ = c1.StdoutPipe()
	c2.Stdout = &c2Output

	c2.Start()

	err := c1.Run()
	if err != nil {
		return nil, err
	}

	err = c2.Wait()
	if err != nil {
		return nil, err
	}

	return NewStatsFromNetcat(c2Output.Bytes()), nil
}

func (mcr *Base) StatsFromFile() (map[string]interface{}, error) {
	addrParts := strings.Split(mcr.HostAndPort, ":")
	port := addrParts[1]

	statsFile := fmt.Sprintf("/var/mcrouter/stats/libmcrouter.mcrouter.%v.stats", port)

	statsJson, err := ioutil.ReadFile(statsFile)
	if err != nil {
		return nil, err
	}

	var stats map[string]interface{}
	err = json.Unmarshal(statsJson, &stats)
	if err != nil {
		return nil, err
	}

	return stats, nil
}

func NewStatsFromNetcat(data []byte) map[string]interface{} {
	stats := make(map[string]interface{})

	scanner := bufio.NewScanner(bytes.NewReader(data))
	for scanner.Scan() {
		parts := strings.Fields(scanner.Text())
		command := parts[0]

		if command != "STAT" {
			continue
		}

		statsKey := parts[1]
		value := strings.Join(parts[2:], " ")

		stats[statsKey] = value

		if statsKey == "pid" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "parent_pid" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "time" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "uptime" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "num_servers") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "num_clients" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "num_suspect_servers" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "mcc_txbuf_reqs" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "mcc_waiting_replies" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "destination_batch_size" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "asynclog_requests" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "proxy_reqs_processing" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "proxy_reqs_waiting" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "client_queue_notify_period" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "rusage") {
			valueFloat64, err := strconv.ParseFloat(value, 64)
			if err == nil {
				stats[statsKey] = valueFloat64
			}
		}
		if statsKey == "ps_num_minor_faults" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "ps_num_major_faults" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "ps_user_time_sec" {
			valueFloat64, err := strconv.ParseFloat(value, 64)
			if err == nil {
				stats[statsKey] = valueFloat64
			}
		}
		if statsKey == "ps_system_time_sec" {
			valueFloat64, err := strconv.ParseFloat(value, 64)
			if err == nil {
				stats[statsKey] = valueFloat64
			}
		}
		if statsKey == "ps_vsize" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "ps_rss" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if statsKey == "successful_client_connections" {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "fibers") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_cas_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_delete_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_get_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_gets_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_set_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
		if strings.HasPrefix(statsKey, "cmd_other_outlier") {
			valueInt64, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				stats[statsKey] = valueInt64
			}
		}
	}

	return stats
}
<|endoftext|>


<commit_before>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package namespace

import (
	"context"
	"net/http"
	"testing"

	"kubedb.dev/apimachinery/apis/kubedb"
	api "kubedb.dev/apimachinery/apis/kubedb/v1alpha2"
	"kubedb.dev/apimachinery/client/clientset/versioned/scheme"

	admission "k8s.io/api/admission/v1beta1"
	authenticationv1 "k8s.io/api/authentication/v1"
	core "k8s.io/api/core/v1"
	kerr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	fake_dynamic "k8s.io/client-go/dynamic/fake"
	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
	meta_util "kmodules.xyz/client-go/meta"
)

func init() {
	utilruntime.Must(scheme.AddToScheme(clientsetscheme.Scheme))
}

var requestKind = metav1.GroupVersionKind{
	Group:   core.SchemeGroupVersion.Group,
	Version: core.SchemeGroupVersion.Version,
	Kind:    "Namespace",
}

func TestNamespaceValidator_Admit(t *testing.T) {
	for _, c := range cases {
		t.Run(c.testName, func(t *testing.T) {
			validator := NamespaceValidator{
				Resources: []string{api.ResourcePluralPostgres},
			}
			validator.initialized = true
			validator.dc = fake_dynamic.NewSimpleDynamicClient(clientsetscheme.Scheme)

			objJS, err := meta_util.MarshalToJson(c.object, core.SchemeGroupVersion)
			if err != nil {
				t.Fatalf("failed create marshal for input %s: %s", c.testName, err)
			}

			req := new(admission.AdmissionRequest)
			req.Kind = c.kind
			req.Name = c.namespace
			req.Operation = c.operation
			req.UserInfo = authenticationv1.UserInfo{}
			req.Object.Raw = objJS

			if c.operation == admission.Delete {
				if _, err := validator.dc.
					Resource(core.SchemeGroupVersion.WithResource("namespaces")).
					Create(context.TODO(), c.object, metav1.CreateOptions{}); err != nil && !kerr.IsAlreadyExists(err) {
					t.Fatalf("failed create namespace for input %s: %s", c.testName, err)
				}
			}
			if len(c.heatUp) > 0 {
				for _, u := range c.heatUp {
					if _, err := validator.dc.
						Resource(api.SchemeGroupVersion.WithResource(api.ResourcePluralPostgres)).
						Namespace("demo").
						Create(context.TODO(), u, metav1.CreateOptions{}); err != nil && !kerr.IsAlreadyExists(err) {
						t.Fatalf("failed create db for input %s: %s", c.testName, err)
					}
				}
			}

			response := validator.Admit(req)
			if c.result == true {
				if response.Allowed != true {
					t.Errorf("expected: 'Allowed=true'. but got response: %v", response)
				}
			} else if c.result == false {
				if response.Allowed == true || response.Result.Code == http.StatusInternalServerError {
					t.Errorf("expected: 'Allowed=false', but got response: %v", response)
				}
			}
		})
	}

}

var cases = []struct {
	testName  string
	kind      metav1.GroupVersionKind
	namespace string
	operation admission.Operation
	object    *unstructured.Unstructured
	heatUp    []*unstructured.Unstructured
	result    bool
}{
	{"Create Namespace",
		requestKind,
		"demo",
		admission.Create,
		sampleNamespace(),
		nil,
		true,
	},
	{"Delete Namespace containing db with terminationPolicy DoNotTerminate",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]*unstructured.Unstructured{editTerminationPolicy(sampleDatabase(), api.TerminationPolicyDoNotTerminate)},
		false,
	},
	{"Delete Namespace containing db with terminationPolicy Pause",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]*unstructured.Unstructured{editTerminationPolicy(sampleDatabase(), api.TerminationPolicyHalt)},
		false,
	},
	{"Delete Namespace containing db with terminationPolicy Delete",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]*unstructured.Unstructured{editTerminationPolicy(sampleDatabase(), api.TerminationPolicyDelete)},
		true,
	},
	{"Delete Namespace containing db with terminationPolicy WipeOut",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]*unstructured.Unstructured{editTerminationPolicy(sampleDatabase(), api.TerminationPolicyWipeOut)},
		true,
	},
	{"Delete Namespace containing db with NO terminationPolicy",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]*unstructured.Unstructured{deleteTerminationPolicy(sampleDatabase())},
		true,
	},
}

func sampleNamespace() *unstructured.Unstructured {
	return &unstructured.Unstructured{
		Object: map[string]interface{}{
			"apiVersion": core.SchemeGroupVersion.String(),
			"kind":       "Namespace",
			"metadata": map[string]interface{}{
				"name": "demo",
			},
			"spec": map[string]interface{}{},
		},
	}
}

func sampleDatabase() *unstructured.Unstructured {
	return &unstructured.Unstructured{
		Object: map[string]interface{}{
			"apiVersion": api.SchemeGroupVersion.String(),
			"kind":       "Postgres",
			"metadata": map[string]interface{}{
				"name":      "foo",
				"namespace": "demo",
				"labels": map[string]interface{}{
					meta_util.ManagedByLabelKey: kubedb.GroupName,
				},
			},
			"spec": map[string]interface{}{
				"terminationPolicy": string(api.TerminationPolicyDelete),
			},
			"status": map[string]interface{}{},
		},
	}
}

func editTerminationPolicy(db *unstructured.Unstructured, terminationPolicy api.TerminationPolicy) *unstructured.Unstructured {
	err := unstructured.SetNestedField(db.Object, string(terminationPolicy), "spec", "terminationPolicy")
	if err != nil {
		panic(err)
	}
	return db
}

func deleteTerminationPolicy(db *unstructured.Unstructured) *unstructured.Unstructured {
	unstructured.RemoveNestedField(db.Object, "spec", "terminationPolicy")
	return db
}
<commit_msg>Fix tests (#749)<commit_after>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package namespace

import (
	"net/http"
	"testing"

	"kubedb.dev/apimachinery/apis/kubedb"
	api "kubedb.dev/apimachinery/apis/kubedb/v1alpha2"
	"kubedb.dev/apimachinery/client/clientset/versioned/scheme"

	admission "k8s.io/api/admission/v1beta1"
	authenticationv1 "k8s.io/api/authentication/v1"
	core "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	fake_dynamic "k8s.io/client-go/dynamic/fake"
	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
	meta_util "kmodules.xyz/client-go/meta"
)

func init() {
	utilruntime.Must(scheme.AddToScheme(clientsetscheme.Scheme))
}

var requestKind = metav1.GroupVersionKind{
	Group:   core.SchemeGroupVersion.Group,
	Version: core.SchemeGroupVersion.Version,
	Kind:    "Namespace",
}

func TestNamespaceValidator_Admit(t *testing.T) {
	for _, c := range cases {
		t.Run(c.testName, func(t *testing.T) {
			validator := NamespaceValidator{
				Resources: []string{api.ResourcePluralPostgres},
			}
			validator.initialized = true

			objJS, err := meta_util.MarshalToJson(c.object, core.SchemeGroupVersion)
			if err != nil {
				t.Fatalf("failed create marshal for input %s: %s", c.testName, err)
			}

			req := new(admission.AdmissionRequest)
			req.Kind = c.kind
			req.Name = c.namespace
			req.Operation = c.operation
			req.UserInfo = authenticationv1.UserInfo{}
			req.Object.Raw = objJS

			var storeObjects []runtime.Object

			if c.operation == admission.Delete {
				storeObjects = append(storeObjects, c.object)
			}
			storeObjects = append(storeObjects, c.heatUp...)
			validator.dc = fake_dynamic.NewSimpleDynamicClient(clientsetscheme.Scheme, storeObjects...)

			response := validator.Admit(req)
			if c.result == true {
				if response.Allowed != true {
					t.Errorf("expected: 'Allowed=true'. but got response: %v", response)
				}
			} else if c.result == false {
				if response.Allowed == true || response.Result.Code == http.StatusInternalServerError {
					t.Errorf("expected: 'Allowed=false', but got response: %v", response)
				}
			}
		})
	}

}

var cases = []struct {
	testName  string
	kind      metav1.GroupVersionKind
	namespace string
	operation admission.Operation
	object    runtime.Object
	heatUp    []runtime.Object
	result    bool
}{
	{"Create Namespace",
		requestKind,
		"demo",
		admission.Create,
		sampleNamespace(),
		nil,
		true,
	},
	{"Delete Namespace containing db with terminationPolicy DoNotTerminate",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]runtime.Object{setTerminationPolicy(sampleDatabase(), api.TerminationPolicyDoNotTerminate)},
		false,
	},
	{"Delete Namespace containing db with terminationPolicy Pause",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]runtime.Object{setTerminationPolicy(sampleDatabase(), api.TerminationPolicyHalt)},
		false,
	},
	{"Delete Namespace containing db with terminationPolicy Delete",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]runtime.Object{setTerminationPolicy(sampleDatabase(), api.TerminationPolicyDelete)},
		true,
	},
	{"Delete Namespace containing db with terminationPolicy WipeOut",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]runtime.Object{setTerminationPolicy(sampleDatabase(), api.TerminationPolicyWipeOut)},
		true,
	},
	{"Delete Namespace containing db with NO terminationPolicy",
		requestKind,
		"demo",
		admission.Delete,
		sampleNamespace(),
		[]runtime.Object{deleteTerminationPolicy(sampleDatabase())},
		true,
	},
}

func sampleNamespace() *core.Namespace {
	return &core.Namespace{
		TypeMeta: metav1.TypeMeta{
			APIVersion: core.SchemeGroupVersion.String(),
			Kind:       "Namespace",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: "demo",
		},
	}
}

func sampleDatabase() *api.Postgres {
	return &api.Postgres{
		TypeMeta: metav1.TypeMeta{
			APIVersion: api.SchemeGroupVersion.String(),
			Kind:       "Postgres",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      "foo",
			Namespace: "demo",
			Labels: map[string]string{
				meta_util.ManagedByLabelKey: kubedb.GroupName,
			},
		},
		Spec: api.PostgresSpec{
			TerminationPolicy: api.TerminationPolicyDelete,
		},
	}
}

func setTerminationPolicy(obj runtime.Object, terminationPolicy api.TerminationPolicy) runtime.Object {
	db := obj.(*api.Postgres)
	db.Spec.TerminationPolicy = terminationPolicy
	return obj
}

func deleteTerminationPolicy(obj runtime.Object) runtime.Object {
	db := obj.(*api.Postgres)
	db.Spec.TerminationPolicy = ""
	return obj
}
<|endoftext|>


<commit_before>package configconverters

import (
	"fmt"

	"github.com/ghodss/yaml"
	kfapis "github.com/kubeflow/kfctl/v3/pkg/apis"
	kftypesv3 "github.com/kubeflow/kfctl/v3/pkg/apis/apps"
	kfconfig "github.com/kubeflow/kfctl/v3/pkg/apis/apps/kfconfig"
	kfdeftypes "github.com/kubeflow/kfctl/v3/pkg/apis/apps/kfdef/v1beta1"
)

// Empty struct - used to implement Converter interface.
type V1beta1 struct {
}

func maybeGetPlatform(pluginKind string) string {
	platforms := map[string]string{
		string(kfconfig.AWS_PLUGIN_KIND):              kftypesv3.AWS,
		string(kfconfig.GCP_PLUGIN_KIND):              kftypesv3.GCP,
		string(kfconfig.EXISTING_ARRIKTO_PLUGIN_KIND): kftypesv3.EXISTING_ARRIKTO,
	}

	p, ok := platforms[pluginKind]
	if ok {
		return p
	} else {
		return ""
	}
}

func (v V1beta1) ToKfConfig(kfdefBytes []byte) (*kfconfig.KfConfig, error) {
	kfdef := &kfdeftypes.KfDef{}
	if err := yaml.Unmarshal(kfdefBytes, kfdef); err != nil {
		return nil, &kfapis.KfError{
			Code:    int(kfapis.INTERNAL_ERROR),
			Message: fmt.Sprintf("could not unmarshal config file onto KfDef struct: %v", err),
		}
	}

	// Set UseBasicAuth later.
	config := &kfconfig.KfConfig{
		Spec: kfconfig.KfConfigSpec{
			UseBasicAuth: false,
		},
	}
	config.Name = kfdef.Name
	config.Namespace = kfdef.Namespace
	config.APIVersion = kfdef.APIVersion
	config.Kind = "KfConfig"
	config.Labels = kfdef.Labels
	config.Annotations = kfdef.Annotations
	config.Spec.Version = kfdef.Spec.Version
	for _, app := range kfdef.Spec.Applications {
		application := kfconfig.Application{
			Name: app.Name,
		}
		if app.KustomizeConfig != nil {
			kconfig := &kfconfig.KustomizeConfig{
				Overlays: app.KustomizeConfig.Overlays,
			}
			if app.KustomizeConfig.RepoRef != nil {
				kref := &kfconfig.RepoRef{
					Name: app.KustomizeConfig.RepoRef.Name,
					Path: app.KustomizeConfig.RepoRef.Path,
				}
				kconfig.RepoRef = kref

				// Use application to infer whether UseBasicAuth is true.
				if kref.Path == "common/basic-auth" {
					config.Spec.UseBasicAuth = true
				}
			}
			for _, param := range app.KustomizeConfig.Parameters {
				p := kfconfig.NameValue{
					Name:  param.Name,
					Value: param.Value,
				}
				kconfig.Parameters = append(kconfig.Parameters, p)
			}
			application.KustomizeConfig = kconfig
		}
		config.Spec.Applications = append(config.Spec.Applications, application)
	}

	for _, plugin := range kfdef.Spec.Plugins {
		p := kfconfig.Plugin{
			Name:      plugin.Name,
			Namespace: kfdef.Namespace,
			Kind:      kfconfig.PluginKindType(plugin.Kind),
			Spec:      plugin.Spec,
		}
		config.Spec.Plugins = append(config.Spec.Plugins, p)

		if plugin.Kind == string(kfconfig.GCP_PLUGIN_KIND) {
			specBytes, err := yaml.Marshal(plugin.Spec)
			if err != nil {
				return nil, &kfapis.KfError{
					Code:    int(kfapis.INTERNAL_ERROR),
					Message: fmt.Sprintf("could not marshal GCP plugin spec: %v", err),
				}
			}
			var s map[string]interface{}
			err = yaml.Unmarshal(specBytes, &s)
			if err != nil {
				return nil, &kfapis.KfError{
					Code:    int(kfapis.INTERNAL_ERROR),
					Message: fmt.Sprintf("could not unmarshal GCP plugin spec: %v", err),
				}
			}
			if p, ok := s["project"]; ok {
				config.Spec.Project = p.(string)
			}
			if e, ok := s["email"]; ok {
				config.Spec.Email = e.(string)
			}
			if i, ok := s["ipName"]; ok {
				config.Spec.IpName = i.(string)
			}
			if h, ok := s["hostname"]; ok {
				config.Spec.Hostname = h.(string)
			}
			if h, ok := s["skipInitProject"]; ok {
				config.Spec.SkipInitProject = h.(bool)
			}
			if z, ok := s["zone"]; ok {
				config.Spec.Zone = z.(string)
			}
		}
		if p := maybeGetPlatform(plugin.Kind); p != "" {
			config.Spec.Platform = p
		}
	}

	for _, secret := range kfdef.Spec.Secrets {
		s := kfconfig.Secret{
			Name: secret.Name,
		}
		// We don't want to store literalSource explictly, becasue we want the config to be checked into source control and don't want secrets in source control.
		if secret.SecretSource == nil || secret.SecretSource.LiteralSource != nil {
			config.Spec.Secrets = append(config.Spec.Secrets, s)
			continue
		}
		src := &kfconfig.SecretSource{}
		if secret.SecretSource.EnvSource != nil {
			src.EnvSource = &kfconfig.EnvSource{
				Name: secret.SecretSource.EnvSource.Name,
			}
		}
		s.SecretSource = src
		config.Spec.Secrets = append(config.Spec.Secrets, s)
	}

	for _, repo := range kfdef.Spec.Repos {
		r := kfconfig.Repo{
			Name: repo.Name,
			URI:  repo.URI,
		}
		config.Spec.Repos = append(config.Spec.Repos, r)
	}

	for _, cond := range kfdef.Status.Conditions {
		c := kfconfig.Condition{
			Type:               kfconfig.ConditionType(cond.Type),
			Status:             cond.Status,
			LastUpdateTime:     cond.LastUpdateTime,
			LastTransitionTime: cond.LastTransitionTime,
			Reason:             cond.Reason,
			Message:            cond.Message,
		}
		config.Status.Conditions = append(config.Status.Conditions, c)
	}
	for _, cache := range kfdef.Status.ReposCache {
		c := kfconfig.Cache{
			Name:      cache.Name,
			LocalPath: cache.LocalPath,
		}
		config.Status.Caches = append(config.Status.Caches, c)
	}

	return config, nil

}

func (v V1beta1) ToKfDefSerialized(config kfconfig.KfConfig) ([]byte, error) {
	kfdef := &kfdeftypes.KfDef{}
	kfdef.Name = config.Name
	kfdef.Namespace = config.Namespace
	kfdef.APIVersion = config.APIVersion
	kfdef.Kind = "KfDef"
	kfdef.Labels = config.Labels
	kfdef.Annotations = config.Annotations
	kfdef.Spec.Version = config.Spec.Version

	for _, app := range config.Spec.Applications {
		application := kfdeftypes.Application{
			Name: app.Name,
		}
		if app.KustomizeConfig != nil {
			kconfig := &kfdeftypes.KustomizeConfig{
				Overlays: app.KustomizeConfig.Overlays,
			}
			if app.KustomizeConfig.RepoRef != nil {
				kref := &kfdeftypes.RepoRef{
					Name: app.KustomizeConfig.RepoRef.Name,
					Path: app.KustomizeConfig.RepoRef.Path,
				}
				kconfig.RepoRef = kref
			}
			for _, param := range app.KustomizeConfig.Parameters {
				p := kfdeftypes.NameValue{
					Name:  param.Name,
					Value: param.Value,
				}
				kconfig.Parameters = append(kconfig.Parameters, p)
			}
			application.KustomizeConfig = kconfig
		}
		kfdef.Spec.Applications = append(kfdef.Spec.Applications, application)
	}

	for _, plugin := range config.Spec.Plugins {
		p := kfdeftypes.Plugin{
			Spec: plugin.Spec,
		}
		p.Name = plugin.Name
		p.Kind = string(plugin.Kind)
		kfdef.Spec.Plugins = append(kfdef.Spec.Plugins, p)
	}

	for _, secret := range config.Spec.Secrets {
		s := kfdeftypes.Secret{
			Name: secret.Name,
		}
		if secret.SecretSource != nil {
			s.SecretSource = &kfdeftypes.SecretSource{}
			if secret.SecretSource.LiteralSource != nil {
				s.SecretSource.LiteralSource = &kfdeftypes.LiteralSource{
					Value: secret.SecretSource.LiteralSource.Value,
				}
			}
			if secret.SecretSource.EnvSource != nil {
				s.SecretSource.EnvSource = &kfdeftypes.EnvSource{
					Name: secret.SecretSource.EnvSource.Name,
				}
			}
		}
		kfdef.Spec.Secrets = append(kfdef.Spec.Secrets, s)
	}

	for _, repo := range config.Spec.Repos {
		r := kfdeftypes.Repo{
			Name: repo.Name,
			URI:  repo.URI,
		}
		kfdef.Spec.Repos = append(kfdef.Spec.Repos, r)
	}

	for _, cond := range config.Status.Conditions {
		c := kfdeftypes.KfDefCondition{
			Type:               kfdeftypes.KfDefConditionType(cond.Type),
			Status:             cond.Status,
			LastUpdateTime:     cond.LastUpdateTime,
			LastTransitionTime: cond.LastTransitionTime,
			Reason:             cond.Reason,
			Message:            cond.Message,
		}
		kfdef.Status.Conditions = append(kfdef.Status.Conditions, c)
	}

	for _, cache := range config.Status.Caches {
		c := kfdeftypes.RepoCache{
			Name:      cache.Name,
			LocalPath: cache.LocalPath,
		}
		kfdef.Status.ReposCache = append(kfdef.Status.ReposCache, c)
	}

	kfdefBytes, err := yaml.Marshal(kfdef)
	if err == nil {
		return kfdefBytes, nil
	} else {
		return nil, &kfapis.KfError{
			Code:    int(kfapis.INTERNAL_ERROR),
			Message: fmt.Sprintf("error when marshaling to KfDef: %v", err),
		}
	}
}
<commit_msg>keep literal secret when convert kfdef to kfconfig (#98)<commit_after>package configconverters

import (
	"fmt"

	"github.com/ghodss/yaml"
	kfapis "github.com/kubeflow/kfctl/v3/pkg/apis"
	kftypesv3 "github.com/kubeflow/kfctl/v3/pkg/apis/apps"
	kfconfig "github.com/kubeflow/kfctl/v3/pkg/apis/apps/kfconfig"
	kfdeftypes "github.com/kubeflow/kfctl/v3/pkg/apis/apps/kfdef/v1beta1"
)

// Empty struct - used to implement Converter interface.
type V1beta1 struct {
}

func maybeGetPlatform(pluginKind string) string {
	platforms := map[string]string{
		string(kfconfig.AWS_PLUGIN_KIND):              kftypesv3.AWS,
		string(kfconfig.GCP_PLUGIN_KIND):              kftypesv3.GCP,
		string(kfconfig.EXISTING_ARRIKTO_PLUGIN_KIND): kftypesv3.EXISTING_ARRIKTO,
	}

	p, ok := platforms[pluginKind]
	if ok {
		return p
	} else {
		return ""
	}
}

func (v V1beta1) ToKfConfig(kfdefBytes []byte) (*kfconfig.KfConfig, error) {
	kfdef := &kfdeftypes.KfDef{}
	if err := yaml.Unmarshal(kfdefBytes, kfdef); err != nil {
		return nil, &kfapis.KfError{
			Code:    int(kfapis.INTERNAL_ERROR),
			Message: fmt.Sprintf("could not unmarshal config file onto KfDef struct: %v", err),
		}
	}

	// Set UseBasicAuth later.
	config := &kfconfig.KfConfig{
		Spec: kfconfig.KfConfigSpec{
			UseBasicAuth: false,
		},
	}
	config.Name = kfdef.Name
	config.Namespace = kfdef.Namespace
	config.APIVersion = kfdef.APIVersion
	config.Kind = "KfConfig"
	config.Labels = kfdef.Labels
	config.Annotations = kfdef.Annotations
	config.Spec.Version = kfdef.Spec.Version
	for _, app := range kfdef.Spec.Applications {
		application := kfconfig.Application{
			Name: app.Name,
		}
		if app.KustomizeConfig != nil {
			kconfig := &kfconfig.KustomizeConfig{
				Overlays: app.KustomizeConfig.Overlays,
			}
			if app.KustomizeConfig.RepoRef != nil {
				kref := &kfconfig.RepoRef{
					Name: app.KustomizeConfig.RepoRef.Name,
					Path: app.KustomizeConfig.RepoRef.Path,
				}
				kconfig.RepoRef = kref

				// Use application to infer whether UseBasicAuth is true.
				if kref.Path == "common/basic-auth" {
					config.Spec.UseBasicAuth = true
				}
			}
			for _, param := range app.KustomizeConfig.Parameters {
				p := kfconfig.NameValue{
					Name:  param.Name,
					Value: param.Value,
				}
				kconfig.Parameters = append(kconfig.Parameters, p)
			}
			application.KustomizeConfig = kconfig
		}
		config.Spec.Applications = append(config.Spec.Applications, application)
	}

	for _, plugin := range kfdef.Spec.Plugins {
		p := kfconfig.Plugin{
			Name:      plugin.Name,
			Namespace: kfdef.Namespace,
			Kind:      kfconfig.PluginKindType(plugin.Kind),
			Spec:      plugin.Spec,
		}
		config.Spec.Plugins = append(config.Spec.Plugins, p)

		if plugin.Kind == string(kfconfig.GCP_PLUGIN_KIND) {
			specBytes, err := yaml.Marshal(plugin.Spec)
			if err != nil {
				return nil, &kfapis.KfError{
					Code:    int(kfapis.INTERNAL_ERROR),
					Message: fmt.Sprintf("could not marshal GCP plugin spec: %v", err),
				}
			}
			var s map[string]interface{}
			err = yaml.Unmarshal(specBytes, &s)
			if err != nil {
				return nil, &kfapis.KfError{
					Code:    int(kfapis.INTERNAL_ERROR),
					Message: fmt.Sprintf("could not unmarshal GCP plugin spec: %v", err),
				}
			}
			if p, ok := s["project"]; ok {
				config.Spec.Project = p.(string)
			}
			if e, ok := s["email"]; ok {
				config.Spec.Email = e.(string)
			}
			if i, ok := s["ipName"]; ok {
				config.Spec.IpName = i.(string)
			}
			if h, ok := s["hostname"]; ok {
				config.Spec.Hostname = h.(string)
			}
			if h, ok := s["skipInitProject"]; ok {
				config.Spec.SkipInitProject = h.(bool)
			}
			if z, ok := s["zone"]; ok {
				config.Spec.Zone = z.(string)
			}
		}
		if p := maybeGetPlatform(plugin.Kind); p != "" {
			config.Spec.Platform = p
		}
	}

	for _, secret := range kfdef.Spec.Secrets {
		s := kfconfig.Secret{
			Name: secret.Name,
		}
		src := &kfconfig.SecretSource{}
		if secret.SecretSource.EnvSource != nil {
			src.EnvSource = &kfconfig.EnvSource{
				Name: secret.SecretSource.EnvSource.Name,
			}
		}
		if secret.SecretSource.LiteralSource != nil {
			src.LiteralSource = &kfconfig.LiteralSource{
				Value: secret.SecretSource.LiteralSource.Value,
			}
		}
		s.SecretSource = src
		config.Spec.Secrets = append(config.Spec.Secrets, s)
	}

	for _, repo := range kfdef.Spec.Repos {
		r := kfconfig.Repo{
			Name: repo.Name,
			URI:  repo.URI,
		}
		config.Spec.Repos = append(config.Spec.Repos, r)
	}

	for _, cond := range kfdef.Status.Conditions {
		c := kfconfig.Condition{
			Type:               kfconfig.ConditionType(cond.Type),
			Status:             cond.Status,
			LastUpdateTime:     cond.LastUpdateTime,
			LastTransitionTime: cond.LastTransitionTime,
			Reason:             cond.Reason,
			Message:            cond.Message,
		}
		config.Status.Conditions = append(config.Status.Conditions, c)
	}
	for _, cache := range kfdef.Status.ReposCache {
		c := kfconfig.Cache{
			Name:      cache.Name,
			LocalPath: cache.LocalPath,
		}
		config.Status.Caches = append(config.Status.Caches, c)
	}

	return config, nil

}

func (v V1beta1) ToKfDefSerialized(config kfconfig.KfConfig) ([]byte, error) {
	kfdef := &kfdeftypes.KfDef{}
	kfdef.Name = config.Name
	kfdef.Namespace = config.Namespace
	kfdef.APIVersion = config.APIVersion
	kfdef.Kind = "KfDef"
	kfdef.Labels = config.Labels
	kfdef.Annotations = config.Annotations
	kfdef.Spec.Version = config.Spec.Version

	for _, app := range config.Spec.Applications {
		application := kfdeftypes.Application{
			Name: app.Name,
		}
		if app.KustomizeConfig != nil {
			kconfig := &kfdeftypes.KustomizeConfig{
				Overlays: app.KustomizeConfig.Overlays,
			}
			if app.KustomizeConfig.RepoRef != nil {
				kref := &kfdeftypes.RepoRef{
					Name: app.KustomizeConfig.RepoRef.Name,
					Path: app.KustomizeConfig.RepoRef.Path,
				}
				kconfig.RepoRef = kref
			}
			for _, param := range app.KustomizeConfig.Parameters {
				p := kfdeftypes.NameValue{
					Name:  param.Name,
					Value: param.Value,
				}
				kconfig.Parameters = append(kconfig.Parameters, p)
			}
			application.KustomizeConfig = kconfig
		}
		kfdef.Spec.Applications = append(kfdef.Spec.Applications, application)
	}

	for _, plugin := range config.Spec.Plugins {
		p := kfdeftypes.Plugin{
			Spec: plugin.Spec,
		}
		p.Name = plugin.Name
		p.Kind = string(plugin.Kind)
		kfdef.Spec.Plugins = append(kfdef.Spec.Plugins, p)
	}

	for _, secret := range config.Spec.Secrets {
		s := kfdeftypes.Secret{
			Name: secret.Name,
		}
		if secret.SecretSource != nil {
			s.SecretSource = &kfdeftypes.SecretSource{}
			if secret.SecretSource.LiteralSource != nil {
				s.SecretSource.LiteralSource = &kfdeftypes.LiteralSource{
					Value: secret.SecretSource.LiteralSource.Value,
				}
			}
			if secret.SecretSource.EnvSource != nil {
				s.SecretSource.EnvSource = &kfdeftypes.EnvSource{
					Name: secret.SecretSource.EnvSource.Name,
				}
			}
		}
		kfdef.Spec.Secrets = append(kfdef.Spec.Secrets, s)
	}

	for _, repo := range config.Spec.Repos {
		r := kfdeftypes.Repo{
			Name: repo.Name,
			URI:  repo.URI,
		}
		kfdef.Spec.Repos = append(kfdef.Spec.Repos, r)
	}

	for _, cond := range config.Status.Conditions {
		c := kfdeftypes.KfDefCondition{
			Type:               kfdeftypes.KfDefConditionType(cond.Type),
			Status:             cond.Status,
			LastUpdateTime:     cond.LastUpdateTime,
			LastTransitionTime: cond.LastTransitionTime,
			Reason:             cond.Reason,
			Message:            cond.Message,
		}
		kfdef.Status.Conditions = append(kfdef.Status.Conditions, c)
	}

	for _, cache := range config.Status.Caches {
		c := kfdeftypes.RepoCache{
			Name:      cache.Name,
			LocalPath: cache.LocalPath,
		}
		kfdef.Status.ReposCache = append(kfdef.Status.ReposCache, c)
	}

	kfdefBytes, err := yaml.Marshal(kfdef)
	if err == nil {
		return kfdefBytes, nil
	} else {
		return nil, &kfapis.KfError{
			Code:    int(kfapis.INTERNAL_ERROR),
			Message: fmt.Sprintf("error when marshaling to KfDef: %v", err),
		}
	}
}
<|endoftext|>


<commit_before>package rbd

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestGetFeatures(t *testing.T) {
	conn := radosConnect(t)
	require.NotNil(t, conn)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	require.NoError(t, err)
	defer conn.DeletePool(poolname)

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()

	options := NewRbdImageOptions()
	err = options.SetUint64(RbdImageOptionFeatures, FeatureLayering|FeatureStripingV2)
	require.NoError(t, err)
	// FeatureStripingV2 only works with additional arguments
	err = options.SetUint64(RbdImageOptionStripeUnit, 1024*1024)
	require.NoError(t, err)
	err = options.SetUint64(RbdImageOptionStripeCount, 4)
	require.NoError(t, err)

	err = CreateImage(ioctx, name, 16*1024*1024, options)
	require.NoError(t, err)
	defer func() { assert.NoError(t, RemoveImage(ioctx, name)) }()

	image, err := OpenImageReadOnly(ioctx, name, NoSnapshot)
	assert.NoError(t, err)
	defer func() { assert.NoError(t, image.Close()) }()

	features, err := image.GetFeatures()
	assert.NoError(t, err)

	t.Run("compareBits", func(t *testing.T) {
		hasLayering := (features & FeatureLayering) == FeatureLayering
		hasStripingV2 := (features & FeatureStripingV2) == FeatureStripingV2
		assert.True(t, hasLayering, "FeatureLayering is not set")
		assert.True(t, hasStripingV2, "FeatureStripingV2 is not set")
	})
}

func TestUpdateFeatures(t *testing.T) {
	conn := radosConnect(t)
	require.NotNil(t, conn)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	require.NoError(t, err)
	defer conn.DeletePool(poolname)

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()

	options := NewRbdImageOptions()
	// test with FeatureExclusiveLock as that is mutable
	err = options.SetUint64(RbdImageOptionFeatures, FeatureExclusiveLock)
	require.NoError(t, err)

	err = CreateImage(ioctx, name, 16*1024*1024, options)
	require.NoError(t, err)
	defer func() { assert.NoError(t, RemoveImage(ioctx, name)) }()

	image, err := OpenImage(ioctx, name, NoSnapshot)
	require.NoError(t, err)
	defer func() { assert.NoError(t, image.Close()) }()

	t.Run("imageNotOpen", func(t *testing.T) {
		img, err := OpenImageReadOnly(ioctx, name, NoSnapshot)
		require.NoError(t, err)
		require.NotNil(t, img)

		err = img.Close()
		require.NoError(t, err)

		err = img.UpdateFeatures(FeatureExclusiveLock, false)
		assert.Equal(t, err, ErrImageNotOpen)
	})

	t.Run("verifyFeatureEnabled", func(t *testing.T) {
		features, err := image.GetFeatures()
		require.NoError(t, err)

		hasExclusiveLock := (features & FeatureExclusiveLock) == FeatureExclusiveLock
		require.True(t, hasExclusiveLock, "FeatureExclusiveLock is not set")
	})

	t.Run("disableFeature", func(t *testing.T) {
		err = image.UpdateFeatures(FeatureExclusiveLock, false)
		require.NoError(t, err)

		features, err := image.GetFeatures()
		require.NoError(t, err)

		hasExclusiveLock := (features & FeatureExclusiveLock) == FeatureExclusiveLock
		require.False(t, hasExclusiveLock, "FeatureExclusiveLock is set")
	})

	t.Run("enableFeature", func(t *testing.T) {
		err = image.UpdateFeatures(FeatureExclusiveLock, true)
		require.NoError(t, err)

		features, err := image.GetFeatures()
		require.NoError(t, err)

		hasExclusiveLock := (features & FeatureExclusiveLock) == FeatureExclusiveLock
		require.True(t, hasExclusiveLock, "FeatureExclusiveLock is not set")
	})
}
<commit_msg>rbd/features: add test for FeatureSet<commit_after>package rbd

import (
	"sort"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestGetFeatures(t *testing.T) {
	conn := radosConnect(t)
	require.NotNil(t, conn)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	require.NoError(t, err)
	defer conn.DeletePool(poolname)

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()

	options := NewRbdImageOptions()
	err = options.SetUint64(RbdImageOptionFeatures, FeatureLayering|FeatureStripingV2)
	require.NoError(t, err)
	// FeatureStripingV2 only works with additional arguments
	err = options.SetUint64(RbdImageOptionStripeUnit, 1024*1024)
	require.NoError(t, err)
	err = options.SetUint64(RbdImageOptionStripeCount, 4)
	require.NoError(t, err)

	err = CreateImage(ioctx, name, 16*1024*1024, options)
	require.NoError(t, err)
	defer func() { assert.NoError(t, RemoveImage(ioctx, name)) }()

	image, err := OpenImageReadOnly(ioctx, name, NoSnapshot)
	assert.NoError(t, err)
	defer func() { assert.NoError(t, image.Close()) }()

	features, err := image.GetFeatures()
	assert.NoError(t, err)

	t.Run("compareBits", func(t *testing.T) {
		hasLayering := (features & FeatureLayering) == FeatureLayering
		hasStripingV2 := (features & FeatureStripingV2) == FeatureStripingV2
		assert.True(t, hasLayering, "FeatureLayering is not set")
		assert.True(t, hasStripingV2, "FeatureStripingV2 is not set")
	})

	t.Run("compareFeatureSet", func(t *testing.T) {
		fs := FeatureSet(features)
		assert.Contains(t, fs.Names(), FeatureNameLayering)
		assert.Contains(t, fs.Names(), FeatureNameStripingV2)
	})
}

func TestFeatureSet(t *testing.T) {
	fsBits := FeatureSet(FeatureExclusiveLock | FeatureDeepFlatten)
	fsNames := FeatureSetFromNames([]string{FeatureNameExclusiveLock, FeatureNameDeepFlatten})
	assert.Equal(t, fsBits, fsNames)

	fsBitsSorted := fsBits.Names()
	sort.Strings(fsBitsSorted)

	fsNamesSorted := fsNames.Names()
	sort.Strings(fsNamesSorted)

	assert.Equal(t, fsBitsSorted, fsNamesSorted)
}

func TestUpdateFeatures(t *testing.T) {
	conn := radosConnect(t)
	require.NotNil(t, conn)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	require.NoError(t, err)
	defer conn.DeletePool(poolname)

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()

	options := NewRbdImageOptions()
	// test with FeatureExclusiveLock as that is mutable
	err = options.SetUint64(RbdImageOptionFeatures, FeatureExclusiveLock)
	require.NoError(t, err)

	err = CreateImage(ioctx, name, 16*1024*1024, options)
	require.NoError(t, err)
	defer func() { assert.NoError(t, RemoveImage(ioctx, name)) }()

	image, err := OpenImage(ioctx, name, NoSnapshot)
	require.NoError(t, err)
	defer func() { assert.NoError(t, image.Close()) }()

	t.Run("imageNotOpen", func(t *testing.T) {
		img, err := OpenImageReadOnly(ioctx, name, NoSnapshot)
		require.NoError(t, err)
		require.NotNil(t, img)

		err = img.Close()
		require.NoError(t, err)

		err = img.UpdateFeatures(FeatureExclusiveLock, false)
		assert.Equal(t, err, ErrImageNotOpen)
	})

	t.Run("verifyFeatureEnabled", func(t *testing.T) {
		features, err := image.GetFeatures()
		require.NoError(t, err)

		hasExclusiveLock := (features & FeatureExclusiveLock) == FeatureExclusiveLock
		require.True(t, hasExclusiveLock, "FeatureExclusiveLock is not set")
	})

	t.Run("disableFeature", func(t *testing.T) {
		err = image.UpdateFeatures(FeatureExclusiveLock, false)
		require.NoError(t, err)

		features, err := image.GetFeatures()
		require.NoError(t, err)

		hasExclusiveLock := (features & FeatureExclusiveLock) == FeatureExclusiveLock
		require.False(t, hasExclusiveLock, "FeatureExclusiveLock is set")
	})

	t.Run("enableFeature", func(t *testing.T) {
		err = image.UpdateFeatures(FeatureExclusiveLock, true)
		require.NoError(t, err)

		features, err := image.GetFeatures()
		require.NoError(t, err)

		hasExclusiveLock := (features & FeatureExclusiveLock) == FeatureExclusiveLock
		require.True(t, hasExclusiveLock, "FeatureExclusiveLock is not set")
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package process

import (
	"fmt"

	"github.com/richardlehane/siegfried/config"
	"github.com/richardlehane/siegfried/pkg/core/bytematcher/frames"
)

// positioning information: min/max offsets (in relation to BOF or EOF) and min/max lengths
type keyFramePos struct {
	// Minimum and maximum position
	PMin int
	PMax int
	// Minimum and maximum length
	LMin int
	LMax int
}

// Each segment in a signature is represented by a single keyFrame. A slice of keyFrames represents a full signature.
// The keyFrame includes the range of offsets that need to match for a successful hit.
// The segment (Seg) offsets are relative (to preceding/succeding segments or to BOF/EOF if the first or last segment).
// The keyframe (Key) offsets are absolute to the BOF or EOF.
type keyFrame struct {
	Typ frames.OffType // BOF|PREV|SUCC|EOF
	Seg keyFramePos    // relative positioning info for segment as a whole (min/max length and offset in relation to BOF/EOF/PREV/SUCC)
	Key keyFramePos    // absolute positioning info for keyFrame portion of segment (min/max length and offset in relation to BOF/EOF)
}

func (kf keyFrame) String() string {
	return fmt.Sprintf("%s Min:%d Max:%d", frames.OffString[kf.Typ], kf.Seg.PMin, kf.Seg.PMax)
}

// A double index: the first int is for the signature's position within the set of all signatures,
// the second int is for the keyFrames position within the segments of the signature.
type KeyFrameID [2]int

func (kf KeyFrameID) String() string {
	return fmt.Sprintf("[%d:%d]", kf[0], kf[1])
}

// Turn a signature segment into a keyFrame and left and right frame slices.
// The left and right frame slices are converted into BMH sequences where possible
func toKeyFrame(seg frames.Signature, pos position) (keyFrame, []frames.Frame, []frames.Frame) {
	var left, right []frames.Frame
	var typ frames.OffType
	var segPos, keyPos keyFramePos
	segPos.LMin, segPos.LMax = calcLen(seg)
	keyPos.LMin, keyPos.LMax = calcLen(seg[pos.start:pos.end])
	// BOF and PREV segments
	if seg[0].Orientation() < frames.SUCC {
		typ, segPos.PMin, segPos.PMax = seg[0].Orientation(), seg[0].Min(), seg[0].Max()
		keyPos.PMin, keyPos.PMax = segPos.PMin, segPos.PMax
		for i, f := range seg[:pos.start+1] {
			if pos.start > i {
				min, max := f.Length()
				keyPos.PMin += min
				keyPos.PMin += seg[i+1].Min()
				if keyPos.PMax > -1 {
					keyPos.PMax += max
					keyPos.PMax += seg[i+1].Max()
				}
				left = append([]frames.Frame{frames.SwitchFrame(seg[i+1], f.Pat())}, left...)
			}
		}
		if pos.end < len(seg) {
			right = seg[pos.end:]
		}
		return keyFrame{typ, segPos, keyPos}, frames.BMHConvert(left, true), frames.BMHConvert(right, false)
	}
	// EOF and SUCC segments
	typ, segPos.PMin, segPos.PMax = seg[len(seg)-1].Orientation(), seg[len(seg)-1].Min(), seg[len(seg)-1].Max()
	keyPos.PMin, keyPos.PMax = segPos.PMin, segPos.PMax
	if pos.end < len(seg) {
		for i, f := range seg[pos.end:] {
			min, max := f.Length()
			keyPos.PMin += min
			keyPos.PMin += seg[pos.end+i-1].Min()
			if keyPos.PMax > -1 {
				keyPos.PMax += max
				keyPos.PMax += seg[pos.end+i-1].Max()
			}
			right = append(right, frames.SwitchFrame(seg[pos.end+i-1], f.Pat()))
		}
	}
	for _, f := range seg[:pos.start] {
		left = append([]frames.Frame{f}, left...)
	}
	return keyFrame{typ, segPos, keyPos}, frames.BMHConvert(left, true), frames.BMHConvert(right, false)
}

// calculate minimum and maximum lengths for a segment (slice of frames)
func calcLen(fs []frames.Frame) (int, int) {
	var min, max int
	for _, f := range fs {
		fmin, fmax := f.Length()
		min += fmin
		max += fmax
	}
	return min, max
}

func calcMinMax(min, max int, sp keyFramePos) (int, int) {
	min = min + sp.PMin + sp.LMin
	if max < 0 || sp.PMax < 0 {
		return min, -1
	}
	max = max + sp.PMax + sp.LMax
	return min, max
}

// update the absolute positional information (distance from the BOF or EOF)
// for keyFrames based on the other keyFrames in the signature
// This function is also responsible for applying MAX BOF and MAX EOF settings from config
// MAX BOF and MAX EOF don't affect segment offsets (relative to other segments), just affect the Abs position offsets
func updatePositions(ks []keyFrame) {
	var min, max int
	// first forwards, for BOF and PREV
	for i := range ks {
		if ks[i].Typ == frames.BOF {
			min, max = calcMinMax(0, 0, ks[i].Seg)
		}
		if ks[i].Typ == frames.PREV {
			ks[i].Key.PMin = min + ks[i].Key.PMin
			if max > -1 && ks[i].Key.PMax > -1 {
				ks[i].Key.PMax = max + ks[i].Key.PMax
			} else {
				ks[i].Key.PMax = -1
			}
			min, max = calcMinMax(min, max, ks[i].Seg)
		}
		// Apply config max bof setting (if any) to PMax
		if config.MaxBOF() > 0 {
			if ks[i].Key.PMax < 0 || ks[i].Key.PMax > config.MaxBOF() {
				ks[i].Key.PMax = config.MaxBOF()
			}
		}
	}
	// now backwards for EOF and SUCC
	min, max = 0, 0
	for i := len(ks) - 1; i >= 0; i-- {
		if ks[i].Typ == frames.EOF {
			min, max = calcMinMax(0, 0, ks[i].Seg)
		}
		if ks[i].Typ == frames.SUCC {
			ks[i].Key.PMin = min + ks[i].Key.PMin
			if max > -1 && ks[i].Key.PMax > -1 {
				ks[i].Key.PMax = max + ks[i].Key.PMax
			} else {
				ks[i].Key.PMax = -1
			}
			min, max = calcMinMax(min, max, ks[i].Seg)
		}
		// Apply config max eof setting (if any) to PMax
		if config.MaxEOF() > 0 {
			if ks[i].Key.PMax < 0 || ks[i].Key.PMax > config.MaxEOF() {
				ks[i].Key.PMax = config.MaxEOF()
			}
		}
	}
}

// for doing a running total of the maxBOF:
// is the maxBOF we already have, further from the BOF than the maxBOF of the current signature?
func maxBOF(max int, ks []keyFrame) int {
	if max < 0 {
		return -1
	}
	for _, v := range ks {
		if v.Typ < frames.SUCC {
			if v.Key.PMax < 0 {
				return -1
			}
			if v.Key.PMax+v.Key.LMax > max {
				max = v.Key.PMax + v.Key.LMax
			}
		}
	}
	return max
}

func maxEOF(max int, ks []keyFrame) int {
	if max < 0 {
		return -1
	}
	for _, v := range ks {
		if v.Typ > frames.PREV {
			if v.Key.PMax < 0 {
				return -1
			}
			if v.Key.PMax+v.Key.LMax > max {
				max = v.Key.PMax + v.Key.LMax
			}
		}
	}
	return max
}

// quick check performed before applying a keyFrame ID
func (kf keyFrame) Check(o int) bool {
	if kf.Key.PMin > o {
		return false
	}
	if kf.Key.PMax == -1 {
		return true
	}
	if kf.Key.PMax < o {
		return false
	}
	return true
}

// proper segment check before committing an incomplete keyframe (necessary when there are left or right tests)
func (kf keyFrame) CheckSeg(o int) bool {
	if kf.Seg.PMin > o {
		return false
	}
	if kf.Seg.PMax == -1 {
		return true
	}
	if kf.Seg.PMax < o {
		return false
	}
	return true
}

// test two key frames (current and previous) to see if they are connected and, if so, at what offsets
func (kf keyFrame) CheckRelated(prevKf keyFrame, thisOff, prevOff [][2]int) ([][2]int, bool) {
	// quick test for wild kf
	if prevKf.Seg.PMax == -1 && prevKf.Seg.PMin == 0 {
		return thisOff, true
	}
	switch kf.Typ {
	case frames.BOF:
		return thisOff, true
	case frames.EOF, frames.SUCC:
		if prevKf.Typ == frames.SUCC {
			ret := make([][2]int, 0, len(thisOff))
			success := false
			for _, v := range thisOff {
				for _, v1 := range prevOff {
					dif := v[0] - v1[0] - v1[1]
					if dif > -1 {
						if dif < prevKf.Seg.PMin || (prevKf.Seg.PMax > -1 && dif > prevKf.Seg.PMax) {
							continue
						} else {
							ret = append(ret, v)
							success = true
						}
					}
				}
			}
			return ret, success
		} else {
			return thisOff, true
		}
	default:
		ret := make([][2]int, 0, len(thisOff))
		success := false
		for _, v := range thisOff {
			for _, v1 := range prevOff {
				dif := v[0] - v1[0] - v1[1] // current offset, minus previous offset, minus previous length
				if dif > -1 {
					if dif < kf.Seg.PMin || (kf.Seg.PMax > -1 && dif > kf.Seg.PMax) {
						continue
					} else {
						ret = append(ret, v)
						success = true
					}
				}
			}
		}
		return ret, success
	}
}
<commit_msg>fix to maxbof maxeof<commit_after>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package process

import (
	"fmt"

	"github.com/richardlehane/siegfried/config"
	"github.com/richardlehane/siegfried/pkg/core/bytematcher/frames"
)

// positioning information: min/max offsets (in relation to BOF or EOF) and min/max lengths
type keyFramePos struct {
	// Minimum and maximum position
	PMin int
	PMax int
	// Minimum and maximum length
	LMin int
	LMax int
}

// Each segment in a signature is represented by a single keyFrame. A slice of keyFrames represents a full signature.
// The keyFrame includes the range of offsets that need to match for a successful hit.
// The segment (Seg) offsets are relative (to preceding/succeding segments or to BOF/EOF if the first or last segment).
// The keyframe (Key) offsets are absolute to the BOF or EOF.
type keyFrame struct {
	Typ frames.OffType // BOF|PREV|SUCC|EOF
	Seg keyFramePos    // relative positioning info for segment as a whole (min/max length and offset in relation to BOF/EOF/PREV/SUCC)
	Key keyFramePos    // absolute positioning info for keyFrame portion of segment (min/max length and offset in relation to BOF/EOF)
}

func (kf keyFrame) String() string {
	return fmt.Sprintf("%s Min:%d Max:%d", frames.OffString[kf.Typ], kf.Seg.PMin, kf.Seg.PMax)
}

// A double index: the first int is for the signature's position within the set of all signatures,
// the second int is for the keyFrames position within the segments of the signature.
type KeyFrameID [2]int

func (kf KeyFrameID) String() string {
	return fmt.Sprintf("[%d:%d]", kf[0], kf[1])
}

// Turn a signature segment into a keyFrame and left and right frame slices.
// The left and right frame slices are converted into BMH sequences where possible
func toKeyFrame(seg frames.Signature, pos position) (keyFrame, []frames.Frame, []frames.Frame) {
	var left, right []frames.Frame
	var typ frames.OffType
	var segPos, keyPos keyFramePos
	segPos.LMin, segPos.LMax = calcLen(seg)
	keyPos.LMin, keyPos.LMax = calcLen(seg[pos.start:pos.end])
	// BOF and PREV segments
	if seg[0].Orientation() < frames.SUCC {
		typ, segPos.PMin, segPos.PMax = seg[0].Orientation(), seg[0].Min(), seg[0].Max()
		keyPos.PMin, keyPos.PMax = segPos.PMin, segPos.PMax
		for i, f := range seg[:pos.start+1] {
			if pos.start > i {
				min, max := f.Length()
				keyPos.PMin += min
				keyPos.PMin += seg[i+1].Min()
				if keyPos.PMax > -1 {
					keyPos.PMax += max
					keyPos.PMax += seg[i+1].Max()
				}
				left = append([]frames.Frame{frames.SwitchFrame(seg[i+1], f.Pat())}, left...)
			}
		}
		if pos.end < len(seg) {
			right = seg[pos.end:]
		}
		return keyFrame{typ, segPos, keyPos}, frames.BMHConvert(left, true), frames.BMHConvert(right, false)
	}
	// EOF and SUCC segments
	typ, segPos.PMin, segPos.PMax = seg[len(seg)-1].Orientation(), seg[len(seg)-1].Min(), seg[len(seg)-1].Max()
	keyPos.PMin, keyPos.PMax = segPos.PMin, segPos.PMax
	if pos.end < len(seg) {
		for i, f := range seg[pos.end:] {
			min, max := f.Length()
			keyPos.PMin += min
			keyPos.PMin += seg[pos.end+i-1].Min()
			if keyPos.PMax > -1 {
				keyPos.PMax += max
				keyPos.PMax += seg[pos.end+i-1].Max()
			}
			right = append(right, frames.SwitchFrame(seg[pos.end+i-1], f.Pat()))
		}
	}
	for _, f := range seg[:pos.start] {
		left = append([]frames.Frame{f}, left...)
	}
	return keyFrame{typ, segPos, keyPos}, frames.BMHConvert(left, true), frames.BMHConvert(right, false)
}

// calculate minimum and maximum lengths for a segment (slice of frames)
func calcLen(fs []frames.Frame) (int, int) {
	var min, max int
	for _, f := range fs {
		fmin, fmax := f.Length()
		min += fmin
		max += fmax
	}
	return min, max
}

func calcMinMax(min, max int, sp keyFramePos) (int, int) {
	min = min + sp.PMin + sp.LMin
	if max < 0 || sp.PMax < 0 {
		return min, -1
	}
	max = max + sp.PMax + sp.LMax
	return min, max
}

// update the absolute positional information (distance from the BOF or EOF)
// for keyFrames based on the other keyFrames in the signature
func updatePositions(ks []keyFrame) {
	var min, max int
	// first forwards, for BOF and PREV
	for i := range ks {
		if ks[i].Typ == frames.BOF {
			min, max = calcMinMax(0, 0, ks[i].Seg)
			// Apply max bof
			if config.MaxBOF() > 0 {
				if ks[i].Key.PMax < 0 || ks[i].Key.PMax > config.MaxBOF() {
					ks[i].Key.PMax = config.MaxBOF()
				}
			}
		}
		if ks[i].Typ == frames.PREV {
			ks[i].Key.PMin = min + ks[i].Key.PMin
			if max > -1 && ks[i].Key.PMax > -1 {
				ks[i].Key.PMax = max + ks[i].Key.PMax
			} else {
				ks[i].Key.PMax = -1
			}
			min, max = calcMinMax(min, max, ks[i].Seg)
			// Apply max bof
			if config.MaxBOF() > 0 {
				if ks[i].Key.PMax < 0 || ks[i].Key.PMax > config.MaxBOF() {
					ks[i].Key.PMax = config.MaxBOF()
				}
			}
		}
	}
	// now backwards for EOF and SUCC
	min, max = 0, 0
	for i := len(ks) - 1; i >= 0; i-- {
		if ks[i].Typ == frames.EOF {
			min, max = calcMinMax(0, 0, ks[i].Seg)
			// apply max eof
			if config.MaxEOF() > 0 {
				if ks[i].Key.PMax < 0 || ks[i].Key.PMax > config.MaxEOF() {
					ks[i].Key.PMax = config.MaxEOF()
				}
			}
		}
		if ks[i].Typ == frames.SUCC {
			ks[i].Key.PMin = min + ks[i].Key.PMin
			if max > -1 && ks[i].Key.PMax > -1 {
				ks[i].Key.PMax = max + ks[i].Key.PMax
			} else {
				ks[i].Key.PMax = -1
			}
			min, max = calcMinMax(min, max, ks[i].Seg)
			// apply max eof
			if config.MaxEOF() > 0 {
				if ks[i].Key.PMax < 0 || ks[i].Key.PMax > config.MaxEOF() {
					ks[i].Key.PMax = config.MaxEOF()
				}
			}
		}
	}
}

// for doing a running total of the maxBOF:
// is the maxBOF we already have, further from the BOF than the maxBOF of the current signature?
func maxBOF(max int, ks []keyFrame) int {
	if max < 0 {
		return -1
	}
	for _, v := range ks {
		if v.Typ < frames.SUCC {
			if v.Key.PMax < 0 {
				return -1
			}
			if v.Key.PMax+v.Key.LMax > max {
				max = v.Key.PMax + v.Key.LMax
			}
		}
	}
	return max
}

func maxEOF(max int, ks []keyFrame) int {
	if max < 0 {
		return -1
	}
	for _, v := range ks {
		if v.Typ > frames.PREV {
			if v.Key.PMax < 0 {
				return -1
			}
			if v.Key.PMax+v.Key.LMax > max {
				max = v.Key.PMax + v.Key.LMax
			}
		}
	}
	return max
}

// quick check performed before applying a keyFrame ID
func (kf keyFrame) Check(o int) bool {
	if kf.Key.PMin > o {
		return false
	}
	if kf.Key.PMax == -1 {
		return true
	}
	if kf.Key.PMax < o {
		return false
	}
	return true
}

// proper segment check before committing an incomplete keyframe (necessary when there are left or right tests)
func (kf keyFrame) CheckSeg(o int) bool {
	if kf.Seg.PMin > o {
		return false
	}
	if kf.Seg.PMax == -1 {
		return true
	}
	if kf.Seg.PMax < o {
		return false
	}
	return true
}

// test two key frames (current and previous) to see if they are connected and, if so, at what offsets
func (kf keyFrame) CheckRelated(prevKf keyFrame, thisOff, prevOff [][2]int) ([][2]int, bool) {
	// quick test for wild kf
	if prevKf.Seg.PMax == -1 && prevKf.Seg.PMin == 0 {
		return thisOff, true
	}
	switch kf.Typ {
	case frames.BOF:
		return thisOff, true
	case frames.EOF, frames.SUCC:
		if prevKf.Typ == frames.SUCC {
			ret := make([][2]int, 0, len(thisOff))
			success := false
			for _, v := range thisOff {
				for _, v1 := range prevOff {
					dif := v[0] - v1[0] - v1[1]
					if dif > -1 {
						if dif < prevKf.Seg.PMin || (prevKf.Seg.PMax > -1 && dif > prevKf.Seg.PMax) {
							continue
						} else {
							ret = append(ret, v)
							success = true
						}
					}
				}
			}
			return ret, success
		} else {
			return thisOff, true
		}
	default:
		ret := make([][2]int, 0, len(thisOff))
		success := false
		for _, v := range thisOff {
			for _, v1 := range prevOff {
				dif := v[0] - v1[0] - v1[1] // current offset, minus previous offset, minus previous length
				if dif > -1 {
					if dif < kf.Seg.PMin || (kf.Seg.PMax > -1 && dif > kf.Seg.PMax) {
						continue
					} else {
						ret = append(ret, v)
						success = true
					}
				}
			}
		}
		return ret, success
	}
}
<|endoftext|>