<commit_before>// Manage the configuration of setup_instruments.
package setup_instruments

import (
	"database/sql"
	"fmt"
	"log"

	"github.com/sjmudd/pstop/lib"
)

// We only match on the error number
// Error 1142: UPDATE command denied to user 'cacti'@'10.164.132.182' for table 'setup_instruments'
// Error 1290: The MySQL server is running with the --read-only option so it cannot execute this statement
var EXPECTED_UPDATE_ERRORS = []string{
	"Error 1142:",
	"Error 1290:",
}

// one row of performance_schema.setup_instruments
type table_row struct {
	NAME    string
	ENABLED string
	TIMED   string
}

type table_rows []table_row

// SetupInstruments "object"
type SetupInstruments struct {
	update_tried     bool
	update_succeeded bool
	rows             table_rows
	dbh              *sql.DB
}

// Return a newly initialised SetupInstruments structure with a handle to the database.
// Better to return a pointer ?
func NewSetupInstruments(dbh *sql.DB) SetupInstruments {
	return SetupInstruments{dbh: dbh}
}

// enable mutex and stage monitoring
func (si *SetupInstruments) EnableMonitoring() {
	si.EnableMutexMonitoring()
	si.EnableStageMonitoring()
}

// Change settings to monitor stage/sql/%
func (si *SetupInstruments) EnableStageMonitoring() {
	lib.Logger.Println("EnableStageMonitoring")
	sql := "SELECT NAME, ENABLED, TIMED FROM setup_instruments WHERE NAME LIKE 'stage/sql/%' AND ( enabled <> 'YES' OR timed <> 'YES' )"
	collecting := "Collecting setup_instruments stage/sql configuration settings"
	updating := "Updating setup_instruments configuration for: stage/sql"

	si.Configure(sql, collecting, updating)
	lib.Logger.Println("EnableStageMonitoring finishes")
}

// Change settings to monitor wait/synch/mutex/%
func (si *SetupInstruments) EnableMutexMonitoring() {
	lib.Logger.Println("EnableMutexMonitoring")
	sql := "SELECT NAME, ENABLED, TIMED FROM setup_instruments WHERE NAME LIKE 'wait/synch/mutex/%' AND ( enabled <> 'YES' OR timed <> 'YES' )"
	collecting := "Collecting setup_instruments wait/synch/mutex configuration settings"
	updating := "Updating setup_instruments configuration for: wait/synch/mutex"

	si.Configure(sql, collecting, updating)
	lib.Logger.Println("EnableMutexMonitoring finishes")
}

// return true if the error is not in the expected list
func error_in_expected_list(actual_error string, expected_errors []string) bool {
	lib.Logger.Println("checking if", actual_error, "is in", expected_errors)
	e := actual_error[0:11]
	expected_error := false
	for i := range expected_errors {
		if e == expected_errors[i] {
			lib.Logger.Println("found an expected error", expected_errors[i])
			expected_error = true
			break
		}
	}
	lib.Logger.Println("returning", expected_error)
	return expected_error
}

// generic routine (now) to update some rows in setup instruments
func (si *SetupInstruments) Configure(select_sql string, collecting, updating string) {
	lib.Logger.Println(fmt.Sprintf("Configure(%q,%q,%q)", select_sql, collecting, updating))
	// skip if we've tried and failed
	if si.update_tried && !si.update_succeeded {
		lib.Logger.Println("Configure() - Skipping further configuration")
		return
	}

	// setup the old values in case they're not set
	if si.rows == nil {
		si.rows = make([]table_row, 0, 500)
	}

	lib.Logger.Println(collecting)

	lib.Logger.Println("dbh.query", select_sql)
	rows, err := si.dbh.Query(select_sql)
	if err != nil {
		log.Fatal(err)
	}

	count := 0
	for rows.Next() {
		var r table_row
		if err := rows.Scan(
			&r.NAME,
			&r.ENABLED,
			&r.TIMED); err != nil {
			log.Fatal(err)
		}
		si.rows = append(si.rows, r)
		count++
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
	rows.Close()
	lib.Logger.Println("- found", count, "rows whose configuration need changing")

	// update the rows which need to be set - do multiple updates but I don't care
	lib.Logger.Println(updating)

	const update_sql = "UPDATE setup_instruments SET enabled = ?, TIMED = ? WHERE NAME = ?"
	lib.Logger.Println("Preparing statement:", update_sql)
	si.update_tried = true
	lib.Logger.Println("dbh.Prepare", update_sql)
	stmt, err := si.dbh.Prepare(update_sql)
	if err != nil {
		lib.Logger.Println("- prepare gave error:", err.Error())
		if !error_in_expected_list(err.Error(), EXPECTED_UPDATE_ERRORS) {
			log.Fatal("Not expected error so giving up")
		} else {
			lib.Logger.Println("- expected error so not running statement")
		}
	} else {
		lib.Logger.Println("Prepare succeeded, trying to update", len(si.rows), "row(s)")
		count = 0
		for i := range si.rows {
			lib.Logger.Println("- changing row:", si.rows[i].NAME)
			lib.Logger.Println("stmt.Exec", "YES", "YES", si.rows[i].NAME)
			if res, err := stmt.Exec("YES", "YES", si.rows[i].NAME); err == nil {
				lib.Logger.Println("update succeeded")
				si.update_succeeded = true
				c, _ := res.RowsAffected()
				count += int(c)
			} else {
				si.update_succeeded = false
				if error_in_expected_list(err.Error(), EXPECTED_UPDATE_ERRORS) {
					lib.Logger.Println("Insufficient privileges to UPDATE setup_instruments: " + err.Error())
					lib.Logger.Println("Not attempting further updates")
					return
				} else {
					log.Fatal(err)
				}
			}
		}
		if si.update_succeeded {
			lib.Logger.Println(count, "rows changed in p_s.setup_instruments")
		}
		stmt.Close()
	}
	lib.Logger.Println("Configure() returns update_tried", si.update_tried, ", update_succeeded", si.update_succeeded)
}

// restore setup_instruments rows to their previous settings
func (si *SetupInstruments) RestoreConfiguration() {
	lib.Logger.Println("RestoreConfiguration()")
	// If the previous update didn't work then don't try to restore
	if !si.update_succeeded {
		lib.Logger.Println("Not restoring p_s.setup_instruments to original settings as initial configuration attempt failed")
		return
	} else {
		lib.Logger.Println("Restoring p_s.setup_instruments to its original settings")
	}

	// update the rows which need to be set - do multiple updates but I don't care
	update_sql := "UPDATE setup_instruments SET enabled = ?, TIMED = ? WHERE NAME = ?"
	lib.Logger.Println("dbh.Prepare(", update_sql, ")")
	stmt, err := si.dbh.Prepare(update_sql)
	if err != nil {
		log.Fatal(err)
	}
	count := 0
	for i := range si.rows {
		lib.Logger.Println("stmt.Exec(", si.rows[i].ENABLED, si.rows[i].TIMED, si.rows[i].NAME, ")")
		if _, err := stmt.Exec(si.rows[i].ENABLED, si.rows[i].TIMED, si.rows[i].NAME); err != nil {
			log.Fatal(err)
		}
		count++
	}
	lib.Logger.Println("stmt.Close()")
	stmt.Close()
	lib.Logger.Println(count, "rows changed in p_s.setup_instruments")
}
<commit_msg>setup_instruments: use query with parameter<commit_after>// Manage the configuration of setup_instruments.
package setup_instruments

import (
	"database/sql"
	"fmt"
	"log"

	"github.com/sjmudd/pstop/lib"
)

// constants
const sql_select = "SELECT NAME, ENABLED, TIMED FROM setup_instruments WHERE NAME LIKE ? AND 'YES NOT IN (enabled,timed)"

// We only match on the error number
// Error 1142: UPDATE command denied to user 'cacti'@'10.164.132.182' for table 'setup_instruments'
// Error 1290: The MySQL server is running with the --read-only option so it cannot execute this statement
var EXPECTED_UPDATE_ERRORS = []string{
	"Error 1142:",
	"Error 1290:",
}

// one row of performance_schema.setup_instruments
type table_row struct {
	NAME    string
	ENABLED string
	TIMED   string
}

type table_rows []table_row

// SetupInstruments "object"
type SetupInstruments struct {
	update_tried     bool
	update_succeeded bool
	rows             table_rows
	dbh              *sql.DB
}

// Return a newly initialised SetupInstruments structure with a handle to the database.
// Better to return a pointer ?
func NewSetupInstruments(dbh *sql.DB) SetupInstruments {
	return SetupInstruments{dbh: dbh}
}

// enable mutex and stage monitoring
func (si *SetupInstruments) EnableMonitoring() {
	si.EnableMutexMonitoring()
	si.EnableStageMonitoring()
}

// Change settings to monitor stage/sql/%
func (si *SetupInstruments) EnableStageMonitoring() {
	lib.Logger.Println("EnableStageMonitoring")
	sql_match := "stage/sql/%"
	collecting := "Collecting setup_instruments stage/sql configuration settings"
	updating := "Updating setup_instruments configuration for: stage/sql"

	si.Configure(sql_match, collecting, updating)
	lib.Logger.Println("EnableStageMonitoring finishes")
}

// Change settings to monitor wait/synch/mutex/%
func (si *SetupInstruments) EnableMutexMonitoring() {
	lib.Logger.Println("EnableMutexMonitoring")
	sql_match := "wait/synch/mutex/%"
	collecting := "Collecting setup_instruments wait/synch/mutex configuration settings"
	updating := "Updating setup_instruments configuration for: wait/synch/mutex"

	si.Configure(sql_match, collecting, updating)
	lib.Logger.Println("EnableMutexMonitoring finishes")
}

// return true if the error is not in the expected list
func error_in_expected_list(actual_error string, expected_errors []string) bool {
	lib.Logger.Println("checking if", actual_error, "is in", expected_errors)
	e := actual_error[0:11]
	expected_error := false
	for i := range expected_errors {
		if e == expected_errors[i] {
			lib.Logger.Println("found an expected error", expected_errors[i])
			expected_error = true
			break
		}
	}
	lib.Logger.Println("returning", expected_error)
	return expected_error
}

// generic routine (now) to update some rows in setup instruments
func (si *SetupInstruments) Configure(sql_match string, collecting, updating string) {
	lib.Logger.Println(fmt.Sprintf("Configure(%q,%q,%q)", sql_match, collecting, updating))
	// skip if we've tried and failed
	if si.update_tried && !si.update_succeeded {
		lib.Logger.Println("Configure() - Skipping further configuration")
		return
	}

	// setup the old values in case they're not set
	if si.rows == nil {
		si.rows = make([]table_row, 0, 500)
	}

	lib.Logger.Println(collecting)

	lib.Logger.Println("dbh.query", sql_select, sql_match)
	rows, err := si.dbh.Query(sql_select, sql_match)
	if err != nil {
		log.Fatal(err)
	}

	count := 0
	for rows.Next() {
		var r table_row
		if err := rows.Scan(
			&r.NAME,
			&r.ENABLED,
			&r.TIMED); err != nil {
			log.Fatal(err)
		}
		si.rows = append(si.rows, r)
		count++
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
	rows.Close()
	lib.Logger.Println("- found", count, "rows whose configuration need changing")

	// update the rows which need to be set - do multiple updates but I don't care
	lib.Logger.Println(updating)

	const update_sql = "UPDATE setup_instruments SET enabled = ?, TIMED = ? WHERE NAME = ?"
	lib.Logger.Println("Preparing statement:", update_sql)
	si.update_tried = true
	lib.Logger.Println("dbh.Prepare", update_sql)
	stmt, err := si.dbh.Prepare(update_sql)
	if err != nil {
		lib.Logger.Println("- prepare gave error:", err.Error())
		if !error_in_expected_list(err.Error(), EXPECTED_UPDATE_ERRORS) {
			log.Fatal("Not expected error so giving up")
		} else {
			lib.Logger.Println("- expected error so not running statement")
		}
	} else {
		lib.Logger.Println("Prepare succeeded, trying to update", len(si.rows), "row(s)")
		count = 0
		for i := range si.rows {
			lib.Logger.Println("- changing row:", si.rows[i].NAME)
			lib.Logger.Println("stmt.Exec", "YES", "YES", si.rows[i].NAME)
			if res, err := stmt.Exec("YES", "YES", si.rows[i].NAME); err == nil {
				lib.Logger.Println("update succeeded")
				si.update_succeeded = true
				c, _ := res.RowsAffected()
				count += int(c)
			} else {
				si.update_succeeded = false
				if error_in_expected_list(err.Error(), EXPECTED_UPDATE_ERRORS) {
					lib.Logger.Println("Insufficient privileges to UPDATE setup_instruments: " + err.Error())
					lib.Logger.Println("Not attempting further updates")
					return
				} else {
					log.Fatal(err)
				}
			}
		}
		if si.update_succeeded {
			lib.Logger.Println(count, "rows changed in p_s.setup_instruments")
		}
		stmt.Close()
	}
	lib.Logger.Println("Configure() returns update_tried", si.update_tried, ", update_succeeded", si.update_succeeded)
}

// restore setup_instruments rows to their previous settings
func (si *SetupInstruments) RestoreConfiguration() {
	lib.Logger.Println("RestoreConfiguration()")
	// If the previous update didn't work then don't try to restore
	if !si.update_succeeded {
		lib.Logger.Println("Not restoring p_s.setup_instruments to original settings as initial configuration attempt failed")
		return
	} else {
		lib.Logger.Println("Restoring p_s.setup_instruments to its original settings")
	}

	// update the rows which need to be set - do multiple updates but I don't care
	update_sql := "UPDATE setup_instruments SET enabled = ?, TIMED = ? WHERE NAME = ?"
	lib.Logger.Println("dbh.Prepare(", update_sql, ")")
	stmt, err := si.dbh.Prepare(update_sql)
	if err != nil {
		log.Fatal(err)
	}
	count := 0
	for i := range si.rows {
		lib.Logger.Println("stmt.Exec(", si.rows[i].ENABLED, si.rows[i].TIMED, si.rows[i].NAME, ")")
		if _, err := stmt.Exec(si.rows[i].ENABLED, si.rows[i].TIMED, si.rows[i].NAME); err != nil {
			log.Fatal(err)
		}
		count++
	}
	lib.Logger.Println("stmt.Close()")
	stmt.Close()
	lib.Logger.Println(count, "rows changed in p_s.setup_instruments")
}
<|endoftext|>


<commit_before>package model

import (
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/rpc"
	"strconv"
	"time"
)

// the grid scheduler
type GridSdr struct {
	id            int
	addr          string
	basePort      int
	others        []string // other GridSdr's
	clusters      []string
	leader        string // the lead GridSdr
	jobs          []Job
	tasks         chan Task // these tasks require CS
	inElection    SyncedVal
	mutexRespChan chan int
	mutexReqChan  chan Task
	mutexState    SyncedVal
	clock         SyncedVal
	reqClock      int64
}

type GridSdrArgs struct {
	Id    int
	Addr  string
	Type  MsgType
	Clock int64
}

func InitGridSdr(id int, n int, basePort int, prefix string) GridSdr {
	addr := prefix + strconv.Itoa(basePort+id)
	// TODO read from config file or have bootstrap/discovery server
	var others []string
	for i := 0; i < n; i++ {
		if i != id {
			others = append(others, prefix+strconv.Itoa(basePort+i))
		}
	}
	// TODO see above
	var clusters []string
	leader := ""
	return GridSdr{id, addr, basePort, others, clusters, leader,
		make([]Job, 0),
		make(chan Task, 100),
		SyncedVal{val: false},
		make(chan int, n-1),
		make(chan Task, 100),
		SyncedVal{val: StateReleased},
		SyncedVal{val: 0},
		0,
	}
}

// TODO how should the user submit request
// via REST API or RPC call from a client?

func (gs *GridSdr) Run(genJobs bool) {
	rand.Seed(time.Now().UTC().UnixNano())
	go gs.runRPC()
	go gs.pollLeader()
	go gs.runTasks()

	for {
		// TODO get all the clusters
		// TODO arrange them in loaded order
		// TODO allocate *all* jobs

		if genJobs {
			reply := 0
			gs.AddJob(nil, &reply)
			time.Sleep(time.Second * 1)
		}
	}
}

func addSendJobToRM(addr string, args ResManArgs) (int, error) {
	log.Printf("Sending job to %v\n", addr)
	remote, e := rpc.DialHTTP("tcp", addr)
	if e != nil {
		log.Printf("Node %v not online (DialHTTP)\n", addr)
		return -1, e
	}
	reply := -1
	err := remote.Call("ResMan.AddJob", args, &reply)
	if err != nil {
		log.Printf("Node %v not online (ResMan.AddJob)\n", addr)
	}
	return reply, remote.Close()
}

func sendMsgToGS(addr string, args GridSdrArgs) (int, error) {
	log.Printf("Sending message to %v\n", addr)
	remote, e := rpc.DialHTTP("tcp", addr)
	if e != nil {
		log.Printf("Node %v not online (DialHTTP)\n", addr)
		return -1, e
	}
	reply := -1
	if e := remote.Call("GridSdr.RecvMsg", args, &reply); e != nil {
		log.Printf("Node %v not online (RecvMsg)\n", addr)
	}
	return reply, remote.Close()
}

// send the critical section request and then wait for responses until some timeout
// don't wait for response for nodes that are already offline
// NOTE: this function isn't designed to be thread safe, it is run periodically in `runTasks`
func (gs *GridSdr) obtainCritSection() {
	if gs.mutexState.get().(MutexState) != StateReleased {
		log.Panicf("Should not be in CS, state: %v\n", gs)
	}

	// empty the channel before starting just in case
	for len(gs.mutexRespChan) > 0 {
		<-gs.mutexRespChan
	}

	gs.clock.tick()
	successes := 0
	for _, o := range gs.others {
		_, e := sendMsgToGS(o, GridSdrArgs{gs.id, gs.addr, MutexReq, gs.clock.geti64()})
		if e == nil {
			successes++
		}
	}
	gs.reqClock = gs.clock.geti64()

	// wait until others has written to mutexRespChan or time out (2s)
	t := time.Now().Add(2 * time.Second)
	for t.After(time.Now()) {
		if len(gs.mutexRespChan) >= successes {
			break
		}
		time.Sleep(time.Microsecond)
	}

	// empty the channel
	// NOTE: nodes following the protocol shouldn't send more messages
	for len(gs.mutexRespChan) > 0 {
		<-gs.mutexRespChan
	}

	// here we're in critical section
	gs.mutexState.set(StateHeld)
}

func (gs *GridSdr) releaseCritSection() {
	gs.mutexState.set(StateReleased)
	for len(gs.mutexReqChan) > 0 {
		resp := <-gs.mutexReqChan
		_, e := resp()
		if e != nil {
			log.Panic("task failed with", e)
		}
	}
}

// send messages to procs with higher id
func (gs *GridSdr) elect(withDelay bool) {
	defer func() {
		gs.inElection.set(false)
	}()
	gs.inElection.set(true)

	gs.clock.tick()
	oks := 0
	for _, o := range gs.others {
		if idFromAddr(o, gs.basePort) < gs.id {
			continue // do nothing to lower ids
		}
		_, e := sendMsgToGS(o, GridSdrArgs{gs.id, gs.addr, ElectionMsg, gs.clock.geti64()})
		if e != nil {
			continue
		}
		oks++
	}

	// if no responses, then set the node itself as leader, and tell others
	gs.clock.tick()
	gs.leader = gs.addr
	log.Printf("I'm the leader (%v).\n", gs.leader)
	if oks == 0 {
		for i := range gs.others {
			args := GridSdrArgs{gs.id, gs.addr, CoordinateMsg, gs.clock.geti64()}
			_, e := sendMsgToGS(gs.others[i], args)
			if e != nil {
				// ok to fail the send, because nodes might be done
				continue
			}
		}
	}

	// artificially make the election last longer so that multiple messages
	// requests won't initialise multiple election runs
	if withDelay {
		time.Sleep(time.Second)
	}
}

func (gs *GridSdr) RecvMsg(args *GridSdrArgs, reply *int) error {
	log.Printf("Msg received %v\n", *args)
	*reply = 1
	gs.clock.set(max64(gs.clock.geti64(), args.Clock) + 1) // update Lamport clock
	if args.Type == CoordinateMsg {
		gs.leader = args.Addr
		log.Printf("Leader set to %v\n", gs.leader)
	} else if args.Type == ElectionMsg {
		// don't start a new election if one is already running
		if !gs.inElection.get().(bool) {
			go gs.elect(true)
		}
	} else if args.Type == MutexReq {
		go gs.respCritSection(*args)
	} else if args.Type == MutexResp {
		gs.mutexRespChan <- 0
	} else {
		log.Panic("Invalid message!", args)
	}
	return nil
}

func (gs *GridSdr) AddJob(args *GridSdrArgs, reply *int) error {
	log.Println("Dummy job added to tasks", gs.id)
	gs.tasks <- func() (interface{}, error) {
		// TODO add proper job
		log.Println("Finished Dummy job", gs.id)
		return 0, nil
	}
	return nil
}

func (gs *GridSdr) runTasks() {
	for {
		// acquire CS, run the tasks, run for 1ms at most, then release CS
		if len(gs.tasks) > 0 {
			gs.obtainCritSection()
			log.Println("In CS!", gs.id)
			t := time.Now().Add(time.Millisecond)
			for t.After(time.Now()) && len(gs.tasks) > 0 {
				task := <-gs.tasks
				_, e := task()
				if e != nil {
					log.Panic("task failed with", e)
				}
			}
			gs.releaseCritSection()
			log.Println("Out CS!", gs.id)
		}
		// sleep between 1ms to 500ms
		time.Sleep(time.Duration(time.Millisecond))
	}
}

func (gs *GridSdr) argsIsLater(args GridSdrArgs) bool {
	return gs.clock.geti64() < args.Clock || (gs.clock.geti64() == args.Clock && gs.id < args.Id)
}

func (gs *GridSdr) respCritSection(args GridSdrArgs) {
	resp := func() (interface{}, error) {
		sendMsgToGS(args.Addr, GridSdrArgs{gs.id, gs.addr, MutexResp, gs.reqClock})
		return 0, nil
	}

	st := gs.mutexState.get().(MutexState)
	if st == StateHeld || (st == StateWanted && gs.argsIsLater(args)) {
		gs.mutexReqChan <- resp
	} else {
		resp()
	}
}

func (gs *GridSdr) pollLeader() {
	for {
		time.Sleep(time.Second)
		if gs.addr == gs.leader {
			continue // don't do anything if I'm leader
		}
		remote, e := rpc.DialHTTP("tcp", gs.leader) // TODO should we have a mutex on `gs.leader?`
		if e != nil {
			log.Printf("Leader %v not online (DialHTTP), initialising election.\n", gs.leader)
			gs.elect(false)
		} else {
			remote.Close()
		}
	}
}

func (gs *GridSdr) runRPC() {
	log.Printf("Initialising RPC on addr %v\n", gs.addr)
	rpc.Register(gs)
	rpc.HandleHTTP()
	l, e := net.Listen("tcp", gs.addr)
	if e != nil {
		log.Panic("runRPC failed", e)
	}
	// the Serve function runs until death
	http.Serve(l, nil)
}
<commit_msg>Added documentation<commit_after>package model

import (
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/rpc"
	"strconv"
	"time"
)

type GridSdr struct {
	id            int
	addr          string
	basePort      int
	others        []string // other GridSdr's
	clusters      []string
	leader        string // the lead GridSdr
	jobs          []Job
	tasks         chan Task // these tasks require CS
	inElection    SyncedVal
	mutexRespChan chan int
	mutexReqChan  chan Task
	mutexState    SyncedVal
	clock         SyncedVal
	reqClock      int64
}

type GridSdrArgs struct {
	Id    int
	Addr  string
	Type  MsgType
	Clock int64
}

// InitGridSdr creates a grid scheduler.
func InitGridSdr(id int, n int, basePort int, prefix string) GridSdr {
	addr := prefix + strconv.Itoa(basePort+id)
	// TODO read from config file or have bootstrap/discovery server
	var others []string
	for i := 0; i < n; i++ {
		if i != id {
			others = append(others, prefix+strconv.Itoa(basePort+i))
		}
	}
	// TODO see above
	var clusters []string
	leader := ""
	return GridSdr{id, addr, basePort, others, clusters, leader,
		make([]Job, 0),
		make(chan Task, 100),
		SyncedVal{val: false},
		make(chan int, n-1),
		make(chan Task, 100),
		SyncedVal{val: StateReleased},
		SyncedVal{val: int64(0)},
		0,
	}
}

// Run is the main function for GridSdr, it starts all the services.
func (gs *GridSdr) Run(genJobs bool) {
	rand.Seed(time.Now().UTC().UnixNano())
	go gs.runRPC()
	go gs.pollLeader()
	go gs.runTasks()

	for {
		// TODO get all the clusters
		// TODO arrange them in loaded order
		// TODO allocate *all* jobs

		if genJobs {
			reply := 0
			gs.AddJob(nil, &reply)
			time.Sleep(time.Second * 1)
		}
	}
}

// addSendJobToRM creates an RPC connection with a ResMan and does one remote call on AddJob.
func addSendJobToRM(addr string, args ResManArgs) (int, error) {
	log.Printf("Sending job to %v\n", addr)
	remote, e := rpc.DialHTTP("tcp", addr)
	if e != nil {
		log.Printf("Node %v not online (DialHTTP)\n", addr)
		return -1, e
	}
	reply := -1
	err := remote.Call("ResMan.AddJob", args, &reply)
	if err != nil {
		log.Printf("Node %v not online (ResMan.AddJob)\n", addr)
	}
	return reply, remote.Close()
}

// sendMsgToGS creates an RPC connection with another GridSdr and does one remote call on RecvMsg.
func sendMsgToGS(addr string, args GridSdrArgs) (int, error) {
	log.Printf("Sending message to %v\n", addr)
	remote, e := rpc.DialHTTP("tcp", addr)
	if e != nil {
		log.Printf("Node %v not online (DialHTTP)\n", addr)
		return -1, e
	}
	reply := -1
	if e := remote.Call("GridSdr.RecvMsg", args, &reply); e != nil {
		log.Printf("Node %v not online (RecvMsg)\n", addr)
	}
	return reply, remote.Close()
}

// obtainCritSection implements most of the Ricart-Agrawala algorithm, it sends the critical section request and then wait for responses until some timeout.
// Initially we set the mutexState to StateWanted, if the critical section is obtained we set it to StateHeld.
// NOTE: this function isn't designed to be thread safe, it is run periodically in `runTasks`.
func (gs *GridSdr) obtainCritSection() {
	if gs.mutexState.get().(MutexState) != StateReleased {
		log.Panicf("Should not be in CS, state: %v\n", gs)
	}

	gs.mutexState.set(StateWanted)

	// empty the channel before starting just in case
	for len(gs.mutexRespChan) > 0 {
		<-gs.mutexRespChan
	}

	gs.clock.tick()
	successes := 0
	for _, o := range gs.others {
		_, e := sendMsgToGS(o, GridSdrArgs{gs.id, gs.addr, MutexReq, gs.clock.geti64()})
		if e == nil {
			successes++
		}
	}
	gs.reqClock = gs.clock.geti64()

	// wait until others has written to mutexRespChan or time out (2s)
	t := time.Now().Add(2 * time.Second)
	for t.After(time.Now()) {
		if len(gs.mutexRespChan) >= successes {
			break
		}
		time.Sleep(time.Microsecond)
	}

	// empty the channel
	// NOTE: nodes following the protocol shouldn't send more messages
	for len(gs.mutexRespChan) > 0 {
		<-gs.mutexRespChan
	}

	// here we're in critical section
	gs.mutexState.set(StateHeld)
}

// releaseCritSection sets the mutexState to StateReleased and then runs all the queued requests.
func (gs *GridSdr) releaseCritSection() {
	gs.mutexState.set(StateReleased)
	for len(gs.mutexReqChan) > 0 {
		resp := <-gs.mutexReqChan
		_, e := resp()
		if e != nil {
			log.Panic("task failed with", e)
		}
	}
}

// elect implements the Bully algorithm.
func (gs *GridSdr) elect() {
	defer func() {
		gs.inElection.set(false)
	}()
	gs.inElection.set(true)

	gs.clock.tick()
	oks := 0
	for _, o := range gs.others {
		if idFromAddr(o, gs.basePort) < gs.id {
			continue // do nothing to lower ids
		}
		_, e := sendMsgToGS(o, GridSdrArgs{gs.id, gs.addr, ElectionMsg, gs.clock.geti64()})
		if e != nil {
			continue
		}
		oks++
	}

	// if no responses, then set the node itself as leader, and tell others
	gs.clock.tick()
	gs.leader = gs.addr
	log.Printf("I'm the leader (%v).\n", gs.leader)
	if oks == 0 {
		for i := range gs.others {
			args := GridSdrArgs{gs.id, gs.addr, CoordinateMsg, gs.clock.geti64()}
			_, e := sendMsgToGS(gs.others[i], args)
			if e != nil {
				// ok to fail the send, because nodes might be done
				continue
			}
		}
	}

	// artificially make the election last longer so that multiple messages
	// requests won't initialise multiple election runs
	time.Sleep(time.Second)
}

// RecvMsg is called remotely, it updates the Lamport clock first and then performs tasks depending on the message type.
func (gs *GridSdr) RecvMsg(args *GridSdrArgs, reply *int) error {
	log.Printf("Msg received %v\n", *args)
	*reply = 1
	gs.clock.set(max64(gs.clock.geti64(), args.Clock) + 1) // update Lamport clock
	if args.Type == CoordinateMsg {
		gs.leader = args.Addr
		log.Printf("Leader set to %v\n", gs.leader)
	} else if args.Type == ElectionMsg {
		// don't start a new election if one is already running
		if !gs.inElection.get().(bool) {
			go gs.elect()
		}
	} else if args.Type == MutexReq {
		go gs.respCritSection(*args)
	} else if args.Type == MutexResp {
		gs.mutexRespChan <- 0
	} else {
		log.Panic("Invalid message!", args)
	}
	return nil
}

// AddJob is called by the client to add a job to the tasks queue.
// NOTE: this does not add the job to the job queue, that is done by `runTasks`.
func (gs *GridSdr) AddJob(args *GridSdrArgs, reply *int) error {
	log.Println("Dummy job added to tasks", gs.id)
	gs.tasks <- func() (interface{}, error) {
		// TODO add proper job
		log.Println("Finished Dummy job", gs.id)
		return 0, nil
	}
	return nil
}

// runTasks queries the tasks queue and if there are outstanding tasks it will request for critical and run the tasks.
func (gs *GridSdr) runTasks() {
	for {
		// acquire CS, run the tasks, run for 1ms at most, then release CS
		if len(gs.tasks) > 0 {
			gs.obtainCritSection()
			log.Println("In CS!", gs.id)
			t := time.Now().Add(time.Millisecond)
			for t.After(time.Now()) && len(gs.tasks) > 0 {
				task := <-gs.tasks
				_, e := task()
				if e != nil {
					log.Panic("task failed with", e)
				}
			}
			gs.releaseCritSection()
			log.Println("Out CS!", gs.id)
		}
		// sleep between 1ms to 500ms
		time.Sleep(time.Duration(time.Millisecond))
	}
}

// argsIsLater checks whether args has a later Lamport clock, tie break using node ID.
func (gs *GridSdr) argsIsLater(args GridSdrArgs) bool {
	return gs.clock.geti64() < args.Clock || (gs.clock.geti64() == args.Clock && gs.id < args.Id)
}

// respCritSection puts the critical section response into the response queue when it can't respond straight away.
func (gs *GridSdr) respCritSection(args GridSdrArgs) {
	resp := func() (interface{}, error) {
		sendMsgToGS(args.Addr, GridSdrArgs{gs.id, gs.addr, MutexResp, gs.reqClock})
		return 0, nil
	}

	st := gs.mutexState.get().(MutexState)
	if st == StateHeld || (st == StateWanted && gs.argsIsLater(args)) {
		gs.mutexReqChan <- resp
	} else {
		resp()
	}
}

// pollLeader polls the leader node and initiates the election algorithm is the leader goes offline.
func (gs *GridSdr) pollLeader() {
	for {
		time.Sleep(time.Second)

		// don't do anything if election is running or I'm leader
		if gs.inElection.get().(bool) || gs.addr == gs.leader {
			continue
		}

		remote, e := rpc.DialHTTP("tcp", gs.leader)
		if e != nil {
			log.Printf("Leader %v not online (DialHTTP), initialising election.\n", gs.leader)
			gs.elect()
		} else {
			remote.Close()
		}
	}
}

// runRPC registers and runs the RPC server.
func (gs *GridSdr) runRPC() {
	log.Printf("Initialising RPC on addr %v\n", gs.addr)
	rpc.Register(gs)
	rpc.HandleHTTP()
	l, e := net.Listen("tcp", gs.addr)
	if e != nil {
		log.Panic("runRPC failed", e)
	}
	// the Serve function runs until death
	http.Serve(l, nil)
}
<|endoftext|>


<commit_before>package driver

import (
	"bytes"
	"fmt"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver/executor"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/client/getter"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/mitchellh/mapstructure"
)

// JavaDriver is a simple driver to execute applications packaged in Jars.
// It literally just fork/execs tasks with the java command.
type JavaDriver struct {
	DriverContext
	fingerprint.StaticFingerprinter
}

type JavaDriverConfig struct {
	JvmOpts        string `mapstructure:"jvm_options"`
	ArtifactSource string `mapstructure:"artifact_source`
	Checksum       string `mapstructure:"checksum"`
	Args           string `mapstructure:"args"`
}

// javaHandle is returned from Start/Open as a handle to the PID
type javaHandle struct {
	cmd    executor.Executor
	waitCh chan error
	doneCh chan struct{}
}

// NewJavaDriver is used to create a new exec driver
func NewJavaDriver(ctx *DriverContext) Driver {
	return &JavaDriver{DriverContext: *ctx}
}

func (d *JavaDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
	// Only enable if we are root when running on non-windows systems.
	if runtime.GOOS == "linux" && syscall.Geteuid() != 0 {
		d.logger.Printf("[DEBUG] driver.java: must run as root user on linux, disabling")
		return false, nil
	}

	// Find java version
	var out bytes.Buffer
	var erOut bytes.Buffer
	cmd := exec.Command("java", "-version")
	cmd.Stdout = &out
	cmd.Stderr = &erOut
	err := cmd.Run()
	if err != nil {
		// assume Java wasn't found
		return false, nil
	}

	// 'java -version' returns output on Stderr typically.
	// Check stdout, but it's probably empty
	var infoString string
	if out.String() != "" {
		infoString = out.String()
	}

	if erOut.String() != "" {
		infoString = erOut.String()
	}

	if infoString == "" {
		d.logger.Println("[WARN] driver.java: error parsing Java version information, aborting")
		return false, nil
	}

	// Assume 'java -version' returns 3 lines:
	//    java version "1.6.0_36"
	//    OpenJDK Runtime Environment (IcedTea6 1.13.8) (6b36-1.13.8-0ubuntu1~12.04)
	//    OpenJDK 64-Bit Server VM (build 23.25-b01, mixed mode)
	// Each line is terminated by \n
	info := strings.Split(infoString, "\n")
	versionString := info[0]
	versionString = strings.TrimPrefix(versionString, "java version ")
	versionString = strings.Trim(versionString, "\"")
	node.Attributes["driver.java"] = "1"
	node.Attributes["driver.java.version"] = versionString
	node.Attributes["driver.java.runtime"] = info[1]
	node.Attributes["driver.java.vm"] = info[2]

	return true, nil
}

func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
	var driverConfig JavaDriverConfig
	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
		return nil, err
	}
	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
	if !ok {
		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
	}

	// Proceed to download an artifact to be executed.
	path, err := getter.GetArtifact(
		filepath.Join(taskDir, allocdir.TaskLocal),
		driverConfig.ArtifactSource,
		driverConfig.Checksum,
		d.logger,
	)
	if err != nil {
		return nil, err
	}

	jarName := filepath.Base(path)

	// Get the environment variables.
	envVars := TaskEnvironmentVariables(ctx, task)

	args := []string{}
	// Look for jvm options
	jvm_options := driverConfig.JvmOpts
	if jvm_options != "" {
		d.logger.Printf("[DEBUG] driver.java: found JVM options: %s", jvm_options)
		args = append(args, jvm_options)
	}

	// Build the argument list.
	args = append(args, "-jar", filepath.Join(allocdir.TaskLocal, jarName))
	if argRaw := driverConfig.Args; argRaw != "" {
		args = append(args, argRaw)
	}

	// Setup the command
	// Assumes Java is in the $PATH, but could probably be detected
	cmd := executor.Command("java", args...)

	// Populate environment variables
	cmd.Command().Env = envVars.List()

	if err := cmd.Limit(task.Resources); err != nil {
		return nil, fmt.Errorf("failed to constrain resources: %s", err)
	}

	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
		return nil, fmt.Errorf("failed to configure task directory: %v", err)
	}

	if err := cmd.Start(); err != nil {
		return nil, fmt.Errorf("failed to start source: %v", err)
	}

	// Return a driver handle
	h := &javaHandle{
		cmd:    cmd,
		doneCh: make(chan struct{}),
		waitCh: make(chan error, 1),
	}

	go h.run()
	return h, nil
}

func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
	// Find the process
	cmd, err := executor.OpenId(handleID)
	if err != nil {
		return nil, fmt.Errorf("failed to open ID %v: %v", handleID, err)
	}

	// Return a driver handle
	h := &javaHandle{
		cmd:    cmd,
		doneCh: make(chan struct{}),
		waitCh: make(chan error, 1),
	}

	go h.run()
	return h, nil
}

func (h *javaHandle) ID() string {
	id, _ := h.cmd.ID()
	return id
}

func (h *javaHandle) WaitCh() chan error {
	return h.waitCh
}

func (h *javaHandle) Update(task *structs.Task) error {
	// Update is not possible
	return nil
}

func (h *javaHandle) Kill() error {
	h.cmd.Shutdown()
	select {
	case <-h.doneCh:
		return nil
	case <-time.After(5 * time.Second):
		return h.cmd.ForceStop()
	}
}

func (h *javaHandle) run() {
	err := h.cmd.Wait()
	close(h.doneCh)
	if err != nil {
		h.waitCh <- err
	}
	close(h.waitCh)
}
<commit_msg>Fixed the java driver config<commit_after>package driver

import (
	"bytes"
	"fmt"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver/executor"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/client/getter"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/mitchellh/mapstructure"
)

// JavaDriver is a simple driver to execute applications packaged in Jars.
// It literally just fork/execs tasks with the java command.
type JavaDriver struct {
	DriverContext
	fingerprint.StaticFingerprinter
}

type JavaDriverConfig struct {
	JvmOpts        string `mapstructure:"jvm_options"`
	ArtifactSource string `mapstructure:"artifact_source"`
	Checksum       string `mapstructure:"checksum"`
	Args           string `mapstructure:"args"`
}

// javaHandle is returned from Start/Open as a handle to the PID
type javaHandle struct {
	cmd    executor.Executor
	waitCh chan error
	doneCh chan struct{}
}

// NewJavaDriver is used to create a new exec driver
func NewJavaDriver(ctx *DriverContext) Driver {
	return &JavaDriver{DriverContext: *ctx}
}

func (d *JavaDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
	// Only enable if we are root when running on non-windows systems.
	if runtime.GOOS == "linux" && syscall.Geteuid() != 0 {
		d.logger.Printf("[DEBUG] driver.java: must run as root user on linux, disabling")
		return false, nil
	}

	// Find java version
	var out bytes.Buffer
	var erOut bytes.Buffer
	cmd := exec.Command("java", "-version")
	cmd.Stdout = &out
	cmd.Stderr = &erOut
	err := cmd.Run()
	if err != nil {
		// assume Java wasn't found
		return false, nil
	}

	// 'java -version' returns output on Stderr typically.
	// Check stdout, but it's probably empty
	var infoString string
	if out.String() != "" {
		infoString = out.String()
	}

	if erOut.String() != "" {
		infoString = erOut.String()
	}

	if infoString == "" {
		d.logger.Println("[WARN] driver.java: error parsing Java version information, aborting")
		return false, nil
	}

	// Assume 'java -version' returns 3 lines:
	//    java version "1.6.0_36"
	//    OpenJDK Runtime Environment (IcedTea6 1.13.8) (6b36-1.13.8-0ubuntu1~12.04)
	//    OpenJDK 64-Bit Server VM (build 23.25-b01, mixed mode)
	// Each line is terminated by \n
	info := strings.Split(infoString, "\n")
	versionString := info[0]
	versionString = strings.TrimPrefix(versionString, "java version ")
	versionString = strings.Trim(versionString, "\"")
	node.Attributes["driver.java"] = "1"
	node.Attributes["driver.java.version"] = versionString
	node.Attributes["driver.java.runtime"] = info[1]
	node.Attributes["driver.java.vm"] = info[2]

	return true, nil
}

func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
	var driverConfig JavaDriverConfig
	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
		return nil, err
	}
	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
	if !ok {
		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
	}

	// Proceed to download an artifact to be executed.
	path, err := getter.GetArtifact(
		filepath.Join(taskDir, allocdir.TaskLocal),
		driverConfig.ArtifactSource,
		driverConfig.Checksum,
		d.logger,
	)
	if err != nil {
		return nil, err
	}

	jarName := filepath.Base(path)

	// Get the environment variables.
	envVars := TaskEnvironmentVariables(ctx, task)

	args := []string{}
	// Look for jvm options
	jvm_options := driverConfig.JvmOpts
	if jvm_options != "" {
		d.logger.Printf("[DEBUG] driver.java: found JVM options: %s", jvm_options)
		args = append(args, jvm_options)
	}

	// Build the argument list.
	args = append(args, "-jar", filepath.Join(allocdir.TaskLocal, jarName))
	if argRaw := driverConfig.Args; argRaw != "" {
		args = append(args, argRaw)
	}

	// Setup the command
	// Assumes Java is in the $PATH, but could probably be detected
	cmd := executor.Command("java", args...)

	// Populate environment variables
	cmd.Command().Env = envVars.List()

	if err := cmd.Limit(task.Resources); err != nil {
		return nil, fmt.Errorf("failed to constrain resources: %s", err)
	}

	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
		return nil, fmt.Errorf("failed to configure task directory: %v", err)
	}

	if err := cmd.Start(); err != nil {
		return nil, fmt.Errorf("failed to start source: %v", err)
	}

	// Return a driver handle
	h := &javaHandle{
		cmd:    cmd,
		doneCh: make(chan struct{}),
		waitCh: make(chan error, 1),
	}

	go h.run()
	return h, nil
}

func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
	// Find the process
	cmd, err := executor.OpenId(handleID)
	if err != nil {
		return nil, fmt.Errorf("failed to open ID %v: %v", handleID, err)
	}

	// Return a driver handle
	h := &javaHandle{
		cmd:    cmd,
		doneCh: make(chan struct{}),
		waitCh: make(chan error, 1),
	}

	go h.run()
	return h, nil
}

func (h *javaHandle) ID() string {
	id, _ := h.cmd.ID()
	return id
}

func (h *javaHandle) WaitCh() chan error {
	return h.waitCh
}

func (h *javaHandle) Update(task *structs.Task) error {
	// Update is not possible
	return nil
}

func (h *javaHandle) Kill() error {
	h.cmd.Shutdown()
	select {
	case <-h.doneCh:
		return nil
	case <-time.After(5 * time.Second):
		return h.cmd.ForceStop()
	}
}

func (h *javaHandle) run() {
	err := h.cmd.Wait()
	close(h.doneCh)
	if err != nil {
		h.waitCh <- err
	}
	close(h.waitCh)
}
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package cmd

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
	"github.com/vespa-engine/vespa/client/go/vespa/xml"
)

func init() {
	rootCmd.AddCommand(prodCmd)
	prodCmd.AddCommand(prodInitCmd)
	prodCmd.AddCommand(prodSubmitCmd)
}

var prodCmd = &cobra.Command{
	Use:   "prod",
	Short: "Deploy an application package to production in Vespa Cloud",
	Long: `Deploy an application package to production in Vespa Cloud.

Configure and deploy your application package to production in Vespa Cloud.`,
	Example: `$ vespa prod init
$ vespa prod submit`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		// Root command does nothing
		cmd.Help()
		exitFunc(1)
	},
}

var prodInitCmd = &cobra.Command{
	Use:   "init",
	Short: "Modify service.xml and deployment.xml for production deployment",
	Long: `Modify service.xml and deployment.xml for production deployment.

Only basic deployment configuration is available through this command. For
advanced configuration see the relevant Vespa Cloud documentation and make
changes to deployment.xml and services.xml directly.

Reference:
https://cloud.vespa.ai/en/reference/services
https://cloud.vespa.ai/en/reference/deployment`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		appSource := applicationSource(args)
		pkg, err := vespa.FindApplicationPackage(appSource, false)
		if err != nil {
			fatalErr(err)
			return
		}
		if pkg.IsZip() {
			fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path),
				"Try running 'mvn clean' and run this command again")
			return
		}

		deploymentXML, err := readDeploymentXML(pkg)
		if err != nil {
			fatalErr(err, "Could not read deployment.xml")
			return
		}
		servicesXML, err := readServicesXML(pkg)
		if err != nil {
			fatalErr(err, "A services.xml declaring your cluster(s) must exist")
			return
		}

		fmt.Fprint(stdout, "This will modify any existing ", color.Yellow("deployment.xml"), " and ", color.Yellow("services.xml"),
			"!\nBefore modification a backup of the original file will be created.\n\n")
		fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n")
		fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n")
		r := bufio.NewReader(stdin)
		deploymentXML = updateRegions(r, deploymentXML)
		servicesXML = updateNodes(r, servicesXML)

		fmt.Fprintln(stdout)
		if err := writeWithBackup(pkg, "deployment.xml", deploymentXML.String()); err != nil {
			fatalErr(err)
			return
		}
		if err := writeWithBackup(pkg, "services.xml", servicesXML.String()); err != nil {
			fatalErr(err)
			return
		}
	},
}

var prodSubmitCmd = &cobra.Command{
	Use:   "submit",
	Short: "Submit your application for production deployment",
	Long: `Submit your application for production deployment.

This commands uploads your application package to Vespa Cloud and deploys it to
the production zones specified in deployment.xml.

Nodes are allocated to your application according to resources specified in
services.xml.

While submitting an application from a local development environment is
supported, it's strongly recommended that production deployments are performed
by a continuous build system.

For more information about production deployments in Vespa Cloud see:
https://cloud.vespa.ai/en/getting-to-production
https://cloud.vespa.ai/en/automated-deployments`,
	DisableAutoGenTag: true,
	Example: `$ mvn package # when adding custom Java components
$ vespa prod submit`,
	Run: func(cmd *cobra.Command, args []string) {
		target := getTarget()
		if target.Type() != "cloud" {
			fatalErr(fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type()))
			return
		}
		appSource := applicationSource(args)
		pkg, err := vespa.FindApplicationPackage(appSource, true)
		if err != nil {
			fatalErr(err)
			return
		}
		cfg, err := LoadConfig()
		if err != nil {
			fatalErr(err, "Could not load config")
			return
		}
		if !pkg.HasDeployment() {
			fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init")
			return
		}
		if pkg.TestPath == "" {
			fatalErrHint(fmt.Errorf("No tests found"),
				"The application must be a Java maven project, or include basic HTTP tests under src/test/application/",
				"See https://cloud.vespa.ai/en/getting-to-production")
			return
		}
		verifyTests(pkg.TestPath, target)
		isCI := os.Getenv("CI") != ""
		if !isCI {
			fmt.Fprintln(stderr, color.Yellow("Warning:"), "We recommend doing this only from a CD job")
			printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production")
		}
		opts := getDeploymentOpts(cfg, pkg, target)
		if err := vespa.Submit(opts); err != nil {
			fatalErr(err, "Could not submit application for deployment")
		} else {
			printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment")
			log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment",
				getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application)))
		}
	},
}

func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error {
	dst := filepath.Join(pkg.Path, filename)
	if util.PathExists(dst) {
		data, err := ioutil.ReadFile(dst)
		if err != nil {
			return err
		}
		if bytes.Equal(data, []byte(contents)) {
			fmt.Fprintf(stdout, "Not writing %s: File is unchanged\n", color.Yellow(filename))
			return nil
		}
		renamed := false
		for i := 1; i <= 1000; i++ {
			bak := fmt.Sprintf("%s.%d.bak", dst, i)
			if !util.PathExists(bak) {
				fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.Yellow(filename), color.Yellow(bak))
				if err := os.Rename(dst, bak); err != nil {
					return err
				}
				renamed = true
				break
			}
		}
		if !renamed {
			return fmt.Errorf("could not find an unused backup name for %s", dst)
		}
	}
	fmt.Fprintf(stdout, "Writing %s\n", color.Green(dst))
	return ioutil.WriteFile(dst, []byte(contents), 0644)
}

func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment {
	regions := promptRegions(r, deploymentXML)
	parts := strings.Split(regions, ",")
	regionElements := xml.Regions(parts...)
	if err := deploymentXML.Replace("prod", "region", regionElements); err != nil {
		fatalErr(err, "Could not update region elements in deployment.xml")
	}
	// TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we
	//       therefore remove <test> elements here.
	//       This can be improved by supporting <test> elements in xml package and allow specifying testing as part of
	//       region prompt, e.g. region1;test,region2
	if err := deploymentXML.Replace("prod", "test", nil); err != nil {
		fatalErr(err, "Could not remove test elements in deployment.xml")
	}
	return deploymentXML
}

func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string {
	fmt.Fprintln(stdout, color.Cyan("> Deployment regions"))
	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/zones"))
	fmt.Fprintf(stdout, "Example: %s\n\n", color.Yellow("aws-us-east-1c,aws-us-west-2a"))
	var currentRegions []string
	for _, r := range deploymentXML.Prod.Regions {
		currentRegions = append(currentRegions, r.Name)
	}
	if len(deploymentXML.Instance) > 0 {
		for _, r := range deploymentXML.Instance[0].Prod.Regions {
			currentRegions = append(currentRegions, r.Name)
		}
	}
	validator := func(input string) error {
		regions := strings.Split(input, ",")
		for _, r := range regions {
			if !xml.IsProdRegion(r, getSystem()) {
				return fmt.Errorf("invalid region %s", r)
			}
		}
		return nil
	}
	return prompt(r, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator)
}

func updateNodes(r *bufio.Reader, servicesXML xml.Services) xml.Services {
	for _, c := range servicesXML.Container {
		nodes := promptNodes(r, c.ID, c.Nodes)
		if err := servicesXML.Replace("container#"+c.ID, "nodes", nodes); err != nil {
			fatalErr(err)
			return xml.Services{}
		}
	}
	for _, c := range servicesXML.Content {
		nodes := promptNodes(r, c.ID, c.Nodes)
		if err := servicesXML.Replace("content#"+c.ID, "nodes", nodes); err != nil {
			fatalErr(err)
			return xml.Services{}
		}
	}
	return servicesXML
}

func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) xml.Nodes {
	count := promptNodeCount(r, clusterID, defaultValue.Count)
	const autoSpec = "auto"
	defaultSpec := autoSpec
	resources := defaultValue.Resources
	if resources != nil {
		defaultSpec = defaultValue.Resources.String()
	}
	spec := promptResources(r, clusterID, defaultSpec)
	if spec == autoSpec {
		resources = nil
	} else {
		r, err := xml.ParseResources(spec)
		if err != nil {
			fatalErr(err) // Should not happen as resources have already been validated
			return xml.Nodes{}
		}
		resources = &r
	}
	return xml.Nodes{Count: count, Resources: resources}
}

func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) string {
	fmt.Fprintln(stdout, color.Cyan("\n> Node count: "+clusterID+" cluster"))
	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("4"), color.Yellow("[2,8]"))
	validator := func(input string) error {
		_, _, err := xml.ParseNodeCount(input)
		return err
	}
	return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.Cyan(clusterID)), nodeCount, validator)
}

func promptResources(r *bufio.Reader, clusterID string, resources string) string {
	fmt.Fprintln(stdout, color.Cyan("\n> Node resources: "+clusterID+" cluster"))
	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("auto"), color.Yellow("vcpu=4,memory=8Gb,disk=100Gb"))
	validator := func(input string) error {
		if input == "auto" {
			return nil
		}
		_, err := xml.ParseResources(input)
		return err
	}
	return prompt(r, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.Cyan(clusterID)), resources, validator)
}

func readDeploymentXML(pkg vespa.ApplicationPackage) (xml.Deployment, error) {
	f, err := os.Open(filepath.Join(pkg.Path, "deployment.xml"))
	if errors.Is(err, os.ErrNotExist) {
		// Return a default value if there is no current deployment.xml
		return xml.DefaultDeployment, nil
	} else if err != nil {
		return xml.Deployment{}, err
	}
	defer f.Close()
	return xml.ReadDeployment(f)
}

func readServicesXML(pkg vespa.ApplicationPackage) (xml.Services, error) {
	f, err := os.Open(filepath.Join(pkg.Path, "services.xml"))
	if err != nil {
		return xml.Services{}, err
	}
	defer f.Close()
	return xml.ReadServices(f)
}

func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(input string) error) string {
	var input string
	for input == "" {
		fmt.Fprint(stdout, question)
		if defaultAnswer != "" {
			fmt.Fprint(stdout, " [", color.Yellow(defaultAnswer), "]")
		}
		fmt.Fprint(stdout, " ")

		var err error
		input, err = r.ReadString('\n')
		if err != nil {
			fatalErr(err)
			return ""
		}
		input = strings.TrimSpace(input)
		if input == "" {
			input = defaultAnswer
		}

		if err := validator(input); err != nil {
			printErr(err)
			fmt.Fprintln(stderr)
			input = ""
		}
	}
	return input
}

func verifyTests(testsParent string, target vespa.Target) {
	verifyTest(testsParent, "system-test", target, true)
	verifyTest(testsParent, "staging-setup", target, true)
	verifyTest(testsParent, "staging-test", target, true)
	verifyTest(testsParent, "production-test", target, false)
}

func verifyTest(testsParent string, suite string, target vespa.Target, required bool) {
	testDirectory := filepath.Join(testsParent, "tests", suite)
	_, err := os.Stat(testDirectory)
	if err != nil {
		if required {
			if errors.Is(err, os.ErrNotExist) {
				fatalErrHint(fmt.Errorf("No %s tests found", suite),
					fmt.Sprintf("No such directory: %s", testDirectory),
					"See https://cloud.vespa.ai/en/reference/testing")
			}
			fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
		}
		return
	}

	runTests(testDirectory, true)
}
<commit_msg>Link to sizing guide<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package cmd

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
	"github.com/vespa-engine/vespa/client/go/vespa/xml"
)

func init() {
	rootCmd.AddCommand(prodCmd)
	prodCmd.AddCommand(prodInitCmd)
	prodCmd.AddCommand(prodSubmitCmd)
}

var prodCmd = &cobra.Command{
	Use:   "prod",
	Short: "Deploy an application package to production in Vespa Cloud",
	Long: `Deploy an application package to production in Vespa Cloud.

Configure and deploy your application package to production in Vespa Cloud.`,
	Example: `$ vespa prod init
$ vespa prod submit`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		// Root command does nothing
		cmd.Help()
		exitFunc(1)
	},
}

var prodInitCmd = &cobra.Command{
	Use:   "init",
	Short: "Modify service.xml and deployment.xml for production deployment",
	Long: `Modify service.xml and deployment.xml for production deployment.

Only basic deployment configuration is available through this command. For
advanced configuration see the relevant Vespa Cloud documentation and make
changes to deployment.xml and services.xml directly.

Reference:
https://cloud.vespa.ai/en/reference/services
https://cloud.vespa.ai/en/reference/deployment`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		appSource := applicationSource(args)
		pkg, err := vespa.FindApplicationPackage(appSource, false)
		if err != nil {
			fatalErr(err)
			return
		}
		if pkg.IsZip() {
			fatalErrHint(fmt.Errorf("Cannot modify compressed application package %s", pkg.Path),
				"Try running 'mvn clean' and run this command again")
			return
		}

		deploymentXML, err := readDeploymentXML(pkg)
		if err != nil {
			fatalErr(err, "Could not read deployment.xml")
			return
		}
		servicesXML, err := readServicesXML(pkg)
		if err != nil {
			fatalErr(err, "A services.xml declaring your cluster(s) must exist")
			return
		}

		fmt.Fprint(stdout, "This will modify any existing ", color.Yellow("deployment.xml"), " and ", color.Yellow("services.xml"),
			"!\nBefore modification a backup of the original file will be created.\n\n")
		fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n")
		fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n")
		fmt.Fprint(stdout, "See this guide for sizing a Vespa deployment:\n", color.Green("https://docs.vespa.ai/en/performance/sizing-search.html\n\n"))
		r := bufio.NewReader(stdin)
		deploymentXML = updateRegions(r, deploymentXML)
		servicesXML = updateNodes(r, servicesXML)

		fmt.Fprintln(stdout)
		if err := writeWithBackup(pkg, "deployment.xml", deploymentXML.String()); err != nil {
			fatalErr(err)
			return
		}
		if err := writeWithBackup(pkg, "services.xml", servicesXML.String()); err != nil {
			fatalErr(err)
			return
		}
	},
}

var prodSubmitCmd = &cobra.Command{
	Use:   "submit",
	Short: "Submit your application for production deployment",
	Long: `Submit your application for production deployment.

This commands uploads your application package to Vespa Cloud and deploys it to
the production zones specified in deployment.xml.

Nodes are allocated to your application according to resources specified in
services.xml.

While submitting an application from a local development environment is
supported, it's strongly recommended that production deployments are performed
by a continuous build system.

For more information about production deployments in Vespa Cloud see:
https://cloud.vespa.ai/en/getting-to-production
https://cloud.vespa.ai/en/automated-deployments`,
	DisableAutoGenTag: true,
	Example: `$ mvn package # when adding custom Java components
$ vespa prod submit`,
	Run: func(cmd *cobra.Command, args []string) {
		target := getTarget()
		if target.Type() != "cloud" {
			fatalErr(fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type()))
			return
		}
		appSource := applicationSource(args)
		pkg, err := vespa.FindApplicationPackage(appSource, true)
		if err != nil {
			fatalErr(err)
			return
		}
		cfg, err := LoadConfig()
		if err != nil {
			fatalErr(err, "Could not load config")
			return
		}
		if !pkg.HasDeployment() {
			fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init")
			return
		}
		if pkg.TestPath == "" {
			fatalErrHint(fmt.Errorf("No tests found"),
				"The application must be a Java maven project, or include basic HTTP tests under src/test/application/",
				"See https://cloud.vespa.ai/en/getting-to-production")
			return
		}
		verifyTests(pkg.TestPath, target)
		isCI := os.Getenv("CI") != ""
		if !isCI {
			fmt.Fprintln(stderr, color.Yellow("Warning:"), "We recommend doing this only from a CD job")
			printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production")
		}
		opts := getDeploymentOpts(cfg, pkg, target)
		if err := vespa.Submit(opts); err != nil {
			fatalErr(err, "Could not submit application for deployment")
		} else {
			printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment")
			log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment",
				getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application)))
		}
	},
}

func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) error {
	dst := filepath.Join(pkg.Path, filename)
	if util.PathExists(dst) {
		data, err := ioutil.ReadFile(dst)
		if err != nil {
			return err
		}
		if bytes.Equal(data, []byte(contents)) {
			fmt.Fprintf(stdout, "Not writing %s: File is unchanged\n", color.Yellow(filename))
			return nil
		}
		renamed := false
		for i := 1; i <= 1000; i++ {
			bak := fmt.Sprintf("%s.%d.bak", dst, i)
			if !util.PathExists(bak) {
				fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.Yellow(filename), color.Yellow(bak))
				if err := os.Rename(dst, bak); err != nil {
					return err
				}
				renamed = true
				break
			}
		}
		if !renamed {
			return fmt.Errorf("could not find an unused backup name for %s", dst)
		}
	}
	fmt.Fprintf(stdout, "Writing %s\n", color.Green(dst))
	return ioutil.WriteFile(dst, []byte(contents), 0644)
}

func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) xml.Deployment {
	regions := promptRegions(r, deploymentXML)
	parts := strings.Split(regions, ",")
	regionElements := xml.Regions(parts...)
	if err := deploymentXML.Replace("prod", "region", regionElements); err != nil {
		fatalErr(err, "Could not update region elements in deployment.xml")
	}
	// TODO: Some sample apps come with production <test> elements, but not necessarily working production tests, we
	//       therefore remove <test> elements here.
	//       This can be improved by supporting <test> elements in xml package and allow specifying testing as part of
	//       region prompt, e.g. region1;test,region2
	if err := deploymentXML.Replace("prod", "test", nil); err != nil {
		fatalErr(err, "Could not remove test elements in deployment.xml")
	}
	return deploymentXML
}

func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) string {
	fmt.Fprintln(stdout, color.Cyan("> Deployment regions"))
	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/zones"))
	fmt.Fprintf(stdout, "Example: %s\n\n", color.Yellow("aws-us-east-1c,aws-us-west-2a"))
	var currentRegions []string
	for _, r := range deploymentXML.Prod.Regions {
		currentRegions = append(currentRegions, r.Name)
	}
	if len(deploymentXML.Instance) > 0 {
		for _, r := range deploymentXML.Instance[0].Prod.Regions {
			currentRegions = append(currentRegions, r.Name)
		}
	}
	validator := func(input string) error {
		regions := strings.Split(input, ",")
		for _, r := range regions {
			if !xml.IsProdRegion(r, getSystem()) {
				return fmt.Errorf("invalid region %s", r)
			}
		}
		return nil
	}
	return prompt(r, "Which regions do you wish to deploy in?", strings.Join(currentRegions, ","), validator)
}

func updateNodes(r *bufio.Reader, servicesXML xml.Services) xml.Services {
	for _, c := range servicesXML.Container {
		nodes := promptNodes(r, c.ID, c.Nodes)
		if err := servicesXML.Replace("container#"+c.ID, "nodes", nodes); err != nil {
			fatalErr(err)
			return xml.Services{}
		}
	}
	for _, c := range servicesXML.Content {
		nodes := promptNodes(r, c.ID, c.Nodes)
		if err := servicesXML.Replace("content#"+c.ID, "nodes", nodes); err != nil {
			fatalErr(err)
			return xml.Services{}
		}
	}
	return servicesXML
}

func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) xml.Nodes {
	count := promptNodeCount(r, clusterID, defaultValue.Count)
	const autoSpec = "auto"
	defaultSpec := autoSpec
	resources := defaultValue.Resources
	if resources != nil {
		defaultSpec = defaultValue.Resources.String()
	}
	spec := promptResources(r, clusterID, defaultSpec)
	if spec == autoSpec {
		resources = nil
	} else {
		r, err := xml.ParseResources(spec)
		if err != nil {
			fatalErr(err) // Should not happen as resources have already been validated
			return xml.Nodes{}
		}
		resources = &r
	}
	return xml.Nodes{Count: count, Resources: resources}
}

func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) string {
	fmt.Fprintln(stdout, color.Cyan("\n> Node count: "+clusterID+" cluster"))
	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("4"), color.Yellow("[2,8]"))
	validator := func(input string) error {
		_, _, err := xml.ParseNodeCount(input)
		return err
	}
	return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.Cyan(clusterID)), nodeCount, validator)
}

func promptResources(r *bufio.Reader, clusterID string, resources string) string {
	fmt.Fprintln(stdout, color.Cyan("\n> Node resources: "+clusterID+" cluster"))
	fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services"))
	fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("auto"), color.Yellow("vcpu=4,memory=8Gb,disk=100Gb"))
	validator := func(input string) error {
		if input == "auto" {
			return nil
		}
		_, err := xml.ParseResources(input)
		return err
	}
	return prompt(r, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.Cyan(clusterID)), resources, validator)
}

func readDeploymentXML(pkg vespa.ApplicationPackage) (xml.Deployment, error) {
	f, err := os.Open(filepath.Join(pkg.Path, "deployment.xml"))
	if errors.Is(err, os.ErrNotExist) {
		// Return a default value if there is no current deployment.xml
		return xml.DefaultDeployment, nil
	} else if err != nil {
		return xml.Deployment{}, err
	}
	defer f.Close()
	return xml.ReadDeployment(f)
}

func readServicesXML(pkg vespa.ApplicationPackage) (xml.Services, error) {
	f, err := os.Open(filepath.Join(pkg.Path, "services.xml"))
	if err != nil {
		return xml.Services{}, err
	}
	defer f.Close()
	return xml.ReadServices(f)
}

func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(input string) error) string {
	var input string
	for input == "" {
		fmt.Fprint(stdout, question)
		if defaultAnswer != "" {
			fmt.Fprint(stdout, " [", color.Yellow(defaultAnswer), "]")
		}
		fmt.Fprint(stdout, " ")

		var err error
		input, err = r.ReadString('\n')
		if err != nil {
			fatalErr(err)
			return ""
		}
		input = strings.TrimSpace(input)
		if input == "" {
			input = defaultAnswer
		}

		if err := validator(input); err != nil {
			printErr(err)
			fmt.Fprintln(stderr)
			input = ""
		}
	}
	return input
}

func verifyTests(testsParent string, target vespa.Target) {
	verifyTest(testsParent, "system-test", target, true)
	verifyTest(testsParent, "staging-setup", target, true)
	verifyTest(testsParent, "staging-test", target, true)
	verifyTest(testsParent, "production-test", target, false)
}

func verifyTest(testsParent string, suite string, target vespa.Target, required bool) {
	testDirectory := filepath.Join(testsParent, "tests", suite)
	_, err := os.Stat(testDirectory)
	if err != nil {
		if required {
			if errors.Is(err, os.ErrNotExist) {
				fatalErrHint(fmt.Errorf("No %s tests found", suite),
					fmt.Sprintf("No such directory: %s", testDirectory),
					"See https://cloud.vespa.ai/en/reference/testing")
			}
			fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
		}
		return
	}

	runTests(testDirectory, true)
}
<|endoftext|>


<commit_before>package models

import "github.com/jmoiron/sqlx"

// SearchModel provides functionality to search recipes.
type SearchModel struct {
	*Model
}

// SearchFilter is the primary model class for recipe search
type SearchFilter struct {
	Query string
	Tags  []string
}

// Find retrieves all recipes matching the specified search filter and within the range specified,
// sorted by name.
func (m *SearchModel) Find(filter SearchFilter, page int64, count int64) (*Recipes, int64, error) {
	var total int64
	var search string
	if filter.Query == "" {
		search = "%"
	} else {
		search = "%" + filter.Query + "%"
	}
	var like string
	switch m.cfg.DatabaseDriver {
	case "sqlite3":
		like = "LIKE"
	case "postgres":
		like = "ILIKE"
	}
	partialStmt := "FROM recipe AS r " +
		"LEFT OUTER JOIN recipe_tag AS t ON t.recipe_id = r.id " +
		"LEFT OUTER JOIN recipe_rating AS g ON g.recipe_id = r.id " +
		"WHERE (r.name " + like + " ? OR r.Ingredients " + like + " ? OR r.directions " + like + " ? OR t.tag " + like + " ?)"
	if len(filter.Tags) > 0 {
		partialStmt = partialStmt + " AND (t.tag IN (?))"
	}

	countStmt := "SELECT count(DISTINCT r.id) " + partialStmt
	var err error
	var countArgs []interface{}
	if len(filter.Tags) == 0 {
		countStmt, countArgs, err = sqlx.In(countStmt, search, search, search, search)
	} else {
		countStmt, countArgs, err = sqlx.In(countStmt, search, search, search, search, filter.Tags)
	}
	if err != nil {
		return nil, 0, err
	}
	countStmt = m.db.Rebind(countStmt)
	row := m.db.QueryRow(countStmt, countArgs...)
	if err := row.Scan(&total); err != nil {
		return nil, 0, err
	}

	offset := count * (page - 1)
	selectStmt := "SELECT DISTINCT " +
		"r.id, r.name, r.serving_size, r.nutrition_info, r.ingredients, r.directions, COALESCE(g.rating, 0) " +
		partialStmt +
		" ORDER BY r.name LIMIT ? OFFSET ?"
	var selectArgs []interface{}
	if len(filter.Tags) == 0 {
		selectStmt, selectArgs, err = sqlx.In(selectStmt, search, search, search, search, count, offset)
	} else {
		selectStmt, selectArgs, err = sqlx.In(selectStmt, search, search, search, search, filter.Tags, count, offset)
	}
	if err != nil {
		return nil, 0, err
	}
	selectStmt = m.db.Rebind(selectStmt)
	rows, err := m.db.Query(selectStmt, selectArgs...)
	if err != nil {
		return nil, 0, err
	}

	var recipes Recipes
	for rows.Next() {
		var recipe Recipe
		err = rows.Scan(
			&recipe.ID,
			&recipe.Name,
			&recipe.ServingSize,
			&recipe.NutritionInfo,
			&recipe.Ingredients,
			&recipe.Directions,
			&recipe.AvgRating)
		if err != nil {
			return nil, 0, err
		}

		imgs, err := m.Images.List(recipe.ID)
		if err != nil {
			return nil, 0, err
		}
		if len(*imgs) > 0 {
			recipe.Image = (*imgs)[0].ThumbnailURL
		}

		recipes = append(recipes, recipe)
	}

	return &recipes, total, nil
}
<commit_msg>Implemented backend support to sort on different fields.<commit_after>package models

import "github.com/jmoiron/sqlx"

// SearchModel provides functionality to search recipes.
type SearchModel struct {
	*Model
}

// SortBy represents an enumeration of possible sort fields
type SortBy int

const (
	SortByName   SortBy = 0
	SortByID     SortBy = 1
	SortByDate   SortBy = 2
	SortByRandom SortBy = 3
)

// SearchFilter is the primary model class for recipe search
type SearchFilter struct {
	Query  string
	Tags   []string
	SortBy SortBy
}

// Find retrieves all recipes matching the specified search filter and within the range specified,
// sorted by name.
func (m *SearchModel) Find(filter SearchFilter, page int64, count int64) (*Recipes, int64, error) {
	var total int64
	var search string
	if filter.Query == "" {
		search = "%"
	} else {
		search = "%" + filter.Query + "%"
	}
	var like string
	switch m.cfg.DatabaseDriver {
	case "sqlite3":
		like = "LIKE"
	case "postgres":
		like = "ILIKE"
	}
	partialStmt := "FROM recipe AS r " +
		"WHERE (r.name " + like + " ? OR r.Ingredients " + like + " ? OR r.directions " + like + " ? OR EXISTS (SELECT 1 FROM recipe_tag as t WHERE t.recipe_id = r.id AND t.tag " + like + " ?))"
	if len(filter.Tags) > 0 {
		partialStmt += " AND EXISTS (SELECT 1 FROM recipe_tag AS t WHERE t.recipe_id = r.id AND t.tag IN (?))"
	}

	countStmt := "SELECT count(r.id) " + partialStmt
	var err error
	var countArgs []interface{}
	if len(filter.Tags) == 0 {
		countStmt, countArgs, err = sqlx.In(countStmt, search, search, search, search)
	} else {
		countStmt, countArgs, err = sqlx.In(countStmt, search, search, search, search, filter.Tags)
	}
	if err != nil {
		return nil, 0, err
	}
	countStmt = m.db.Rebind(countStmt)
	row := m.db.QueryRow(countStmt, countArgs...)
	if err := row.Scan(&total); err != nil {
		return nil, 0, err
	}

	offset := count * (page - 1)
	selectStmt := "SELECT " +
		"r.id, r.name, r.serving_size, r.nutrition_info, r.ingredients, r.directions, COALESCE((SELECT g.rating FROM recipe_rating AS g WHERE g.recipe_id = r.id), 0)" +
		partialStmt
	switch filter.SortBy {
	case SortByID:
		selectStmt += " ORDER BY r.id"
	case SortByName:
		selectStmt += " ORDER BY r.name"
	// TODO: Don't have date columns yet
	//case SortByDate:
	//	selectStmt += " ORDER BY r.created_on"
	case SortByRandom:
		selectStmt += " ORDER BY RANDOM()"
	}
	selectStmt += " LIMIT ? OFFSET ?"
	var selectArgs []interface{}
	if len(filter.Tags) == 0 {
		selectStmt, selectArgs, err = sqlx.In(selectStmt, search, search, search, search, count, offset)
	} else {
		selectStmt, selectArgs, err = sqlx.In(selectStmt, search, search, search, search, filter.Tags, count, offset)
	}
	if err != nil {
		return nil, 0, err
	}
	selectStmt = m.db.Rebind(selectStmt)
	rows, err := m.db.Query(selectStmt, selectArgs...)
	if err != nil {
		return nil, 0, err
	}

	var recipes Recipes
	for rows.Next() {
		var recipe Recipe
		err = rows.Scan(
			&recipe.ID,
			&recipe.Name,
			&recipe.ServingSize,
			&recipe.NutritionInfo,
			&recipe.Ingredients,
			&recipe.Directions,
			&recipe.AvgRating)
		if err != nil {
			return nil, 0, err
		}

		imgs, err := m.Images.List(recipe.ID)
		if err != nil {
			return nil, 0, err
		}
		if len(*imgs) > 0 {
			recipe.Image = (*imgs)[0].ThumbnailURL
		}

		recipes = append(recipes, recipe)
	}

	return &recipes, total, nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config_test

import (
	"github.com/jacobsa/comeback/config"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestParse(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ParseTest struct {
	data string
	cfg  *config.Config
	err  error
}

func init() { RegisterTestSuite(&ParseTest{}) }

func (t *ParseTest) parse() {
	t.cfg, t.err = config.Parse([]byte(t.data))
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ParseTest) TotalJunk() {
	t.data = "sdhjklfghdskjghdjkfgj"
	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("invalid")))
}

func (t *ParseTest) Null() {
	t.data = `null`
	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("null")))
}

func (t *ParseTest) Array() {
	t.data = `[17, 19]`
	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("array")))
}

func (t *ParseTest) MissingTrailingBrace() {
	t.data = `
	{
		"jobs": {}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("unexpected end")))
}

func (t *ParseTest) BasePathIsNumber() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": 17
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("number")))
}

func (t *ParseTest) BasePathIsNull() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": null
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("null")))
}

func (t *ParseTest) BasePathIsObject() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": {}
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("object")))
}

func (t *ParseTest) OneExcludeDoesntCompile() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": "/foo",
				"excludes": ["a"]
			},
			"burrito": {
				"base_path": "/bar",
				"excludes": ["b", "(c"]
			},
			"enchilada": {
				"base_path": "/foo",
				"excludes": ["d"]
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("TODO")))
}

func (t *ParseTest) EmptyConfig() {
	t.data = `{}`
	t.parse()

	AssertEq(nil, t.err)
	ExpectNe(nil, t.cfg.Jobs)
	ExpectEq(0, len(t.cfg.Jobs))
}

func (t *ParseTest) MissingExcludesArray() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": "/foo"
			}
		}
	}
	`

	t.parse()

	AssertEq(nil, t.err)
	AssertEq(1, len(t.cfg.Jobs))

	AssertNe(nil, t.cfg.Jobs["taco"])
	ExpectThat(t.cfg.Jobs["taco"].Excludes, ElementsAre())
}

func (t *ParseTest) MultipleValidJobs() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": "/foo"
				"excludes": ["a.b"],
			},
			"burrito": {
				"base_path": "/bar",
				"excludes": ["c", "d"]
			}
		}
	}
	`

	t.parse()

	AssertEq(nil, t.err)
	AssertEq(2, len(t.cfg.Jobs))

	AssertNe(nil, t.cfg.Jobs["taco"])
	ExpectEq("/foo", t.cfg.Jobs["taco"].BasePath)
	AssertThat(t.cfg.Jobs["taco"].Excludes, ElementsAre(Any()))
	ExpectEq("a.b", t.cfg.Jobs["taco"].Excludes[0])

	AssertNe(nil, t.cfg.Jobs["burrito"])
	ExpectEq("/bar", t.cfg.Jobs["burrito"].BasePath)
	AssertThat(t.cfg.Jobs["burrito"].Excludes, ElementsAre(Any(), Any()))
	ExpectEq("c", t.cfg.Jobs["burrito"].Excludes[0])
	ExpectEq("d", t.cfg.Jobs["burrito"].Excludes[1])
}
<commit_msg>Fixed some syntax errors.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config_test

import (
	"github.com/jacobsa/comeback/config"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestParse(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ParseTest struct {
	data string
	cfg  *config.Config
	err  error
}

func init() { RegisterTestSuite(&ParseTest{}) }

func (t *ParseTest) parse() {
	t.cfg, t.err = config.Parse([]byte(t.data))
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ParseTest) TotalJunk() {
	t.data = "sdhjklfghdskjghdjkfgj"
	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("invalid")))
}

func (t *ParseTest) Null() {
	t.data = `null`
	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("null")))
}

func (t *ParseTest) Array() {
	t.data = `[17, 19]`
	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("array")))
}

func (t *ParseTest) MissingTrailingBrace() {
	t.data = `
	{
		"jobs": {}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("unexpected end")))
}

func (t *ParseTest) BasePathIsNumber() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": 17
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("number")))
}

func (t *ParseTest) BasePathIsNull() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": null
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("null")))
}

func (t *ParseTest) BasePathIsObject() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": {}
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("JSON")))
	ExpectThat(t.err, Error(HasSubstr("object")))
}

func (t *ParseTest) OneExcludeDoesntCompile() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": "/foo",
				"excludes": ["a"]
			},
			"burrito": {
				"base_path": "/bar",
				"excludes": ["b", "(c"]
			},
			"enchilada": {
				"base_path": "/foo",
				"excludes": ["d"]
			}
		}
	}
	`

	t.parse()

	ExpectThat(t.err, Error(HasSubstr("TODO")))
}

func (t *ParseTest) EmptyConfig() {
	t.data = `{}`
	t.parse()

	AssertEq(nil, t.err)
	ExpectNe(nil, t.cfg.Jobs)
	ExpectEq(0, len(t.cfg.Jobs))
}

func (t *ParseTest) MissingExcludesArray() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": "/foo"
			}
		}
	}
	`

	t.parse()

	AssertEq(nil, t.err)
	AssertEq(1, len(t.cfg.Jobs))

	AssertNe(nil, t.cfg.Jobs["taco"])
	ExpectThat(t.cfg.Jobs["taco"].Excludes, ElementsAre())
}

func (t *ParseTest) MultipleValidJobs() {
	t.data = `
	{
		"jobs": {
			"taco": {
				"base_path": "/foo",
				"excludes": ["a.b"]
			},
			"burrito": {
				"base_path": "/bar",
				"excludes": ["c", "d"]
			}
		}
	}
	`

	t.parse()

	AssertEq(nil, t.err)
	AssertEq(2, len(t.cfg.Jobs))

	AssertNe(nil, t.cfg.Jobs["taco"])
	ExpectEq("/foo", t.cfg.Jobs["taco"].BasePath)
	AssertThat(t.cfg.Jobs["taco"].Excludes, ElementsAre(Any()))
	ExpectEq("a.b", t.cfg.Jobs["taco"].Excludes[0])

	AssertNe(nil, t.cfg.Jobs["burrito"])
	ExpectEq("/bar", t.cfg.Jobs["burrito"].BasePath)
	AssertThat(t.cfg.Jobs["burrito"].Excludes, ElementsAre(Any(), Any()))
	ExpectEq("c", t.cfg.Jobs["burrito"].Excludes[0])
	ExpectEq("d", t.cfg.Jobs["burrito"].Excludes[1])
}
<|endoftext|>