<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package command

// CmdDatacenter subcommand
import (
	"fmt"

	"github.com/fatih/color"
	"github.com/urfave/cli"
)

// CreateAzureDatacenter : Creates an AWS datacenter
var CreateAzureDatacenter = cli.Command{
	Name:  "azure",
	Usage: "Create a new azure datacenter.",
	Description: `Create a new Azure datacenter on the targeted instance of Ernest.

	Example:
	 $ ernest datacenter create azure --region westus --subscription_id SUBSCRIPTION --client_id USER --client_secret PASSWORD --tenant_id TENANT --environment public my_datacenter

   Template example:
    $ ernest datacenter create azure --template mydatacenter.yml mydatacenter
    Where mydatacenter.yaml will look like:
      ---
      fake: true
      region: westus
			subscription_id: SUBSCRIPTION
			client_id: USER
			client_secret: PASSWORD
			tenant_id: TENANT
			environment: public
	 `,
	ArgsUsage: "<datacenter-name>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "region, r",
			Value: "",
			Usage: "Datacenter region",
		},
		cli.StringFlag{
			Name:  "subscription_id, s",
			Value: "",
			Usage: "Azure subscription id",
		},
		cli.StringFlag{
			Name:  "client_id, c",
			Value: "",
			Usage: "Azure client id",
		},
		cli.StringFlag{
			Name:  "client_secret, p",
			Value: "",
			Usage: "Azure client secret",
		},
		cli.StringFlag{
			Name:  "tenant_id, t",
			Value: "",
			Usage: "Azure tenant_id",
		},
		cli.StringFlag{
			Name:  "environment, e",
			Value: "",
			Usage: "Azure environment. Supported values are public(default), usgovernment, german and chine",
		},
		cli.BoolFlag{
			Name:  "fake, f",
			Usage: "Fake datacenter",
		},
	},
	Action: func(c *cli.Context) error {
		var errs []string
		var region, subscriptionID, clientID, clientSecret, tenantID, environment string
		var fake bool
		m, cfg := setup(c)

		if len(c.Args()) < 1 {
			msg := "You should specify the datacenter name"
			color.Red(msg)
			return nil
		}

		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}
		name := c.Args()[0]

		template := c.String("template")
		if template != "" {
			/*
				var t model.DatacenterTemplate
				if err := getDatacenterTemplate(template, &t); err != nil {
					color.Red(err.Error())
					return nil
				}
				accessKeyID = t.Token
				secretAccessKey = t.Secret
				region = t.Region
				fake = t.Fake
			*/
		}
		if c.String("region") != "" {
			region = c.String("region")
		}
		if c.String("subscription_id") != "" {
			subscriptionID = c.String("subscription_id")
		}
		if c.String("client_id") != "" {
			clientID = c.String("client_id")
		}
		if c.String("client_secret") != "" {
			clientSecret = c.String("client_secret")
		}
		if c.String("tenant_id") != "" {
			tenantID = c.String("tenant_id")
		}
		if c.String("environment") != "" {
			environment = c.String("environment")
		}
		if fake == false {
			fake = c.Bool("fake")
		}

		if subscriptionID == "" {
			errs = append(errs, "Specify a valid subscription id with --subscription_id flag")
		}
		if clientID == "" {
			errs = append(errs, "Specify a valid client id with --client_id flag")
		}
		if region == "" {
			errs = append(errs, "Specify a valid region with --region flag")
		}
		if tenantID == "" {
			errs = append(errs, "Specify a valid tenant id with --tenant_id flag")
		}
		if environment == "" {
			errs = append(errs, "Specify a valid environment with --environment flag")
		}

		if len(errs) > 0 {
			color.Red("Please, fix the error shown below to continue")
			for _, e := range errs {
				fmt.Println("  - " + e)
			}
			return nil
		}

		rtype := "azure"

		if fake {
			rtype = "azure-fake"
		}
		body, err := m.CreateAzureDatacenter(cfg.Token, name, rtype, region, subscriptionID, clientID, clientSecret, tenantID, environment)
		if err != nil {
			color.Red(body)
		} else {
			color.Green("Datacenter '" + name + "' successfully created ")
		}
		return nil
	},
}

// UpdateAzureDatacenter : Updates the specified VCloud datacenter
var UpdateAzureDatacenter = cli.Command{
	Name:      "azure",
	Usage:     "Updates the specified Azure datacenter.",
	ArgsUsage: "<datacenter-name>",
	Description: `Updates the specified Azure datacenter.

   Example:
	 $ ernest datacenter update azure --subscription_id SUBSCRIPTION --client_id USER --client_secret PASSWORD --tenant_id TENANT --environment public my_datacenter
	`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "subscription_id, s",
			Value: "",
			Usage: "Azure subscription id",
		},
		cli.StringFlag{
			Name:  "client_id, c",
			Value: "",
			Usage: "Azure client id",
		},
		cli.StringFlag{
			Name:  "client_secret, p",
			Value: "",
			Usage: "Azure client secret",
		},
		cli.StringFlag{
			Name:  "tenant_id, t",
			Value: "",
			Usage: "Azure tenant_id",
		},
		cli.StringFlag{
			Name:  "environment, e",
			Value: "",
			Usage: "Azure environment. Supported values are public(default), usgovernment, german and chine",
		},
	},
	Action: func(c *cli.Context) error {
		var errs []string
		var subscriptionID, clientID, clientSecret, tenantID, environment string
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) == 0 {
			color.Red("You should specify the datacenter name")
			return nil
		}
		name := c.Args()[0]
		if c.String("subscription_id") != "" {
			subscriptionID = c.String("subscription_id")
		}
		if c.String("client_id") != "" {
			clientID = c.String("client_id")
		}
		if c.String("client_secret") != "" {
			clientSecret = c.String("client_secret")
		}
		if c.String("tenant_id") != "" {
			tenantID = c.String("tenant_id")
		}
		if c.String("environment") != "" {
			environment = c.String("environment")
		}

		if subscriptionID == "" {
			errs = append(errs, "Specify a valid subscription id with --subscription_id flag")
		}
		if clientID == "" {
			errs = append(errs, "Specify a valid client id with --client_id flag")
		}
		if clientSecret == "" {
			errs = append(errs, "Specify a valid client secret with --client_secret flag")
		}
		if tenantID == "" {
			errs = append(errs, "Specify a valid tenant id with --tenant_id flag")
		}
		if environment == "" {
			errs = append(errs, "Specify a valid environment with --environment flag")
		}

		if len(errs) > 0 {
			color.Red("Please, fix the error shown below to continue")
			for _, e := range errs {
				fmt.Println("  - " + e)
			}
			return nil
		}

		err := m.UpdateAzureDatacenter(cfg.Token, name, subscriptionID, clientID, clientSecret, tenantID, environment)
		if err != nil {
			color.Red(err.Error())
			return nil
		}
		color.Green("Datacenter " + name + " successfully updated")

		return nil
	},
}
<commit_msg>Make azure environment non required<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package command

// CmdDatacenter subcommand
import (
	"fmt"

	"github.com/fatih/color"
	"github.com/urfave/cli"
)

// CreateAzureDatacenter : Creates an AWS datacenter
var CreateAzureDatacenter = cli.Command{
	Name:  "azure",
	Usage: "Create a new azure datacenter.",
	Description: `Create a new Azure datacenter on the targeted instance of Ernest.

	Example:
	 $ ernest datacenter create azure --region westus --subscription_id SUBSCRIPTION --client_id USER --client_secret PASSWORD --tenant_id TENANT --environment public my_datacenter

   Template example:
    $ ernest datacenter create azure --template mydatacenter.yml mydatacenter
    Where mydatacenter.yaml will look like:
      ---
      fake: true
      region: westus
			subscription_id: SUBSCRIPTION
			client_id: USER
			client_secret: PASSWORD
			tenant_id: TENANT
			environment: public
	 `,
	ArgsUsage: "<datacenter-name>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "region, r",
			Value: "",
			Usage: "Datacenter region",
		},
		cli.StringFlag{
			Name:  "subscription_id, s",
			Value: "",
			Usage: "Azure subscription id",
		},
		cli.StringFlag{
			Name:  "client_id, c",
			Value: "",
			Usage: "Azure client id",
		},
		cli.StringFlag{
			Name:  "client_secret, p",
			Value: "",
			Usage: "Azure client secret",
		},
		cli.StringFlag{
			Name:  "tenant_id, t",
			Value: "",
			Usage: "Azure tenant_id",
		},
		cli.StringFlag{
			Name:  "environment, e",
			Value: "",
			Usage: "Azure environment. Supported values are public(default), usgovernment, german and chine",
		},
		cli.BoolFlag{
			Name:  "fake, f",
			Usage: "Fake datacenter",
		},
	},
	Action: func(c *cli.Context) error {
		var errs []string
		var region, subscriptionID, clientID, clientSecret, tenantID, environment string
		var fake bool
		m, cfg := setup(c)

		if len(c.Args()) < 1 {
			msg := "You should specify the datacenter name"
			color.Red(msg)
			return nil
		}

		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}
		name := c.Args()[0]

		template := c.String("template")
		if template != "" {
			/*
				var t model.DatacenterTemplate
				if err := getDatacenterTemplate(template, &t); err != nil {
					color.Red(err.Error())
					return nil
				}
				accessKeyID = t.Token
				secretAccessKey = t.Secret
				region = t.Region
				fake = t.Fake
			*/
		}
		if c.String("region") != "" {
			region = c.String("region")
		}
		if c.String("subscription_id") != "" {
			subscriptionID = c.String("subscription_id")
		}
		if c.String("client_id") != "" {
			clientID = c.String("client_id")
		}
		if c.String("client_secret") != "" {
			clientSecret = c.String("client_secret")
		}
		if c.String("tenant_id") != "" {
			tenantID = c.String("tenant_id")
		}
		if c.String("environment") != "" {
			environment = c.String("environment")
		}
		if fake == false {
			fake = c.Bool("fake")
		}

		if subscriptionID == "" {
			errs = append(errs, "Specify a valid subscription id with --subscription_id flag")
		}
		if clientID == "" {
			errs = append(errs, "Specify a valid client id with --client_id flag")
		}
		if region == "" {
			errs = append(errs, "Specify a valid region with --region flag")
		}
		if tenantID == "" {
			errs = append(errs, "Specify a valid tenant id with --tenant_id flag")
		}
		if environment == "" {
			environment = "public"
		}

		if len(errs) > 0 {
			color.Red("Please, fix the error shown below to continue")
			for _, e := range errs {
				fmt.Println("  - " + e)
			}
			return nil
		}

		rtype := "azure"

		if fake {
			rtype = "azure-fake"
		}
		body, err := m.CreateAzureDatacenter(cfg.Token, name, rtype, region, subscriptionID, clientID, clientSecret, tenantID, environment)
		if err != nil {
			color.Red(body)
		} else {
			color.Green("Datacenter '" + name + "' successfully created ")
		}
		return nil
	},
}

// UpdateAzureDatacenter : Updates the specified VCloud datacenter
var UpdateAzureDatacenter = cli.Command{
	Name:      "azure",
	Usage:     "Updates the specified Azure datacenter.",
	ArgsUsage: "<datacenter-name>",
	Description: `Updates the specified Azure datacenter.

   Example:
	 $ ernest datacenter update azure --subscription_id SUBSCRIPTION --client_id USER --client_secret PASSWORD --tenant_id TENANT --environment public my_datacenter
	`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "subscription_id, s",
			Value: "",
			Usage: "Azure subscription id",
		},
		cli.StringFlag{
			Name:  "client_id, c",
			Value: "",
			Usage: "Azure client id",
		},
		cli.StringFlag{
			Name:  "client_secret, p",
			Value: "",
			Usage: "Azure client secret",
		},
		cli.StringFlag{
			Name:  "tenant_id, t",
			Value: "",
			Usage: "Azure tenant_id",
		},
		cli.StringFlag{
			Name:  "environment, e",
			Value: "",
			Usage: "Azure environment. Supported values are public(default), usgovernment, german and chine",
		},
	},
	Action: func(c *cli.Context) error {
		var errs []string
		var subscriptionID, clientID, clientSecret, tenantID, environment string
		m, cfg := setup(c)
		if cfg.Token == "" {
			color.Red("You're not allowed to perform this action, please log in")
			return nil
		}

		if len(c.Args()) == 0 {
			color.Red("You should specify the datacenter name")
			return nil
		}
		name := c.Args()[0]
		if c.String("subscription_id") != "" {
			subscriptionID = c.String("subscription_id")
		}
		if c.String("client_id") != "" {
			clientID = c.String("client_id")
		}
		if c.String("client_secret") != "" {
			clientSecret = c.String("client_secret")
		}
		if c.String("tenant_id") != "" {
			tenantID = c.String("tenant_id")
		}
		if c.String("environment") != "" {
			environment = c.String("environment")
		}

		if subscriptionID == "" {
			errs = append(errs, "Specify a valid subscription id with --subscription_id flag")
		}
		if clientID == "" {
			errs = append(errs, "Specify a valid client id with --client_id flag")
		}
		if clientSecret == "" {
			errs = append(errs, "Specify a valid client secret with --client_secret flag")
		}
		if tenantID == "" {
			errs = append(errs, "Specify a valid tenant id with --tenant_id flag")
		}
		if environment == "" {
			errs = append(errs, "Specify a valid environment with --environment flag")
		}

		if len(errs) > 0 {
			color.Red("Please, fix the error shown below to continue")
			for _, e := range errs {
				fmt.Println("  - " + e)
			}
			return nil
		}

		err := m.UpdateAzureDatacenter(cfg.Token, name, subscriptionID, clientID, clientSecret, tenantID, environment)
		if err != nil {
			color.Red(err.Error())
			return nil
		}
		color.Green("Datacenter " + name + " successfully updated")

		return nil
	},
}
<|endoftext|>


<commit_before>package main

// Installation of required library :
// GOPATH=`pwd` go get github.com/go-sql-driver/mysql

// Create a database user 'reverse-gol' with password 'reverse-gol' with access rights to database 'reverse-gol'

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"

	"encoding/csv"
	"fmt"
	"io"
	"os"
	"strconv"

)

/*
CREATE TABLE `problems` (
	`id` int(11) NOT NULL,
	`steps` int(11) NOT NULL,
	`solution_count` int(11) NOT NULL,
	`currently_processing` int(11) NOT NULL,
	KEY `problems_id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
CREATE TABLE `solutions` (
	`id` int(11) NOT NULL,
	`steps` int(11) NOT NULL,
	`iter` int(11) NOT NULL,
	`seed` int(11) NOT NULL,
	`version` int(11) NOT NULL,
	`mtsi` int(11) DEFAULT NULL,
	`mtei` int(11) NOT NULL,
	`mtsf` int(11) DEFAULT NULL,
	`mtef` int(11) NOT NULL,
	`start` text NOT NULL,
	KEY `solutions_id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
*/

func get_db_connection() *sql.DB {
	//db, err := sql.Open("mysql", "user:password@/database")
	//db, err := sql.Open("mysql", "reverse-gol:reverse-gol@/reverse-gol")
	db, err := sql.Open("mysql", "reverse-gol:reverse-gol@tcp(square.herald:3306)/reverse-gol")

	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}

	// Open doesn't open a connection. Validate DSN data:
	err = db.Ping()
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	return db
}

func test_open_db() {
	db := get_db_connection()
	defer db.Close()
}

func create_list_of_problems_in_db() {
	db := get_db_connection()
	defer db.Close()

	db.Exec("DELETE FROM PROBLEMS")

	ins, err := db.Prepare("INSERT INTO problems SET id=?, steps=?, solution_count=?")
	if err != nil {
		fmt.Println("Insert Prepare Error:", err)
		return
	}

	for i,f := range []string{"data/train.csv", "data/test.csv"} {
		fmt.Printf("Opening %s - %d\n", f, i)

		file, err := os.Open(f)
		if err != nil {
			fmt.Println("Error:", err)
			return
		}
		defer file.Close()

		reader := csv.NewReader(file)

		header, err := reader.Read()
		if header[0] != "id" {
			fmt.Println("Bad Header", err)
			return
		}

		for {
			record, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				fmt.Println("Error:", err)
				return
			}

			id, _ := strconv.Atoi(record[0])
			steps, _ := strconv.Atoi(record[1])

			if i==0 {
				// This is the training data : Let's give it a negative id to avoid confusion
				id = -id
			}
			_, err = ins.Exec(id, steps, 0)
			if err != nil {
				fmt.Println("Insert Error:", err)
				return
			}
		}
	}

}

// training examples are stored in the dbs with negative ids.  All ids output from here are corrected to be positive
func list_of_interesting_problems_from_db(steps int, count int, is_training bool) []int {
	db := get_db_connection()
	defer db.Close()

	problem_list := []int{}

	filter_training_or_test := "id>0"
	if is_training {
		filter_training_or_test = "id<0"
	}

	// Only actually need id back
	rows, err := db.Query("SELECT id, steps, solution_count, currently_processing FROM problems"+
							" WHERE currently_processing=0 AND steps=? AND "+filter_training_or_test+
							" ORDER BY solution_count ASC"+
							" LIMIT ?",
							steps, count)
	if err != nil {
		fmt.Println("Query interesting problems Error:", err)
		return problem_list
	}

	update, err := db.Prepare("UPDATE problems SET currently_processing=1 WHERE id=?")
	if err != nil {
		fmt.Println("Update 'currently_processing' Prepare Error:", err)
		return problem_list
	}

	for rows.Next() {
		var id, steps, solution_count,currently_processing int
		err = rows.Scan(&id, &steps, &solution_count, &currently_processing)
		//err = rows.Err() // get any error encountered during iteration
		if err != nil {
			fmt.Println("Query interesting problem Error:", err)
			return problem_list
		}

		_, err = update.Exec(id)
		if err != nil {
			fmt.Println("Update 'currently_processing' Exec Error:", err)
			return problem_list
		}

		if is_training {
			id=-id
		}
		problem_list = append(problem_list, id)
	}

	return problem_list
}

func reset_all_currently_processing(is_training bool) {
	db := get_db_connection()
	defer db.Close()

	filter_training_or_test := "id>0"
	if is_training {
		filter_training_or_test = "id<0"
	}

	_, err := db.Exec("UPDATE problems SET currently_processing=0 WHERE "+filter_training_or_test)
	if err != nil {
		fmt.Println("Reset currently_processing Error:", err)
		return
	}
}

func get_unprocessed_seed_from_db(id int, is_training bool) int {
	db := get_db_connection()
	defer db.Close()

	if is_training {
		id = -id
	}

	seed:=1
	rows, err := db.Query("SELECT MAX(seed)+1 FROM solutions WHERE id=?", id)
	if err != nil {
		fmt.Println("Query seed_max Error:", err)
		return seed
	}

	for rows.Next() {
		var seed_max sql.NullInt64
		err = rows.Scan(&seed_max)
		if err != nil {
			fmt.Println("Query seed_max row Error:", err)
			return seed
		}
		if seed_max.Valid {
			seed = int(seed_max.Int64)
		}
	}
	return seed
}

func save_solution_to_db(id int, steps int, seed int, individual_result *IndividualResult, is_training bool) {
	// add to solutions
	db := get_db_connection()
	defer db.Close()

	if is_training {
		id = -id // Fix it up
	}

	// insert into the solutions db
	_, err := db.Exec("INSERT INTO solutions SET id=?, steps=?, seed=?, version=?, iter=?,"+
						" mtsi=?, mtei=?,"+
						" mtsf=?, mtef=?,"+
						" start=?",
						id, steps, seed,
						currently_running_version,
						individual_result.iter,
						individual_result.mismatch_from_true_start_initial,
						individual_result.mismatch_from_true_end_initial,
						individual_result.mismatch_from_true_start_final,
						individual_result.mismatch_from_true_end_final,
						individual_result.individual.start.toCompactString(),
					)
	if err != nil {
		fmt.Println("Inserting into solutions table for individual Error:", err)
		return
	}

	// increment problems solution_count, and reset currently_processing
	_, err = db.Exec("UPDATE problems SET solution_count=solution_count+1, currently_processing=0 WHERE id=?", id)
	if err != nil {
		fmt.Println("Updating problems table for individual Error:", err)
		return
	}
}

func create_submission(fname string) {
	id_list := []int{}

	if false { // true for real submission, false for testing
		for i:=1; i<=50000; i++ {
			id_list = append(id_list, i)
		}
	} else {
		//id_list = append(id_list, -50)
		id_list = append(id_list, -54)
	}

	db := get_db_connection()
	defer db.Close()

	file, err := os.Create(fname)
	if err != nil {
		fmt.Println("File Creation Error:", err)
		return
	}
	defer file.Close()

	file.WriteString("id")
	for i:=1; i<=400; i++ {
		file.WriteString(fmt.Sprintf(",start.%d", i))
	}
	file.WriteString("\n")

	query, err := db.Prepare("SELECT start FROM solutions WHERE id=?")
	if err != nil {
		fmt.Println("Query solutions prepare Error:", err)
		return
	}

	for _, id := range id_list {
		rows, err := query.Query(id)
		if err != nil {
			fmt.Println("Query solutions row for id=%d Error:", err)
			return
		}

		stats := NewBoardStats(board_width, board_height)
		for rows.Next() {
			var start string
			err = rows.Scan(&start)
			if err != nil {
				fmt.Println("Query start for id=%d Error:", id, err)
				return
			}

			start_board := NewBoard_BoolPacked(board_width, board_height)
			start_board.fromCompactString(start)
			start_board.AddToStats(stats)
			//fmt.Println(start_board)
		}

		//fmt.Println(stats)

		// Ok, so now let's figure out a board from these stats that's a better guess
		guess_board := NewBoard_BoolPacked(board_width, board_height)
		guess_board.ThresholdStats(stats, 51)
		//fmt.Println(guess_board)

		file.WriteString(fmt.Sprintf("%d", id))
		file.WriteString(guess_board.toCSV())
		file.WriteString("\n")
	}
	fmt.Printf("TODO : gzip %s\n", fname)
}

/* :: Useful SQL ::
 * select id,steps,iter from solutions where id>0 and id<50 order by steps,id,iter
 * select steps,seed,count(id) from solutions where id>0 group by steps, seed
 *
 * How many of each type do we have ::
 * select steps,count(id) from solutions where id>0 group by steps
 *
 * Fix early aborted runs ::
 * update problems set currently_processing=0 where id>0 and steps=3
 *
 * What is the state of all solutions ::
 * select steps,solution_count,count(id) from problems where id>0 group by steps, solution_count order by steps,solution_count
 *
 * Where are there holes to fill ::
 * select steps,solution_count,count(id) from problems where id>0 and solution_count=0 group by steps, solution_count order by steps
 *
 * What is currently being worked on ::
 * select steps,solution_count,count(id) from problems where id>0 and currently_processing=1 group by steps, solution_count order by steps
 *
 */
<commit_msg>Attempt to have a tie-breaking mechanism for multi-row board aggregation<commit_after>package main

// Installation of required library :
// GOPATH=`pwd` go get github.com/go-sql-driver/mysql

// Create a database user 'reverse-gol' with password 'reverse-gol' with access rights to database 'reverse-gol'

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"

	"encoding/csv"
	"fmt"
	"io"
	"os"
	"strconv"

)

/*
CREATE TABLE `problems` (
	`id` int(11) NOT NULL,
	`steps` int(11) NOT NULL,
	`solution_count` int(11) NOT NULL,
	`currently_processing` int(11) NOT NULL,
	KEY `problems_id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
CREATE TABLE `solutions` (
	`id` int(11) NOT NULL,
	`steps` int(11) NOT NULL,
	`iter` int(11) NOT NULL,
	`seed` int(11) NOT NULL,
	`version` int(11) NOT NULL,
	`mtsi` int(11) DEFAULT NULL,
	`mtei` int(11) NOT NULL,
	`mtsf` int(11) DEFAULT NULL,
	`mtef` int(11) NOT NULL,
	`start` text NOT NULL,
	KEY `solutions_id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
*/

func get_db_connection() *sql.DB {
	//db, err := sql.Open("mysql", "user:password@/database")
	//db, err := sql.Open("mysql", "reverse-gol:reverse-gol@/reverse-gol")
	db, err := sql.Open("mysql", "reverse-gol:reverse-gol@tcp(square.herald:3306)/reverse-gol")

	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}

	// Open doesn't open a connection. Validate DSN data:
	err = db.Ping()
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	return db
}

func test_open_db() {
	db := get_db_connection()
	defer db.Close()
}

func create_list_of_problems_in_db() {
	db := get_db_connection()
	defer db.Close()

	db.Exec("DELETE FROM PROBLEMS")

	ins, err := db.Prepare("INSERT INTO problems SET id=?, steps=?, solution_count=?")
	if err != nil {
		fmt.Println("Insert Prepare Error:", err)
		return
	}

	for i,f := range []string{"data/train.csv", "data/test.csv"} {
		fmt.Printf("Opening %s - %d\n", f, i)

		file, err := os.Open(f)
		if err != nil {
			fmt.Println("Error:", err)
			return
		}
		defer file.Close()

		reader := csv.NewReader(file)

		header, err := reader.Read()
		if header[0] != "id" {
			fmt.Println("Bad Header", err)
			return
		}

		for {
			record, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				fmt.Println("Error:", err)
				return
			}

			id, _ := strconv.Atoi(record[0])
			steps, _ := strconv.Atoi(record[1])

			if i==0 {
				// This is the training data : Let's give it a negative id to avoid confusion
				id = -id
			}
			_, err = ins.Exec(id, steps, 0)
			if err != nil {
				fmt.Println("Insert Error:", err)
				return
			}
		}
	}

}

// training examples are stored in the dbs with negative ids.  All ids output from here are corrected to be positive
func list_of_interesting_problems_from_db(steps int, count int, is_training bool) []int {
	db := get_db_connection()
	defer db.Close()

	problem_list := []int{}

	filter_training_or_test := "id>0"
	if is_training {
		filter_training_or_test = "id<0"
	}

	// Only actually need id back
	rows, err := db.Query("SELECT id, steps, solution_count, currently_processing FROM problems"+
							" WHERE currently_processing=0 AND steps=? AND "+filter_training_or_test+
							" ORDER BY solution_count ASC"+
							" LIMIT ?",
							steps, count)
	if err != nil {
		fmt.Println("Query interesting problems Error:", err)
		return problem_list
	}

	update, err := db.Prepare("UPDATE problems SET currently_processing=1 WHERE id=?")
	if err != nil {
		fmt.Println("Update 'currently_processing' Prepare Error:", err)
		return problem_list
	}

	for rows.Next() {
		var id, steps, solution_count,currently_processing int
		err = rows.Scan(&id, &steps, &solution_count, &currently_processing)
		//err = rows.Err() // get any error encountered during iteration
		if err != nil {
			fmt.Println("Query interesting problem Error:", err)
			return problem_list
		}

		_, err = update.Exec(id)
		if err != nil {
			fmt.Println("Update 'currently_processing' Exec Error:", err)
			return problem_list
		}

		if is_training {
			id=-id
		}
		problem_list = append(problem_list, id)
	}

	return problem_list
}

func reset_all_currently_processing(is_training bool) {
	db := get_db_connection()
	defer db.Close()

	filter_training_or_test := "id>0"
	if is_training {
		filter_training_or_test = "id<0"
	}

	_, err := db.Exec("UPDATE problems SET currently_processing=0 WHERE "+filter_training_or_test)
	if err != nil {
		fmt.Println("Reset currently_processing Error:", err)
		return
	}
}

func get_unprocessed_seed_from_db(id int, is_training bool) int {
	db := get_db_connection()
	defer db.Close()

	if is_training {
		id = -id
	}

	seed:=1
	rows, err := db.Query("SELECT MAX(seed)+1 FROM solutions WHERE id=?", id)
	if err != nil {
		fmt.Println("Query seed_max Error:", err)
		return seed
	}

	for rows.Next() {
		var seed_max sql.NullInt64
		err = rows.Scan(&seed_max)
		if err != nil {
			fmt.Println("Query seed_max row Error:", err)
			return seed
		}
		if seed_max.Valid {
			seed = int(seed_max.Int64)
		}
	}
	return seed
}

func save_solution_to_db(id int, steps int, seed int, individual_result *IndividualResult, is_training bool) {
	// add to solutions
	db := get_db_connection()
	defer db.Close()

	if is_training {
		id = -id // Fix it up
	}

	// insert into the solutions db
	_, err := db.Exec("INSERT INTO solutions SET id=?, steps=?, seed=?, version=?, iter=?,"+
						" mtsi=?, mtei=?,"+
						" mtsf=?, mtef=?,"+
						" start=?",
						id, steps, seed,
						currently_running_version,
						individual_result.iter,
						individual_result.mismatch_from_true_start_initial,
						individual_result.mismatch_from_true_end_initial,
						individual_result.mismatch_from_true_start_final,
						individual_result.mismatch_from_true_end_final,
						individual_result.individual.start.toCompactString(),
					)
	if err != nil {
		fmt.Println("Inserting into solutions table for individual Error:", err)
		return
	}

	// increment problems solution_count, and reset currently_processing
	_, err = db.Exec("UPDATE problems SET solution_count=solution_count+1, currently_processing=0 WHERE id=?", id)
	if err != nil {
		fmt.Println("Updating problems table for individual Error:", err)
		return
	}
}

func create_submission(fname string) {
	id_list := []int{}

	if false { // true for real submission, false for testing
		for i:=1; i<=50000; i++ {
			id_list = append(id_list, i)
		}
	} else {
		//id_list = append(id_list, -50)
		id_list = append(id_list, -54)
	}

	db := get_db_connection()
	defer db.Close()

	file, err := os.Create(fname)
	if err != nil {
		fmt.Println("File Creation Error:", err)
		return
	}
	defer file.Close()

	file.WriteString("id")
	for i:=1; i<=400; i++ {
		file.WriteString(fmt.Sprintf(",start.%d", i))
	}
	file.WriteString("\n")

	query, err := db.Prepare("SELECT iter,seed,version,mtei,mtef,start FROM solutions WHERE id=?")
	if err != nil {
		fmt.Println("Query solutions prepare Error:", err)
		return
	}

	count_ids_found:=0
	for _, id := range id_list {
		rows, err := query.Query(id)
		if err != nil {
			fmt.Println("Query solutions row for id=%d Error:", err)
			return
		}

		type BestRow struct {
			start_board *Board_BoolPacked
			iter,seed,version int
			mtei, mtef int
			valid bool
		}
		best := BestRow{valid:false}
		stats := NewBoardStats(board_width, board_height)
		for rows.Next() {
			var start string
			var iter,seed,version int
			var mtei, mtef int
			err = rows.Scan(&iter, &seed, &version, &mtei, &mtef, &start)
			if err != nil {
				fmt.Println("Query start for id=%d Error:", id, err)
				return
			}

			// Perhaps there are some entries we want to exclude altogether
			if version==999 {
				continue
			}

			start_board := NewBoard_BoolPacked(board_width, board_height)
			start_board.fromCompactString(start)

			// Do every entry twice ( so that an additional +1 for the best will tie-break a 50/50 threshold)
			start_board.AddToStats(stats)
			start_board.AddToStats(stats)
			//fmt.Println(start_board)

			// Figure out whether this is going to be the best board for tie-breaking
			this_is_better_than_current_best := !best.valid // This picks up the first one immediately

			// If we got the answer quicker, or ended up with fewer errors, this is probably better
			if iter<best.iter || mtef<best.mtef {
				this_is_better_than_current_best=true
			}

			if this_is_better_than_current_best {
				best = BestRow{start_board, iter, seed, version, mtei, mtef, true}
			}

			count_ids_found++

		}
		// Now add a single instance of best board
		if best.valid {
			best.start_board.AddToStats(stats)
		}

		//fmt.Println(stats)

		// Ok, so now let's figure out a board from these stats that's a better guess
		guess_board := NewBoard_BoolPacked(board_width, board_height)
		guess_board.ThresholdStats(stats, 51)
		//fmt.Println(guess_board)

		file.WriteString(fmt.Sprintf("%d", id))
		file.WriteString(guess_board.toCSV())
		file.WriteString("\n")
	}
	if count_ids_found == len(id_list) {
		fmt.Printf("TODO : gzip %s\n", fname)
	} else {
		fmt.Printf("BAD SUBMISSION FILE :: ONLY HAS %d of %d IDs!\n", count_ids_found, len(id_list))
	}
}

/* :: Useful SQL ::
 * select id,steps,iter from solutions where id>0 and id<50 order by steps,id,iter
 * select steps,seed,count(id) from solutions where id>0 group by steps, seed
 *
 * How many of each type do we have ::
 * select steps,count(id) from solutions where id>0 group by steps
 *
 * Fix early aborted runs ::
 * update problems set currently_processing=0 where id>0 and steps=3
 *
 * What is the state of all solutions ::
 * select steps,solution_count,count(id) from problems where id>0 group by steps, solution_count order by steps,solution_count
 *
 * Where are there holes to fill ::
 * select steps,solution_count,count(id) from problems where id>0 and solution_count=0 group by steps, solution_count order by steps
 *
 * What is currently being worked on ::
 * select steps,solution_count,count(id) from problems where id>0 and currently_processing=1 group by steps, solution_count order by steps
 *
 */
<|endoftext|>


<commit_before>/*

	cyle's simple graph database, version 0.1

*/

package main

import "fmt"
import "net/http"
//import "encoding/json" // documentation: http://golang.org/pkg/encoding/json/

// using gorest: https://code.google.com/p/gorest/wiki/GettingStarted?tm=6
import "code.google.com/p/gorest"

type AllTheData struct {
	nodes []Node
	connections []Connection
}

type Node struct {
	Id int
	Name string
	ExtraJSONBytes []byte
	ExtraJSON []interface{}
}

type Connection struct {
	Id int
	Name string
	Source int
	Target int
}

var theData AllTheData

func main() {

	fmt.Println("Oh dear, a graph database...")

	// create some dummy nodes!
	for i := 1; i <= 10; i++ {
		tmpNode := Node{ i, "Node "+fmt.Sprintf("%d", i), nil, nil }
		theData.nodes = append(theData.nodes, tmpNode)
	}

	// create some dummy connections!
	connOne := Connection{ 1, "Node 1 to 2", 1, 2 }
	connTwo := Connection{ 2, "Node 2 to 4", 2, 4 }
	connThree := Connection{ 3, "Node 4 to 5", 4, 5 }
	connFour := Connection{ 4, "Node 2 to 3", 2, 3 }
	connFive := Connection{ 5, "Node 3 to 5", 3, 5 }
	// add connections to the big data pool
	theData.connections = append(theData.connections, connOne, connTwo, connThree, connFour, connFive)

	//fmt.Printf("%+v \n", theData)

	// start the REST service to access the data
	gorest.RegisterService(new(GraphService))
	http.Handle("/", gorest.Handle())
	http.ListenAndServe(":8777", nil)
}

type GraphService struct{
    // service level config
    gorest.RestService `root:"/" consumes:"application/json" produces:"application/json"`
	// define routes

	// deal with the root
    rootHandler gorest.EndPoint `method:"GET" path:"/" output:"string"`
	getNodeHandler gorest.EndPoint `method:"GET" path:"/node/{Id:int}" output:"Node"`
	getConnectionHandler gorest.EndPoint `method:"GET" path:"/connection/{Id:int}" output:"Connection"`
}

func(serv GraphService) RootHandler() string {
	return "Simple Graph Database, v0.1"
}

func(serv GraphService) GetNodeHandler(Id int) (n Node){

	fmt.Printf("Asking for node ID: %d \n", Id)

	for _, value := range theData.nodes {
		if value.Id == Id {
			n = value
			fmt.Printf("Giving: %+v \n", n)
			return
		}
	}

	/*
	n.Id = Id
	n.Name = "Some Node"
	n.ExtraJSONBytes = []byte(`[{"Name": "Platypus", "Order": "Monotremata"}]`)
	// this technique of taking arbitrary JSON and turning it into something usable came from: http://blog.golang.org/json-and-go
	var tmp interface {}
	err := json.Unmarshal(n.ExtraJSONBytes, &tmp)
	if err != nil {
		fmt.Println("error:", err)
	}
	//fmt.Printf("JSON: %T %+v \n", tmp, tmp)
	n.ExtraJSON = tmp.([]interface{})
	n.ExtraJSONBytes = nil
	*/

	// could not find it! send 404
    serv.ResponseBuilder().SetResponseCode(404).Overide(true)  //Overide causes the entity returned by the method to be ignored. Other wise it would send back zeroed object
    return
}

func(serv GraphService) GetConnectionHandler(Id int) (c Connection){

	fmt.Printf("Asking for connection ID: %d \n", Id)

	for _, value := range theData.connections {
		if value.Id == Id {
			c = value
			fmt.Printf("Giving: %+v \n", c)
			return
		}
	}

	// could not find it! send 404
    serv.ResponseBuilder().SetResponseCode(404).Overide(true)  //Overide causes the entity returned by the method to be ignored. Other wise it would send back zeroed object
    return
}
<commit_msg>- added working functions to add nodes and connections - added dummy functions to delete nodes and connections<commit_after>/*

	cyle's simple graph database, version 0.1

*/

package main

import "fmt"
import "net/http"
//import "encoding/json" // documentation: http://golang.org/pkg/encoding/json/

// using gorest: https://code.google.com/p/gorest/wiki/GettingStarted?tm=6
import "code.google.com/p/gorest"

type AllTheData struct {
	nodes []Node
	connections []Connection
}

type Node struct {
	Id int
	Name string
	ExtraJSONBytes []byte
	ExtraJSON []interface{}
}

type Connection struct {
	Id int
	Name string
	Source int
	Target int
}

var theData AllTheData

func main() {

	fmt.Println("Oh dear, a graph database...")

	// create some dummy nodes!
	for i := 0; i < 10; i++ {
		tmpNode := Node{ i, "Node "+fmt.Sprintf("%d", i), nil, nil }
		theData.nodes = append(theData.nodes, tmpNode)
	}

	// create some dummy connections!
	connOne := Connection{ 0, "Node 1 to 2", 1, 2 }
	connTwo := Connection{ 1, "Node 2 to 4", 2, 4 }
	connThree := Connection{ 2, "Node 4 to 5", 4, 5 }
	connFour := Connection{ 3, "Node 2 to 3", 2, 3 }
	connFive := Connection{ 4, "Node 3 to 5", 3, 5 }
	// add connections to the big data pool
	theData.connections = append(theData.connections, connOne, connTwo, connThree, connFour, connFive)

	//fmt.Printf("%+v \n", theData)

	// start the REST service to access the data
	gorest.RegisterService(new(GraphService))
	http.Handle("/", gorest.Handle())
	http.ListenAndServe(":8777", nil)
}

type GraphService struct{
    // service level config
    gorest.RestService `root:"/" consumes:"application/json" produces:"application/json"`
	// define routes

	// deal with the root
    rootHandler gorest.EndPoint `method:"GET" path:"/" output:"string"`

	// node stuff
	getNodeHandler gorest.EndPoint `method:"GET" path:"/node/{Id:int}" output:"Node"`
	postNodeHandler gorest.EndPoint `method:"POST" path:"/node" postdata:"Node"`
	deleteNodeHandler gorest.EndPoint `method:"DELETE" path:"/node/{Id:int}"`

	// connections stuff
	getConnectionHandler gorest.EndPoint `method:"GET" path:"/connection/{Id:int}" output:"Connection"`
	postConnectionHandler gorest.EndPoint `method:"POST" path:"/connection" postdata:"Connection"`
	deleteConnectionHandler gorest.EndPoint `method:"DELETE" path:"/connection/{Id:int}"`
}

func (serv GraphService) RootHandler() string {
	return "Simple Graph Database, v0.1"
}

/*

	node functions

*/

func (serv GraphService) GetNodeHandler(Id int) (n Node){

	fmt.Printf("Asking for node ID: %d \n", Id)

	for _, value := range theData.nodes {
		if value.Id == Id {
			n = value
			fmt.Printf("Giving: %+v \n", n)
			return
		}
	}

	/*
	n.Id = Id
	n.Name = "Some Node"
	n.ExtraJSONBytes = []byte(`[{"Name": "Platypus", "Order": "Monotremata"}]`)
	// this technique of taking arbitrary JSON and turning it into something usable came from: http://blog.golang.org/json-and-go
	var tmp interface {}
	err := json.Unmarshal(n.ExtraJSONBytes, &tmp)
	if err != nil {
		fmt.Println("error:", err)
	}
	//fmt.Printf("JSON: %T %+v \n", tmp, tmp)
	n.ExtraJSON = tmp.([]interface{})
	n.ExtraJSONBytes = nil
	*/

	// could not find it! send 404
    serv.ResponseBuilder().SetResponseCode(404).Overide(true)  //Overide causes the entity returned by the method to be ignored. Other wise it would send back zeroed object
    return
}

func (serv GraphService) PostNodeHandler(n Node) {
	fmt.Printf("Just got: %+v \n", n)
	n.Id = len(theData.nodes)
	theData.nodes = append(theData.nodes, n)
	serv.ResponseBuilder().SetResponseCode(200)
	return
}

func (serv GraphService) DeleteNodeHandler(Id int) {
	serv.ResponseBuilder().SetResponseCode(200)
	return
}

/*

	connection functions

*/

func (serv GraphService) GetConnectionHandler(Id int) (c Connection){

	fmt.Printf("Asking for connection ID: %d \n", Id)

	for _, value := range theData.connections {
		if value.Id == Id {
			c = value
			fmt.Printf("Giving: %+v \n", c)
			return
		}
	}

	// could not find it! send 404
    serv.ResponseBuilder().SetResponseCode(404).Overide(true)  //Overide causes the entity returned by the method to be ignored. Other wise it would send back zeroed object
    return
}

func (serv GraphService) PostConnectionHandler(c Connection) {
	fmt.Printf("Just got: %+v \n", c)
	c.Id = len(theData.connections)
	theData.connections = append(theData.connections, c)
	serv.ResponseBuilder().SetResponseCode(200)
	return
}

func (serv GraphService) DeleteConnectionHandler(Id int) {
	serv.ResponseBuilder().SetResponseCode(200)
	return
}
<|endoftext|>


<commit_before>package ezdb

import (
	"errors"
	"github.com/jmhodges/levigo"
	"github.com/northbright/fnlog"
	"log"
	"os"
	"strconv"
)

var (
	DefCacheSize           int = 1024 * 1024 * 16
	logger                 *log.Logger
	DefDBFolderPermission  os.FileMode = 0755
	DEBUG                              = true
	ERR_KEY_DOES_NOT_EXIST             = "key does not exist"
)

type DB struct {
	LevigoDB *levigo.DB
	ro       *levigo.ReadOptions
	roIt     *levigo.ReadOptions
	wo       *levigo.WriteOptions
	cache    *levigo.Cache
}

type GoThroughProcessor interface {
	Process(k, v string) error
}

func Open(dbPath string, cacheSize int) (db *DB, err error) {
	db = new(DB)

	logger.Printf("Open(): dbPath = %v, cacheSize = %v", dbPath, cacheSize)

	if db.cache = levigo.NewLRUCache(cacheSize); db.cache == nil {
		err = errors.New("levigo.NewLRUCache() == nil")
		if DEBUG {
			logger.Println(err)
		}
		return nil, err
	}
	opts := levigo.NewOptions()
	opts.SetCache(db.cache)
	opts.SetCreateIfMissing(true)

	if err = os.MkdirAll(dbPath, DefDBFolderPermission); err != nil {
		logger.Printf("os.MkDirAll(%v, %v) err: %v", dbPath, DefDBFolderPermission, err)
		return nil, err
	}

	if db.LevigoDB, err = levigo.Open(dbPath, opts); err != nil {
		logger.Println(err)
		return nil, err
	}

	db.ro = levigo.NewReadOptions()
	db.roIt = levigo.NewReadOptions()
	db.roIt.SetFillCache(false)
	db.wo = levigo.NewWriteOptions()

	return db, err
}

// Close an ezdb.DB database.
func (db *DB) Close() {
	if db == nil {
		return
	}

	if db.roIt != nil {
		db.roIt.Close()
	}

	if db.ro != nil {
		db.ro.Close()
	}

	if db.wo != nil {
		db.wo.Close()
	}

	if db.LevigoDB != nil {
		db.LevigoDB.Close()
	}
	// delete cache AFTER delete db or it will hang.
	// See cache in http://leveldb.googlecode.com/svn/trunk/doc/index.html
	if db.cache != nil {
		db.cache.Close()
	}
}

func (db *DB) Put(key, value []byte) (err error) {
	return db.LevigoDB.Put(db.wo, key, value)
}

func (db *DB) Get(key []byte) (value []byte, err error) {
	return db.LevigoDB.Get(db.ro, key)
}

func (db *DB) Delete(key []byte) (err error) {
	return db.LevigoDB.Delete(db.wo, key)
}

func (db *DB) PutStr(key, value string) (err error) {
	return db.Put([]byte(key), []byte(value))
}

func (db *DB) KeyExist(key string) (exist bool, err error) {
	v, err := db.Get([]byte(key))
	if err != nil {
		return false, err
	}

	if v == nil {
		return false, nil
	} else {
		return true, nil
	}
}

func (db *DB) GetStr(key string) (value string, err error) {
	v, err := db.Get([]byte(key))
	if v == nil {
		return "", errors.New(ERR_KEY_DOES_NOT_EXIST)
	} else {
		return string(v), err
	}
}

func (db *DB) PutInt64(key string, value int64) (err error) {
	s := strconv.FormatInt(value, 10)
	return db.PutStr(key, s)
}

func (db *DB) GetInt64(key string) (value int64, err error) {
	s, err := db.GetStr(key)
	if err != nil {
		return 0, err
	}
	return strconv.ParseInt(s, 10, 64)
}

func (db *DB) PutUint64(key string, value uint64) (err error) {
	s := strconv.FormatUint(value, 10)
	return db.PutStr(key, s)
}

func (db *DB) GetUint64(key string) (value uint64, err error) {
	s, err := db.GetStr(key)
	if err != nil {
		return 0, err
	}
	return strconv.ParseUint(s, 10, 64)
}

func (db *DB) DeleteStr(key string) (err error) {
	return db.Delete([]byte(key))
}

func (db *DB) NewIterator() *levigo.Iterator {
	return db.LevigoDB.NewIterator(db.roIt)
}

func IsIteratorValidForGoThrough(it *levigo.Iterator, keyEnd string) bool {
	if keyEnd != "" {
		return it.Valid() && string(it.Key()) <= keyEnd
	} else {
		return it.Valid()
	}
}

func (db *DB) GoThrough(keyStart, keyEnd string, processor GoThroughProcessor) (err error) {
	it := db.NewIterator()
	defer it.Close()

	if keyStart != "" {
		it.Seek([]byte(keyStart))
	} else {
		it.SeekToFirst()
	}

	k := ""
	v := ""
	for ; IsIteratorValidForGoThrough(it, keyEnd); it.Next() {
		k = string(it.Key())
		v = string(it.Value())
		if err = processor.Process(k, v); err != nil {
			logger.Printf("processor.Process(%v, %v) err: %v", k, v, err)
			return err
		}
	}

	if err := it.GetError(); err != nil {
		logger.Printf("it.GetError(): %s\n", err)
		return err
	}

	return nil
}

func init() {
	logger = fnlog.New("")
}
<commit_msg>Add comments for PutInt64(), GetInt64(), PutUint64(), GetUint64()<commit_after>package ezdb

import (
	"errors"
	"github.com/jmhodges/levigo"
	"github.com/northbright/fnlog"
	"log"
	"os"
	"strconv"
)

var (
	DefCacheSize           int = 1024 * 1024 * 16
	logger                 *log.Logger
	DefDBFolderPermission  os.FileMode = 0755
	DEBUG                              = true
	ERR_KEY_DOES_NOT_EXIST             = "key does not exist"
)

type DB struct {
	LevigoDB *levigo.DB
	ro       *levigo.ReadOptions
	roIt     *levigo.ReadOptions
	wo       *levigo.WriteOptions
	cache    *levigo.Cache
}

type GoThroughProcessor interface {
	Process(k, v string) error
}

func Open(dbPath string, cacheSize int) (db *DB, err error) {
	db = new(DB)

	logger.Printf("Open(): dbPath = %v, cacheSize = %v", dbPath, cacheSize)

	if db.cache = levigo.NewLRUCache(cacheSize); db.cache == nil {
		err = errors.New("levigo.NewLRUCache() == nil")
		if DEBUG {
			logger.Println(err)
		}
		return nil, err
	}
	opts := levigo.NewOptions()
	opts.SetCache(db.cache)
	opts.SetCreateIfMissing(true)

	if err = os.MkdirAll(dbPath, DefDBFolderPermission); err != nil {
		logger.Printf("os.MkDirAll(%v, %v) err: %v", dbPath, DefDBFolderPermission, err)
		return nil, err
	}

	if db.LevigoDB, err = levigo.Open(dbPath, opts); err != nil {
		logger.Println(err)
		return nil, err
	}

	db.ro = levigo.NewReadOptions()
	db.roIt = levigo.NewReadOptions()
	db.roIt.SetFillCache(false)
	db.wo = levigo.NewWriteOptions()

	return db, err
}

// Close an ezdb.DB database.
func (db *DB) Close() {
	if db == nil {
		return
	}

	if db.roIt != nil {
		db.roIt.Close()
	}

	if db.ro != nil {
		db.ro.Close()
	}

	if db.wo != nil {
		db.wo.Close()
	}

	if db.LevigoDB != nil {
		db.LevigoDB.Close()
	}
	// delete cache AFTER delete db or it will hang.
	// See cache in http://leveldb.googlecode.com/svn/trunk/doc/index.html
	if db.cache != nil {
		db.cache.Close()
	}
}

func (db *DB) Put(key, value []byte) (err error) {
	return db.LevigoDB.Put(db.wo, key, value)
}

func (db *DB) Get(key []byte) (value []byte, err error) {
	return db.LevigoDB.Get(db.ro, key)
}

func (db *DB) Delete(key []byte) (err error) {
	return db.LevigoDB.Delete(db.wo, key)
}

func (db *DB) PutStr(key, value string) (err error) {
	return db.Put([]byte(key), []byte(value))
}

func (db *DB) KeyExist(key string) (exist bool, err error) {
	v, err := db.Get([]byte(key))
	if err != nil {
		return false, err
	}

	if v == nil {
		return false, nil
	} else {
		return true, nil
	}
}

func (db *DB) GetStr(key string) (value string, err error) {
	v, err := db.Get([]byte(key))
	if v == nil {
		return "", errors.New(ERR_KEY_DOES_NOT_EXIST)
	} else {
		return string(v), err
	}
}

// Store int64 as string in db.
// Should be used with Getint64().
func (db *DB) PutInt64(key string, value int64) (err error) {
	s := strconv.FormatInt(value, 10)
	return db.PutStr(key, s)
}

// Get string value and convert it to int64.
// Should be used with PutInt64().
func (db *DB) GetInt64(key string) (value int64, err error) {
	s, err := db.GetStr(key)
	if err != nil {
		return 0, err
	}
	return strconv.ParseInt(s, 10, 64)
}

// Store uint64 as string in db.
// Should be used with GetUint64().
func (db *DB) PutUint64(key string, value uint64) (err error) {
	s := strconv.FormatUint(value, 10)
	return db.PutStr(key, s)
}

// Get string value and convert it to uint64.
// Should be used with PutUInt64().
func (db *DB) GetUint64(key string) (value uint64, err error) {
	s, err := db.GetStr(key)
	if err != nil {
		return 0, err
	}
	return strconv.ParseUint(s, 10, 64)
}

func (db *DB) DeleteStr(key string) (err error) {
	return db.Delete([]byte(key))
}

func (db *DB) NewIterator() *levigo.Iterator {
	return db.LevigoDB.NewIterator(db.roIt)
}

func IsIteratorValidForGoThrough(it *levigo.Iterator, keyEnd string) bool {
	if keyEnd != "" {
		return it.Valid() && string(it.Key()) <= keyEnd
	} else {
		return it.Valid()
	}
}

func (db *DB) GoThrough(keyStart, keyEnd string, processor GoThroughProcessor) (err error) {
	it := db.NewIterator()
	defer it.Close()

	if keyStart != "" {
		it.Seek([]byte(keyStart))
	} else {
		it.SeekToFirst()
	}

	k := ""
	v := ""
	for ; IsIteratorValidForGoThrough(it, keyEnd); it.Next() {
		k = string(it.Key())
		v = string(it.Value())
		if err = processor.Process(k, v); err != nil {
			logger.Printf("processor.Process(%v, %v) err: %v", k, v, err)
			return err
		}
	}

	if err := it.GetError(); err != nil {
		logger.Printf("it.GetError(): %s\n", err)
		return err
	}

	return nil
}

func init() {
	logger = fnlog.New("")
}
<|endoftext|>


<commit_before>// Package magicsql is a wrapper around a database handle with added magic
// to ease common SQL operations, and reflection for automatic reading and
// writing of data
package magicsql

import (
	"database/sql"
	"fmt"
	"reflect"
	"sync"
)

// DB wraps an sql.DB, providing the Operation spawner for deferred-error
// database operations.  Like sql.DB, this DB type is meant to live more or
// less globally and be a long-living object.
type DB struct {
	db      *sql.DB
	typemap map[reflect.Type]*magicTable
	namemap map[string]*magicTable
	m       sync.RWMutex
}

// Open attempts to connect to a database, wrapping the sql.Open call,
// returning the new magicsql.DB and error if any.  This isn't storing the
// error for later, as there's nothing which can happen if the database can't
// be opened.
func Open(driverName, dataSourceName string) (*DB, error) {
	var sqldb, err = sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return Wrap(sqldb), nil
}

// Wrap is used to create a new DB from an existing connection
func Wrap(db *sql.DB) *DB {
	return &DB{
		db:      db,
		typemap: make(map[reflect.Type]*magicTable),
		namemap: make(map[string]*magicTable),
	}
}

// DataSource returns the underlying sql.DB pointer so the caller can do
// lower-level work which isn't wrapped in this package
func (db *DB) DataSource() *sql.DB {
	return db.db
}

// RegisterTable registers a table structure using NewMagicTable, then stores
// the table for lookup in an Operation's helper functions
//
// The structure returned by the generator must have tags for explicit table
// names, or else a lowercased version of the field name will be inferred.  Tag
// names must be in the form `sql:"field_name"`.  A field name of "-" tells the
// package to skip that field.  Non-exported fields are skipped.
func (db *DB) RegisterTable(tableName string, generator func() interface{}) *magicTable {
	var t = NewMagicTable(tableName, generator)

	db.m.Lock()
	defer db.m.Unlock()
	db.typemap[t.RType] = t
	db.namemap[tableName] = t

	return t
}

// findTableByName looks up the table by its name, for use in creating SQL queries
// and putting data into structures
func (db *DB) findTableByName(tableName string) *magicTable {
	db.m.RLock()
	defer db.m.RUnlock()

	return db.namemap[tableName]
}

// Operation returns an Operation instance, suitable for a short-lived task.
// This is the entry point for any of the sql wrapped magic.  An Operation
// should be considered a short-lived object which is not safe for concurrent
// access since it needs to be able to halt on any error with any operation it
// performs.  Concurrent access could be extremely confusing in this context
// due to the possibility of Operation.Err() returning an error from a
// different goroutine than the one doing the checking.
func (db *DB) Operation() *Operation {
	return &Operation{parent: db, db: db.db}
}

// Operation represents a short-lived single-purpose combination of database
// calls.  On the first failure, its internal error is set, which all
// "children" (statements, transactions, etc) will see.  All children will
// refuse to perform any functions once an error has occurred, making it safe
// to perform a chain of related database calls and only check for an error
// when it makes sense.
type Operation struct {
	parent *DB
	db     *sql.DB
	err    error
}

type errorable interface {
	Err() error
	SetErr(error)
}

// Err returns the *first* error which occurred on any database call owned by the Operation
func (op *Operation) Err() error {
	return op.err
}

// SetErr tells the Operation to stop handling any more queries.  It shouldn't
// usually be called directly, but it can be if you need to tell the object
// "here's a thing that may be an error; don't do any more work if it is".
func (op *Operation) SetErr(err error) {
	if op.Err() != nil {
		return
	}

	op.err = err
}

// Query wraps sql's Query, returning a wrapped Rows object
func (op *Operation) Query(query string, args ...interface{}) *Rows {
	if op.Err() != nil {
		return &Rows{nil, op}
	}

	var r, err = op.db.Query(query, args...)
	op.SetErr(err)
	return &Rows{r, op}
}

// Exec wraps sql's DB.Exec, returning a wrapped Result
func (op *Operation) Exec(query string, args ...interface{}) *Result {
	if op.Err() != nil {
		return &Result{nil, op}
	}

	var r, err = op.db.Exec(query, args...)
	op.SetErr(err)
	return &Result{r, op}
}

// Prepare wrap's sql's DB.Prepare, returning a wrapped Stmt
func (op *Operation) Prepare(query string) *Stmt {
	if op.Err() != nil {
		return &Stmt{nil, op}
	}

	var st, err = op.db.Prepare(query)
	op.SetErr(err)
	return &Stmt{st, op}
}

// Begin wraps sql's Begin and returns a wrapped Tx.  When the transaction is
// complete, instead of manually rolling back or committing, simply call
// tx.Done() and it will rollback / commit based on the error state.  If you
// need to force a rollback, set an error manually with Operation.SetErr().
func (op *Operation) Begin() *Tx {
	if op.Err() != nil {
		return &Tx{nil, op}
	}

	var tx, err = op.db.Begin()
	op.SetErr(err)
	return &Tx{tx, op}
}

// From starts a scoped SQL chain for firing off a SELECT against the given
// table, allowing things like this:
//
//     operation.From("people").Where("city = ?", varCity).SelectAllInto(peopleSlice)
//     var rows = operation.From("people").Limit(10).SelectAllRows()
//
// A scope can be passed around, but right now its capabilities are extremely
// limited, and I don't plan to make this particularly impressive.
func (op *Operation) From(tableName string) *Select {
	var emptySelect = &Select{parent: op}
	if op.Err() != nil {
		return emptySelect
	}

	var t = op.parent.findTableByName(tableName)
	if t == nil {
		op.SetErr(fmt.Errorf("table %s not registered", tableName))
		return emptySelect
	}

	return newSelect(op, t)
}
<commit_msg>REALLY un-expose magicTable<commit_after>// Package magicsql is a wrapper around a database handle with added magic
// to ease common SQL operations, and reflection for automatic reading and
// writing of data
package magicsql

import (
	"database/sql"
	"fmt"
	"reflect"
	"sync"
)

// DB wraps an sql.DB, providing the Operation spawner for deferred-error
// database operations.  Like sql.DB, this DB type is meant to live more or
// less globally and be a long-living object.
type DB struct {
	db      *sql.DB
	typemap map[reflect.Type]*magicTable
	namemap map[string]*magicTable
	m       sync.RWMutex
}

// Open attempts to connect to a database, wrapping the sql.Open call,
// returning the new magicsql.DB and error if any.  This isn't storing the
// error for later, as there's nothing which can happen if the database can't
// be opened.
func Open(driverName, dataSourceName string) (*DB, error) {
	var sqldb, err = sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return Wrap(sqldb), nil
}

// Wrap is used to create a new DB from an existing connection
func Wrap(db *sql.DB) *DB {
	return &DB{
		db:      db,
		typemap: make(map[reflect.Type]*magicTable),
		namemap: make(map[string]*magicTable),
	}
}

// DataSource returns the underlying sql.DB pointer so the caller can do
// lower-level work which isn't wrapped in this package
func (db *DB) DataSource() *sql.DB {
	return db.db
}

// RegisterTable registers a table structure using NewMagicTable, then stores
// the table for lookup in an Operation's helper functions
//
// The structure returned by the generator must have tags for explicit table
// names, or else a lowercased version of the field name will be inferred.  Tag
// names must be in the form `sql:"field_name"`.  A field name of "-" tells the
// package to skip that field.  Non-exported fields are skipped.
func (db *DB) RegisterTable(tableName string, generator func() interface{}) {
	var t = NewMagicTable(tableName, generator)

	db.m.Lock()
	defer db.m.Unlock()
	db.typemap[t.RType] = t
	db.namemap[tableName] = t
}

// findTableByName looks up the table by its name, for use in creating SQL queries
// and putting data into structures
func (db *DB) findTableByName(tableName string) *magicTable {
	db.m.RLock()
	defer db.m.RUnlock()

	return db.namemap[tableName]
}

// Operation returns an Operation instance, suitable for a short-lived task.
// This is the entry point for any of the sql wrapped magic.  An Operation
// should be considered a short-lived object which is not safe for concurrent
// access since it needs to be able to halt on any error with any operation it
// performs.  Concurrent access could be extremely confusing in this context
// due to the possibility of Operation.Err() returning an error from a
// different goroutine than the one doing the checking.
func (db *DB) Operation() *Operation {
	return &Operation{parent: db, db: db.db}
}

// Operation represents a short-lived single-purpose combination of database
// calls.  On the first failure, its internal error is set, which all
// "children" (statements, transactions, etc) will see.  All children will
// refuse to perform any functions once an error has occurred, making it safe
// to perform a chain of related database calls and only check for an error
// when it makes sense.
type Operation struct {
	parent *DB
	db     *sql.DB
	err    error
}

type errorable interface {
	Err() error
	SetErr(error)
}

// Err returns the *first* error which occurred on any database call owned by the Operation
func (op *Operation) Err() error {
	return op.err
}

// SetErr tells the Operation to stop handling any more queries.  It shouldn't
// usually be called directly, but it can be if you need to tell the object
// "here's a thing that may be an error; don't do any more work if it is".
func (op *Operation) SetErr(err error) {
	if op.Err() != nil {
		return
	}

	op.err = err
}

// Query wraps sql's Query, returning a wrapped Rows object
func (op *Operation) Query(query string, args ...interface{}) *Rows {
	if op.Err() != nil {
		return &Rows{nil, op}
	}

	var r, err = op.db.Query(query, args...)
	op.SetErr(err)
	return &Rows{r, op}
}

// Exec wraps sql's DB.Exec, returning a wrapped Result
func (op *Operation) Exec(query string, args ...interface{}) *Result {
	if op.Err() != nil {
		return &Result{nil, op}
	}

	var r, err = op.db.Exec(query, args...)
	op.SetErr(err)
	return &Result{r, op}
}

// Prepare wrap's sql's DB.Prepare, returning a wrapped Stmt
func (op *Operation) Prepare(query string) *Stmt {
	if op.Err() != nil {
		return &Stmt{nil, op}
	}

	var st, err = op.db.Prepare(query)
	op.SetErr(err)
	return &Stmt{st, op}
}

// Begin wraps sql's Begin and returns a wrapped Tx.  When the transaction is
// complete, instead of manually rolling back or committing, simply call
// tx.Done() and it will rollback / commit based on the error state.  If you
// need to force a rollback, set an error manually with Operation.SetErr().
func (op *Operation) Begin() *Tx {
	if op.Err() != nil {
		return &Tx{nil, op}
	}

	var tx, err = op.db.Begin()
	op.SetErr(err)
	return &Tx{tx, op}
}

// From starts a scoped SQL chain for firing off a SELECT against the given
// table, allowing things like this:
//
//     operation.From("people").Where("city = ?", varCity).SelectAllInto(peopleSlice)
//     var rows = operation.From("people").Limit(10).SelectAllRows()
//
// A scope can be passed around, but right now its capabilities are extremely
// limited, and I don't plan to make this particularly impressive.
func (op *Operation) From(tableName string) *Select {
	var emptySelect = &Select{parent: op}
	if op.Err() != nil {
		return emptySelect
	}

	var t = op.parent.findTableByName(tableName)
	if t == nil {
		op.SetErr(fmt.Errorf("table %s not registered", tableName))
		return emptySelect
	}

	return newSelect(op, t)
}
<|endoftext|>


<commit_before>package tracer

import (
	"math"
)

type Primitive struct {
	Intersectable
	material Material
}

func (p Primitive) Material() Material {
	return p.material
}

type Plane struct {
	Primitive
	Normal Vector
	D      float64
}

func NewPlane(normal Vector, d float64, material Material) *Plane {
	p := Plane{
		Normal: normal.Normalize(),
		D:      d,
	}
	p.material = material
	return &p
}

func (p *Plane) Intersect(ray Ray) (intersected bool, t float64, n Vector) {
	t = (ray.Origin.Dot(p.Normal) + p.D) / ray.Direction.Dot(p.Normal)
	return t >= 0, t, p.Normal
}

type Triangle struct {
	Primitive
	P1, P2, P3 Vector
	Normal     Vector
}

func NewTriangle(p1, p2, p3 Vector, material Material) *Triangle {
	e1 := p2.Subtract(p1)
	e2 := p3.Subtract(p1)
	n := e1.Cross(e2).Normalize()

	t := Triangle{P1: p1, P2: p2, P3: p3, Normal: n}
	t.material = material
	return &t
}

const epsilon = 1e-8

//Intersect Intersects a ray with a triangle using the Möller–Trumbore algorithm
func (tr *Triangle) Intersect(ray Ray) (intersected bool, t float64, n Vector) {
	n = tr.Normal
	// if ray.Direction.Dot(n) < 0 {
	// 	n = n.Multiply(-1)
	// }

	t = 0
	e1 := tr.P2.Subtract(tr.P1)
	e2 := tr.P3.Subtract(tr.P1)

	pvec := ray.Direction.Cross(e2)
	det := e1.Dot(pvec)

	//ray is parallel to plane
	if det < epsilon && det > -epsilon {
		return false, t, n
	}

	invDet := 1 / det
	tvec := ray.Origin.Subtract(tr.P1)
	u := tvec.Dot(pvec) * invDet
	if u < 0 || u > 1 {
		return false, t, n
	}

	qvec := tvec.Cross(e1)
	v := ray.Direction.Dot(qvec) * invDet
	if v < 0 || u+v > 1 {
		return false, t, n
	}

	t = e2.Dot(qvec) * invDet
	return true, t, n
}

type Quad struct {
	Primitive
	P1, P2, P3, P4 Vector
	Normal         Vector
}

func NewQuad(p1, p2, p3, p4 Vector, material Material) *Quad {
	e1 := p2.Subtract(p1)
	e2 := p3.Subtract(p1)
	n := e1.Cross(e2).Normalize()
	q := Quad{P1: p1, P2: p2, P3: p3, P4: p4, Normal: n}
	q.material = material
	return &q
}

//todo don't use triangles
func (q *Quad) Intersect(ray Ray) (intersected bool, t float64, n Vector) {
	t1 := Triangle{P1: q.P1, P2: q.P2, P3: q.P4}
	t2 := Triangle{P1: q.P2, P2: q.P3, P3: q.P4}

	if intersected, t, n := t1.Intersect(ray); intersected {
		return intersected, t, n
	}

	if intersected, t, n := t2.Intersect(ray); intersected {
		return intersected, t, n
	}

	return false, t, n
}

type Sphere struct {
	Primitive
	Center       Vector
	Radius, rad2 float64
}

func NewSphere(center Vector, radius float64, material Material) *Sphere {
	s := Sphere{Center: center, Radius: radius, rad2: radius * radius}
	s.material = material
	return &s
}

func (s *Sphere) Intersect(ray Ray) (intersected bool, t float64, n Vector) {

	// m := ray.Origin.Subtract(s.Center)
	// b := m.Dot(ray.Direction)
	// c := m.Dot(m) - s.rad2

	// if c > 0 && b > 0 {
	// 	return false, t, n
	// }

	// discr := b*b - c

	// if discr < 0 {
	// 	return false, t, n
	// }

	// t = -b - math.Sqrt(discr)
	// if t < 0 {
	// 	t = 0
	// }
	// n = ray.Point(t).Subtract(s.Center).Normalize()
	// return true, t, n

	C := ray.Origin.Subtract(s.Center)

	a := ray.Direction.Dot(ray.Direction)
	b := ray.Direction.Dot(C) * 2
	c := C.Dot(C) - s.rad2
	d := b*b - 4*a*c

	if d < 0 {
		return false, t, n
	}

	sd := math.Sqrt(d)
	t1 := (-b + sd) / (2 * a)
	t2 := (-b - sd) / (2 * a)
	t = math.Max(t1, t2)
	n = ray.Point(t1).Subtract(s.Center).Normalize()

	return t > 0, t, n
	// var a = Vector3.Dot(ray.Direction, ray.Direction);
	//                 var b = Vector3.Dot(2*ray.Direction, C);
	//                 var c = dotC - _rad2;
	//                 var d = b*b - 4*a*c;

	//                 //no intersection
	//                 if (d < 0)
	//                 {
	//                     return false;
	//                 }

	//                 var sd = (float)Math.Sqrt(d);
	//                 var t1 = (-b + sd)/2*a;
	//                 var t2 = (-b - sd)/2*a;
	//                 t = Math.Max(t1, t2);
}

// //fast way to calculate
// t = Vector3.Dot(-C, ray.Direction);
// var q = -C - t*ray.Direction;
// float p2 = Vector3.Dot(q, q);
// if (p2 > _rad2) return false;
// t -= (float) Math.Sqrt(_rad2 - p2);
<commit_msg>Fixed sphere bug...<commit_after>package tracer

import (
	"math"
)

type Primitive struct {
	Intersectable
	material Material
}

func (p Primitive) Material() Material {
	return p.material
}

type Plane struct {
	Primitive
	Normal Vector
	D      float64
}

func NewPlane(normal Vector, d float64, material Material) *Plane {
	p := Plane{
		Normal: normal.Normalize(),
		D:      d,
	}
	p.material = material
	return &p
}

func (p *Plane) Intersect(ray Ray) (intersected bool, t float64, n Vector) {
	t = (ray.Origin.Dot(p.Normal) + p.D) / ray.Direction.Dot(p.Normal)
	return t >= 0, t, p.Normal
}

type Triangle struct {
	Primitive
	P1, P2, P3 Vector
	Normal     Vector
}

func NewTriangle(p1, p2, p3 Vector, material Material) *Triangle {
	e1 := p2.Subtract(p1)
	e2 := p3.Subtract(p1)
	n := e1.Cross(e2).Normalize()

	t := Triangle{P1: p1, P2: p2, P3: p3, Normal: n}
	t.material = material
	return &t
}

const epsilon = 1e-8

//Intersect Intersects a ray with a triangle using the Möller–Trumbore algorithm
func (tr *Triangle) Intersect(ray Ray) (intersected bool, t float64, n Vector) {
	n = tr.Normal
	// if ray.Direction.Dot(n) < 0 {
	// 	n = n.Multiply(-1)
	// }

	t = 0
	e1 := tr.P2.Subtract(tr.P1)
	e2 := tr.P3.Subtract(tr.P1)

	pvec := ray.Direction.Cross(e2)
	det := e1.Dot(pvec)

	//ray is parallel to plane
	if det < epsilon && det > -epsilon {
		return false, t, n
	}

	invDet := 1 / det
	tvec := ray.Origin.Subtract(tr.P1)
	u := tvec.Dot(pvec) * invDet
	if u < 0 || u > 1 {
		return false, t, n
	}

	qvec := tvec.Cross(e1)
	v := ray.Direction.Dot(qvec) * invDet
	if v < 0 || u+v > 1 {
		return false, t, n
	}

	t = e2.Dot(qvec) * invDet
	return t > 0, t, n
}

type Quad struct {
	Primitive
	P1, P2, P3, P4 Vector
	Normal         Vector
}

func NewQuad(p1, p2, p3, p4 Vector, material Material) *Quad {
	e1 := p2.Subtract(p1)
	e2 := p3.Subtract(p1)
	n := e1.Cross(e2).Normalize()
	q := Quad{P1: p1, P2: p2, P3: p3, P4: p4, Normal: n}
	q.material = material
	return &q
}

//todo don't use triangles
func (q *Quad) Intersect(ray Ray) (intersected bool, t float64, n Vector) {
	t1 := Triangle{P1: q.P1, P2: q.P2, P3: q.P4}
	t2 := Triangle{P1: q.P2, P2: q.P3, P3: q.P4}

	if intersected, t, n := t1.Intersect(ray); intersected {
		return intersected, t, n
	}

	if intersected, t, n := t2.Intersect(ray); intersected {
		return intersected, t, n
	}

	return false, t, n
}

type Sphere struct {
	Primitive
	Center       Vector
	Radius, rad2 float64
}

func NewSphere(center Vector, radius float64, material Material) *Sphere {
	s := Sphere{Center: center, Radius: radius, rad2: radius * radius}
	s.material = material
	return &s
}

func (s *Sphere) Intersect(ray Ray) (intersected bool, t float64, n Vector) {

	// m := ray.Origin.Subtract(s.Center)
	// b := m.Dot(ray.Direction)
	// c := m.Dot(m) - s.rad2

	// if c > 0 && b > 0 {
	// 	return false, t, n
	// }

	// discr := b*b - c

	// if discr < 0 {
	// 	return false, t, n
	// }

	// t = -b - math.Sqrt(discr)
	// if t < 0 {
	// 	t = 0
	// }
	// n = ray.Point(t).Subtract(s.Center).Normalize()
	// return true, t, n

	C := ray.Origin.Subtract(s.Center)

	a := ray.Direction.Dot(ray.Direction)
	b := ray.Direction.Dot(C) * 2
	c := C.Dot(C) - s.rad2
	d := b*b - 4*a*c

	if d < 0 {
		return false, t, n
	}

	sd := math.Sqrt(d)
	t1 := (-b + sd) / (2 * a)
	t2 := (-b - sd) / (2 * a)
	t = math.Max(t1, t2)
	n = ray.Point(t1).Subtract(s.Center).Normalize()

	return t > 0, t, n
	// var a = Vector3.Dot(ray.Direction, ray.Direction);
	//                 var b = Vector3.Dot(2*ray.Direction, C);
	//                 var c = dotC - _rad2;
	//                 var d = b*b - 4*a*c;

	//                 //no intersection
	//                 if (d < 0)
	//                 {
	//                     return false;
	//                 }

	//                 var sd = (float)Math.Sqrt(d);
	//                 var t1 = (-b + sd)/2*a;
	//                 var t2 = (-b - sd)/2*a;
	//                 t = Math.Max(t1, t2);
}

// //fast way to calculate
// t = Vector3.Dot(-C, ray.Direction);
// var q = -C - t*ray.Direction;
// float p2 = Vector3.Dot(q, q);
// if (p2 > _rad2) return false;
// t -= (float) Math.Sqrt(_rad2 - p2);
<|endoftext|>


<commit_before>package tradier

import "encoding/json"

// Series represents the series JSON object from /markets/timesales.
type Series struct {
	Data []*Data `json:"data,omitempty"`
}

type series Series

// Data represents the data JSON object from /markets/timesales.
type Data struct {
	Price     *float64 `json:"price,omitempty"`
	Time      *Time    `json:"time,omitempty"`
	Timestamp *int     `json:"timestamp,omitempty"`
	Volume    *int     `json:"volume,omitempty"`
}

type data Data

// UnmarshalJSON unmarshals series into Series object.
func (s *Series) UnmarshalJSON(b []byte) error {
	var seriesCol struct {
		// S *Series `json:"series,omitempty"`
		S struct {
			D []*Data `json:"data,omitempty"`
		} `json:"series,omitempty"`
	}
	var seriesObj struct {
		S struct {
			D *Data `json:"data,omitempty"`
		} `json:"seris,omitempty"`
	}

	var seriesStr struct {
		S string `json:"series,omitempty"`
	}

	var err error

	if err = json.Unmarshal(b, &seriesCol); err == nil {
		*s = Series{
			Data: seriesCol.S.D,
		}
		return nil
	}

	if err = json.Unmarshal(b, &seriesObj); err == nil {
		*s = Series{
			Data: []*Data{
				seriesObj.S.D,
			},
		}
		return nil
	}

	// If positions is null
	if err = json.Unmarshal(b, &seriesStr); err == nil {
		return nil
	}

	return err
}

// MarshalJSON marshals Series into its JSON representation.
func (s *Series) MarshalJSON() ([]byte, error) {
	// If []Event is empty
	if len(s.Data) == 0 {
		return json.Marshal("null")
	}

	// If []Event is size 1, return first and only object
	if len(s.Data) == 1 {
		return json.Marshal(map[string]interface{}{
			"data": s.Data[0],
		})
	}

	// Otherwise mashal entire History object normally
	return json.Marshal(*s)
}
<commit_msg>Fix timesales JSON marshal<commit_after>package tradier

import "encoding/json"

// Series represents the series JSON object from /markets/timesales.
type Series struct {
	Data []*Data `json:"data,omitempty"`
}

type series Series

// Data represents the data JSON object from /markets/timesales.
type Data struct {
	Price     *float64 `json:"price,omitempty"`
	Time      *Time    `json:"time,omitempty"`
	Timestamp *int     `json:"timestamp,omitempty"`
	Volume    *int     `json:"volume,omitempty"`
}

type data Data

// UnmarshalJSON unmarshals series into Series object.
func (s *Series) UnmarshalJSON(b []byte) error {
	var seriesCol struct {
		// S *Series `json:"series,omitempty"`
		S struct {
			D []*Data `json:"data,omitempty"`
		} `json:"series,omitempty"`
	}
	var seriesObj struct {
		S struct {
			D *Data `json:"data,omitempty"`
		} `json:"seris,omitempty"`
	}

	var seriesStr struct {
		S string `json:"series,omitempty"`
	}

	var err error

	if err = json.Unmarshal(b, &seriesCol); err == nil {
		*s = Series{
			Data: seriesCol.S.D,
		}
		return nil
	}

	if err = json.Unmarshal(b, &seriesObj); err == nil {
		*s = Series{
			Data: []*Data{
				seriesObj.S.D,
			},
		}
		return nil
	}

	if err = json.Unmarshal(b, &seriesStr); err == nil {
		return nil
	}

	return err
}

// MarshalJSON marshals Series into its JSON representation.
func (s *Series) MarshalJSON() ([]byte, error) {
	if len(s.Data) == 0 {
		return json.Marshal(map[string]interface{}{
			"series": "null",
		})
	}

	if len(s.Data) == 1 {
		return json.Marshal(map[string]interface{}{
			"series": map[string]interface{}{
				"data": *(s.Data)[0],
			},
		})
	}

	return json.Marshal(map[string]interface{}{
		"series": map[string]interface{}{
			"data": s.Data,
		},
	})
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0
// Details: https://raw.githubusercontent.com/maniksurtani/quotaservice/master/LICENSE

// Package implements configs for the quotaservice
package quotaservice

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/maniksurtani/quotaservice/logging"
	"gopkg.in/yaml.v2"
)

type ServiceConfig struct {
	GlobalDefaultBucket *BucketConfig               `yaml:"global_default_bucket,flow"`
	ListenerBufferSize  int                         `yaml:"listener_buffer_size,flow"`
	Namespaces          map[string]*NamespaceConfig `yaml:",flow"`
}

func (s *ServiceConfig) String() string {
	return fmt.Sprintf("ServiceConfig{default: %v, namespaces: %v}",
		s.GlobalDefaultBucket, s.Namespaces)
}

func (s *ServiceConfig) AddNamespace(namespace string, n *NamespaceConfig) *ServiceConfig {
	s.Namespaces[namespace] = n
	return s
}

func (s *ServiceConfig) applyDefaults() *ServiceConfig {
	if s.GlobalDefaultBucket != nil {
		s.GlobalDefaultBucket.applyDefaults()
	}

	for name, ns := range s.Namespaces {
		if ns.DefaultBucket != nil && ns.DynamicBucketTemplate != nil {
			panic(fmt.Sprintf("Namespace %v is not allowed to have a default bucket as well as allow dynamic buckets.", name))
		}

		// Ensure the namespace's bucket map exists.
		if ns.Buckets == nil {
			ns.Buckets = make(map[string]*BucketConfig)
		}

		if ns.DefaultBucket != nil {
			ns.DefaultBucket.applyDefaults()
		}

		if ns.DynamicBucketTemplate != nil {
			ns.DynamicBucketTemplate.applyDefaults()
		}

		for _, b := range ns.Buckets {
			b.applyDefaults()
		}
	}

	return s
}

type NamespaceConfig struct {
	DefaultBucket         *BucketConfig            `yaml:"default_bucket,flow"`
	DynamicBucketTemplate *BucketConfig            `yaml:"dynamic_bucket_template,flow"`
	MaxDynamicBuckets     int                      `yaml:"max_dynamic_buckets"`
	Buckets               map[string]*BucketConfig `yaml:",flow"`
}

func (n *NamespaceConfig) AddBucket(name string, b *BucketConfig) *NamespaceConfig {
	n.Buckets[name] = b
	return n
}

type BucketConfig struct {
	Size                int64
	FillRate            int64 `yaml:"fill_rate"`
	WaitTimeoutMillis   int64 `yaml:"wait_timeout_millis"`
	MaxIdleMillis       int64 `yaml:"max_idle_millis"`
	MaxDebtMillis       int64 `yaml:"max_debt_millis"`
	MaxTokensPerRequest int64 `yaml:"max_tokens_per_request"`
}

func (b *BucketConfig) String() string {
	return fmt.Sprint(*b)
}

func (b *BucketConfig) applyDefaults() *BucketConfig {
	if b.Size == 0 {
		b.Size = 100
	}

	if b.FillRate == 0 {
		b.FillRate = 50
	}

	if b.WaitTimeoutMillis == 0 {
		b.WaitTimeoutMillis = 1000
	}

	if b.MaxIdleMillis == 0 {
		b.MaxIdleMillis = -1
	}

	if b.MaxDebtMillis == 0 {
		b.MaxDebtMillis = 10000
	}

	if b.MaxTokensPerRequest == 0 {
		b.MaxTokensPerRequest = b.FillRate
	}

	return b
}

func ReadConfigFromFile(filename string) *ServiceConfig {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(fmt.Sprintf("Unable to open file %v. Error: %v", filename, err))
	}

	return readConfigFromBytes(bytes)
}

func ReadConfig(yamlStream io.Reader) *ServiceConfig {
	bytes, err := ioutil.ReadAll(yamlStream)
	if err != nil {
		panic(fmt.Sprintf("Unable to open reader. Error: %v", err))
	}

	return readConfigFromBytes(bytes)
}

func readConfigFromBytes(bytes []byte) *ServiceConfig {
	logging.Print(string(bytes))
	cfg := NewDefaultServiceConfig()
	cfg.GlobalDefaultBucket = nil
	yaml.Unmarshal(bytes, cfg)

	return cfg.applyDefaults()
}

func NewDefaultServiceConfig() *ServiceConfig {
	return &ServiceConfig{
		NewDefaultBucketConfig(),
		10000,
		make(map[string]*NamespaceConfig)}
}

func NewDefaultNamespaceConfig() *NamespaceConfig {
	return &NamespaceConfig{Buckets: make(map[string]*BucketConfig)}
}

func NewDefaultBucketConfig() *BucketConfig {
	return &BucketConfig{Size: 100, FillRate: 50, WaitTimeoutMillis: 1000, MaxIdleMillis: -1, MaxDebtMillis: 10000}
}
<commit_msg>Make ApplyDefaults() publicly accessible<commit_after>// Licensed under the Apache License, Version 2.0
// Details: https://raw.githubusercontent.com/maniksurtani/quotaservice/master/LICENSE

// Package implements configs for the quotaservice
package quotaservice

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/maniksurtani/quotaservice/logging"
	"gopkg.in/yaml.v2"
)

type ServiceConfig struct {
	GlobalDefaultBucket *BucketConfig               `yaml:"global_default_bucket,flow"`
	ListenerBufferSize  int                         `yaml:"listener_buffer_size,flow"`
	Namespaces          map[string]*NamespaceConfig `yaml:",flow"`
}

func (s *ServiceConfig) String() string {
	return fmt.Sprintf("ServiceConfig{default: %v, namespaces: %v}",
		s.GlobalDefaultBucket, s.Namespaces)
}

func (s *ServiceConfig) AddNamespace(namespace string, n *NamespaceConfig) *ServiceConfig {
	s.Namespaces[namespace] = n
	return s
}

func (s *ServiceConfig) ApplyDefaults() *ServiceConfig {
	if s.GlobalDefaultBucket != nil {
		s.GlobalDefaultBucket.ApplyDefaults()
	}

	for name, ns := range s.Namespaces {
		if ns.DefaultBucket != nil && ns.DynamicBucketTemplate != nil {
			panic(fmt.Sprintf("Namespace %v is not allowed to have a default bucket as well as allow dynamic buckets.", name))
		}

		// Ensure the namespace's bucket map exists.
		if ns.Buckets == nil {
			ns.Buckets = make(map[string]*BucketConfig)
		}

		if ns.DefaultBucket != nil {
			ns.DefaultBucket.ApplyDefaults()
		}

		if ns.DynamicBucketTemplate != nil {
			ns.DynamicBucketTemplate.ApplyDefaults()
		}

		for _, b := range ns.Buckets {
			b.ApplyDefaults()
		}
	}

	return s
}

type NamespaceConfig struct {
	DefaultBucket         *BucketConfig            `yaml:"default_bucket,flow"`
	DynamicBucketTemplate *BucketConfig            `yaml:"dynamic_bucket_template,flow"`
	MaxDynamicBuckets     int                      `yaml:"max_dynamic_buckets"`
	Buckets               map[string]*BucketConfig `yaml:",flow"`
}

func (n *NamespaceConfig) AddBucket(name string, b *BucketConfig) *NamespaceConfig {
	n.Buckets[name] = b
	return n
}

type BucketConfig struct {
	Size                int64
	FillRate            int64 `yaml:"fill_rate"`
	WaitTimeoutMillis   int64 `yaml:"wait_timeout_millis"`
	MaxIdleMillis       int64 `yaml:"max_idle_millis"`
	MaxDebtMillis       int64 `yaml:"max_debt_millis"`
	MaxTokensPerRequest int64 `yaml:"max_tokens_per_request"`
}

func (b *BucketConfig) String() string {
	return fmt.Sprint(*b)
}

func (b *BucketConfig) ApplyDefaults() *BucketConfig {
	if b.Size == 0 {
		b.Size = 100
	}

	if b.FillRate == 0 {
		b.FillRate = 50
	}

	if b.WaitTimeoutMillis == 0 {
		b.WaitTimeoutMillis = 1000
	}

	if b.MaxIdleMillis == 0 {
		b.MaxIdleMillis = -1
	}

	if b.MaxDebtMillis == 0 {
		b.MaxDebtMillis = 10000
	}

	if b.MaxTokensPerRequest == 0 {
		b.MaxTokensPerRequest = b.FillRate
	}

	return b
}

func ReadConfigFromFile(filename string) *ServiceConfig {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(fmt.Sprintf("Unable to open file %v. Error: %v", filename, err))
	}

	return readConfigFromBytes(bytes)
}

func ReadConfig(yamlStream io.Reader) *ServiceConfig {
	bytes, err := ioutil.ReadAll(yamlStream)
	if err != nil {
		panic(fmt.Sprintf("Unable to open reader. Error: %v", err))
	}

	return readConfigFromBytes(bytes)
}

func readConfigFromBytes(bytes []byte) *ServiceConfig {
	logging.Print(string(bytes))
	cfg := NewDefaultServiceConfig()
	cfg.GlobalDefaultBucket = nil
	yaml.Unmarshal(bytes, cfg)

	return cfg.ApplyDefaults()
}

func NewDefaultServiceConfig() *ServiceConfig {
	return &ServiceConfig{
		NewDefaultBucketConfig(),
		10000,
		make(map[string]*NamespaceConfig)}
}

func NewDefaultNamespaceConfig() *NamespaceConfig {
	return &NamespaceConfig{Buckets: make(map[string]*BucketConfig)}
}

func NewDefaultBucketConfig() *BucketConfig {
	return &BucketConfig{Size: 100, FillRate: 50, WaitTimeoutMillis: 1000, MaxIdleMillis: -1, MaxDebtMillis: 10000}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/spf13/cobra"
)

func connect(cmd *cobra.Command) (*http.Client, error) {
	url := cmd.Flag("registry").Value.String()

	// XXX construct client using TLS

	client := &http.Client{Transport: &http.Transport{}}

	resp, err := client.Get(url + "/v2/")
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusUnauthorized:
		// error text in resp.Body
		return nil, errors.New("take action based on WWW-Authenticate")
	case http.StatusNotFound:
		return nil, errors.New("registry does not support v2 API")
	case http.StatusOK:
		break
	default:
		return nil, fmt.Errorf("bad status (%s/v2/) %d", url, resp.StatusCode)
	}

	// consume body so connection can be reused
	ioutil.ReadAll(resp.Body)

	ver := resp.Header.Get("Docker-Distribution-API-Version")
	if ver != "registry/2.0" {
		return nil, errors.New("registry does not support v2 API")
	}

	return client, nil
}

type Decoder interface {
	Method() string
	SetHeaders(*http.Header)
	UnmarshalJSON([]byte) error
	ExtractHeaders(*http.Header)
}

func get(conn *http.Client, url string, dec Decoder) error {
	req, err := http.NewRequest(dec.Method(), url, nil)
	if err != nil {
		return fmt.Errorf("new request: %v", err)
	}

	resp, err := conn.Do(req)
	if err != nil {
		return fmt.Errorf("get: %v", err)
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusOK:
	case http.StatusAccepted:
	case http.StatusNotFound:
		fmt.Println("Content-Type: ", resp.Header.Get("Content-Type"))
		fmt.Println("Content-Length: ", resp.Header.Get("Contenet-Length"))
		if strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
			if resp.Header.Get("Contenet-Length") == "None" {
				return fmt.Errorf("bad status: %s", http.StatusText(resp.StatusCode))
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("readall: %v", err)
			}

			var rpy struct {
				Errors []struct {
					Code    string `json:"code"`
					Message string `json:"message"`
				} `json:"errors"`
			}

			if err := json.Unmarshal(body, &rpy); err != nil {
				return fmt.Errorf("unmarshal not found: %v", err)
			}

			if len(rpy.Errors) == 1 {
				return fmt.Errorf("not found: code \"%s\" message \"%s\"", rpy.Errors[0].Code, rpy.Errors[0].Message)
			}
		}
		fallthrough
	default:
		return fmt.Errorf("bad status: %s", http.StatusText(resp.StatusCode))
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("readall: %v", err)
	}

	dec.ExtractHeaders(&resp.Header)

	return dec.UnmarshalJSON(body)
}
<commit_msg>retrieve the top-level digest correctly<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/spf13/cobra"
)

func connect(cmd *cobra.Command) (*http.Client, error) {
	url := cmd.Flag("registry").Value.String()

	// XXX construct client using TLS

	client := &http.Client{Transport: &http.Transport{}}

	resp, err := client.Get(url + "/v2/")
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusUnauthorized:
		// error text in resp.Body
		return nil, errors.New("take action based on WWW-Authenticate")
	case http.StatusNotFound:
		return nil, errors.New("registry does not support v2 API")
	case http.StatusOK:
		break
	default:
		return nil, fmt.Errorf("bad status (%s/v2/) %d", url, resp.StatusCode)
	}

	// consume body so connection can be reused
	ioutil.ReadAll(resp.Body)

	ver := resp.Header.Get("Docker-Distribution-API-Version")
	if ver != "registry/2.0" {
		return nil, errors.New("registry does not support v2 API")
	}

	return client, nil
}

type Decoder interface {
	Method() string
	SetHeaders(*http.Header)
	UnmarshalJSON([]byte) error
	ExtractHeaders(*http.Header)
}

func get(conn *http.Client, url string, dec Decoder) error {
	req, err := http.NewRequest(dec.Method(), url, nil)
	if err != nil {
		return fmt.Errorf("new request: %v", err)
	}

	req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")

	resp, err := conn.Do(req)
	if err != nil {
		return fmt.Errorf("get: %v", err)
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusOK:
	case http.StatusAccepted:
	case http.StatusNotFound:
		fmt.Println("Content-Type: ", resp.Header.Get("Content-Type"))
		fmt.Println("Content-Length: ", resp.Header.Get("Contenet-Length"))
		if strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
			if resp.Header.Get("Contenet-Length") == "None" {
				return fmt.Errorf("bad status: %s", http.StatusText(resp.StatusCode))
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("readall: %v", err)
			}

			var rpy struct {
				Errors []struct {
					Code    string `json:"code"`
					Message string `json:"message"`
				} `json:"errors"`
			}

			if err := json.Unmarshal(body, &rpy); err != nil {
				return fmt.Errorf("unmarshal not found: %v", err)
			}

			if len(rpy.Errors) == 1 {
				return fmt.Errorf("not found: code \"%s\" message \"%s\"", rpy.Errors[0].Code, rpy.Errors[0].Message)
			}
		}
		fallthrough
	default:
		return fmt.Errorf("bad status: %s", http.StatusText(resp.StatusCode))
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("readall: %v", err)
	}

	dec.ExtractHeaders(&resp.Header)

	return dec.UnmarshalJSON(body)
}
<|endoftext|>