<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tfplan

import (
	"encoding/json"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/pkg/errors"
)

const maxChildModuleLevel = 10000

// jsonPlan structure used to parse Terraform 12 plan exported in json format by 'terraform show -json ./binary_plan.tfplan' command.
// https://www.terraform.io/docs/internals/json-format.html#plan-representation
type jsonPlan struct {
	ResourceChanges []ResourceChange `json:"resource_changes"`
}

type ResourceChange struct {
	Address      string `json:"address"`
	Mode         string `json:"mode"`
	Type         string `json:"type"`
	Name         string `json:"name"`
	ProviderName string `json:"provider_name"`
	Change       struct {
		// Valid actions values are:
		//    ["no-op"]
		//    ["create"]
		//    ["read"]
		//    ["update"]
		//    ["delete", "create"]
		//    ["create", "delete"]
		//    ["delete"]
		Actions []string               `json:"actions"`
		Before  map[string]interface{} `json:"before"`
		After   map[string]interface{} `json:"after"`
	} `json:"change"`
}

// isCreate returns true if the action on the resource is ["create"].
func (c *ResourceChange) isCreate() bool {
	return len(c.Change.Actions) == 1 && c.Change.Actions[0] == "create"
}

// compatibility shim until ResourceChange is expected by all callers.
func (c *ResourceChange) Kind() string {
	return c.Type
}

// ComposeTF12Resources is a thin wrapper around ReadResourceChanges as a compatibility shim.
func ComposeTF12Resources(data []byte, schemas map[string]*schema.Resource) ([]ResourceChange, error) {
	return ReadResourceChanges(data)
}

// ReadResourceChanges returns the list of resource changes from a json plan
func ReadResourceChanges(data []byte) ([]ResourceChange, error) {
	plan := jsonPlan{}
	err := json.Unmarshal(data, &plan)
	if err != nil {
		return nil, errors.Wrap(err, "reading JSON plan")
	}

	return plan.ResourceChanges, nil
}
<commit_msg>Removed maxChildModuleLevel<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tfplan

import (
	"encoding/json"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/pkg/errors"
)

// jsonPlan structure used to parse Terraform 12 plan exported in json format by 'terraform show -json ./binary_plan.tfplan' command.
// https://www.terraform.io/docs/internals/json-format.html#plan-representation
type jsonPlan struct {
	ResourceChanges []ResourceChange `json:"resource_changes"`
}

type ResourceChange struct {
	Address      string `json:"address"`
	Mode         string `json:"mode"`
	Type         string `json:"type"`
	Name         string `json:"name"`
	ProviderName string `json:"provider_name"`
	Change       struct {
		// Valid actions values are:
		//    ["no-op"]
		//    ["create"]
		//    ["read"]
		//    ["update"]
		//    ["delete", "create"]
		//    ["create", "delete"]
		//    ["delete"]
		Actions []string               `json:"actions"`
		Before  map[string]interface{} `json:"before"`
		After   map[string]interface{} `json:"after"`
	} `json:"change"`
}

// isCreate returns true if the action on the resource is ["create"].
func (c *ResourceChange) isCreate() bool {
	return len(c.Change.Actions) == 1 && c.Change.Actions[0] == "create"
}

// compatibility shim until ResourceChange is expected by all callers.
func (c *ResourceChange) Kind() string {
	return c.Type
}

// ComposeTF12Resources is a thin wrapper around ReadResourceChanges as a compatibility shim.
func ComposeTF12Resources(data []byte, schemas map[string]*schema.Resource) ([]ResourceChange, error) {
	return ReadResourceChanges(data)
}

// ReadResourceChanges returns the list of resource changes from a json plan
func ReadResourceChanges(data []byte) ([]ResourceChange, error) {
	plan := jsonPlan{}
	err := json.Unmarshal(data, &plan)
	if err != nil {
		return nil, errors.Wrap(err, "reading JSON plan")
	}

	return plan.ResourceChanges, nil
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"time"

	"github.com/go-sql-driver/mysql"
)

/*
	These are more functions for querying the "users" table,
	but these functions are only used for the website
*/

/*
	Data structures
*/

type StatsSeeded struct {
	TrueSkill float32
	Sigma     float32
	NumRaces  int
	LastRace  mysql.NullTime
}

type StatsUnseeded struct {
	AdjustedAverage int
	RealAverage     int
	NumRaces        int
	NumForfeits     int
	ForfeitPenalty  int
	LowestTime      int
	LastRace        mysql.NullTime
}

type StatsDiversity struct {
	TrueSkill    float64
	Sigma        float64
	Change       float64
	NumRaces     int
	LastRace     mysql.NullTime
	NewTrueSkill float64 // Only used when doing new TrueSkill calculation
}

// ProfilesRow gets each row for all profiles
type ProfilesRow struct {
	Username        string
	DatetimeCreated time.Time
	StreamURL       string
	NumAchievements int
	TotalRaces      int
}

// ProfileData has all data for each racer
type ProfileData struct {
	Username          string
	DatetimeCreated   time.Time
	DatetimeLastLogin time.Time
	Admin             int
	Verified          bool
	StatsSeeded       StatsSeeded
	StatsUnseeded     StatsUnseeded
	StatsDiversity    StatsDiversity
	StreamURL         string
}

/*
type LeaderboardRowMostPlayed struct {
	Name     string
	Total    int
	Verified int
}
*/

/*
	Functions
*/

/*
func (*Users) GetStatsSeeded(username string) (StatsSeeded, error) {
	var stats StatsSeeded
	if err := db.QueryRow(`
		SELECT
			seeded_trueskill,
			seeded_trueskill_sigma,
			seeded_num_races,
			seeded_last_race
		FROM
			users
		WHERE
			username = ?
	`, username).Scan(
		&stats.ELO,
		&stats.NumSeededRaces,
		&stats.LastSeededRace,
	); err != nil {
		return stats, err
	} else {
		return stats, nil
	}
}

func (*Users) GetStatsUnseeded(username string) (StatsUnseeded, error) {
	var stats StatsUnseeded
	if err := db.QueryRow(`
		SELECT
			unseeded_adjusted_average,
			unseeded_real_average,
			num_unseeded_races,
			num_forfeits,
			forfeit_penalty,
			lowest_unseeded_time,
			last_unseeded_race
		FROM
			users
		WHERE
			username = ?
	`, username).Scan(
		&stats.UnseededAdjustedAverage,
		&stats.UnseededRealAverage,
		&stats.NumUnseededRaces,
		&stats.NumForfeits,
		&stats.ForfeitPenalty,
		&stats.LowestUnseededTime,
		&stats.LastUnseededRace,
	); err != nil {
		return stats, err
	} else {
		return stats, nil
	}
}
*/

// GetProfileData gets player data to populate the player's profile page
func (*Users) GetProfileData(username string) (ProfileData, error) {
	var profileData ProfileData
	var rawVerified int
	if err := db.QueryRow(`
		SELECT
			username,
			datetime_created,
			datetime_last_login,
			admin,
			verified,
			seeded_trueskill,
			seeded_trueskill_sigma,
			seeded_num_races,
			seeded_last_race,
			unseeded_adjusted_average,
			unseeded_real_average,
			unseeded_num_races,
			unseeded_num_forfeits,
			unseeded_forfeit_penalty,
			unseeded_lowest_time,
			unseeded_last_race,
			stream_url
		FROM
			users
		WHERE
			steam_id > 0 and
			username = ?
	`, username).Scan(
		&profileData.Username,
		&profileData.DatetimeCreated,
		&profileData.DatetimeLastLogin,
		&profileData.Admin,
		&rawVerified,
		&profileData.StatsSeeded.TrueSkill,
		&profileData.StatsSeeded.Sigma,
		&profileData.StatsSeeded.NumRaces,
		&profileData.StatsSeeded.LastRace,
		&profileData.StatsUnseeded.AdjustedAverage,
		&profileData.StatsUnseeded.RealAverage,
		&profileData.StatsUnseeded.NumRaces,
		&profileData.StatsUnseeded.NumForfeits,
		&profileData.StatsUnseeded.ForfeitPenalty,
		&profileData.StatsUnseeded.LowestTime,
		&profileData.StatsUnseeded.LastRace,
		&profileData.StreamURL,
	); err == sql.ErrNoRows {
		return profileData, nil
	} else if err != nil {
		return profileData, err
	} else {
		// Convert the int to a bool
		if rawVerified == 1 {
			profileData.Verified = true
		}
		return profileData, nil
	}
}

// GetUserProfiles gets players data to populate the profiles page
func (*Users) GetUserProfiles(currentPage int, usersPerPage int) ([]ProfilesRow, int, error) {
	usersOffset := (currentPage - 1) * usersPerPage
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
			u.username,
			u.datetime_created,
			u.stream_url,
			count(ua.achievement_id),
			(
				SELECT COUNT(id)
				FROM race_participants
				WHERE user_id = u.id
			) AS num_total_race
		FROM
			users u
		LEFT JOIN
			user_achievements ua
			ON
				u.id = ua.user_id
		WHERE
			u.steam_id > 0
		GROUP BY
			u.username
		ORDER BY
			u.username ASC
		LIMIT
			?
		OFFSET
			?
	`, usersPerPage, usersOffset); err == sql.ErrNoRows {
		return nil, 0, nil
	} else if err != nil {
		return nil, 0, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the user profile results
	profiles := make([]ProfilesRow, 0)
	for rows.Next() {
		var row ProfilesRow
		if err := rows.Scan(
			&row.Username,
			&row.DatetimeCreated,
			&row.StreamURL,
			&row.NumAchievements,
			&row.TotalRaces,
		); err != nil {
			return nil, 0, err
		}

		profiles = append(profiles, row)
	}

	// Find total amount of users
	var allProfilesCount int
	if err := db.QueryRow(`
		SELECT count(id)
		FROM users
		WHERE steam_id > 0
	`).Scan(&allProfilesCount); err != nil {
		return nil, 0, err
	}

	return profiles, allProfilesCount, nil
}

// Make a leaderboard for the unseeded format based on all of the users
type LeaderboardRowUnseeded struct {
	Name            string
	AdjustedAverage int
	RealAverage     int
	NumRaces        int
	NumForfeits     int
	ForfeitPenalty  int
	LowestTime      int
	LastRace        time.Time
	Verified        int
	StreamURL       string
}

func (*Users) GetLeaderboardUnseeded() ([]LeaderboardRowUnseeded, error) {
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
			username,
			verified,
			unseeded_adjusted_average,
			unseeded_real_average,
			unseeded_num_races,
			unseeded_num_forfeits,
			unseeded_forfeit_penalty,
			unseeded_lowest_time,
			unseeded_last_race,
			stream_url
		FROM
			users
		WHERE
			unseeded_num_races >= 20
		ORDER BY
			unseeded_adjusted_average ASC
	`); err != nil {
		return nil, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the users
	leaderboard := make([]LeaderboardRowUnseeded, 0)
	for rows.Next() {
		var row LeaderboardRowUnseeded
		if err := rows.Scan(
			&row.Name,
			&row.Verified,
			&row.AdjustedAverage,
			&row.RealAverage,
			&row.NumRaces,
			&row.NumForfeits,
			&row.ForfeitPenalty,
			&row.LowestTime,
			&row.LastRace,
			&row.StreamURL,
		); err != nil {
			return nil, err
		}

		// Append this row to the leaderboard
		leaderboard = append(leaderboard, row)
	}
	return leaderboard, nil
}

// Make a leaderboard for the seeded format based on all of the users
type LeaderboardRowSeeded struct {
	Name      string
	TrueSkill float64
	NumRaces  int
	LastRace  time.Time
	Verified  int
}

func (*Users) GetLeaderboardSeeded() ([]LeaderboardRowSeeded, error) {
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
			username,
			seeded_trueskill,
			seeded_trueskill_sigma,
			seeded_num_races,
			seeded_last_race,
			verified,
		FROM
			users
		WHERE
			seeded_num_races > 1
	`); err != nil {
		return nil, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the users
	leaderboard := make([]LeaderboardRowSeeded, 0)
	for rows.Next() {
		var row LeaderboardRowSeeded
		if err := rows.Scan(
			&row.Name,
			&row.TrueSkill,
			&row.NumRaces,
			&row.LastRace,
			&row.Verified,
		); err != nil {
			return nil, err
		}

		// Append this row to the leaderboard
		leaderboard = append(leaderboard, row)
	}

	return leaderboard, nil
}

type LeaderboardRowDiversity struct {
	Name              string
	DivTrueSkill      float64
	DivTrueSkillDelta float64
	DivNumRaces       sql.NullInt64
	DivLowestTime     sql.NullInt64
	DivLastRace       time.Time
	Verified          int
	StreamURL         string
}

func (*Users) GetLeaderboardDiversity() ([]LeaderboardRowDiversity, error) {
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
		    u.username,
		    u.verified,
		    u.diversity_trueskill,
		    ROUND(u.diversity_trueskill_change, 2),
		    u.diversity_num_races,
		    (SELECT
		            MIN(run_time)
		        FROM
		            race_participants
				LEFT JOIN races
					ON race_participants.race_id = races.id
		        WHERE
		            place > 0
		            AND u.id = user_id
		            AND races.format = 'diversity') as r_time,
		    u.diversity_last_race,
		    u.stream_url
		FROM
		    users u
		        LEFT JOIN
		    race_participants rp ON rp.user_id = u.id
		        LEFT JOIN
		    races r ON r.id = rp.race_id
		WHERE
		    diversity_num_races >= 5
		        AND r.format = 'diversity'
		        AND rp.place > 0
		GROUP BY u.username
		ORDER BY u.diversity_trueskill DESC

	`); err != nil {
		return nil, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the users
	leaderboard := make([]LeaderboardRowDiversity, 0)
	for rows.Next() {
		var row LeaderboardRowDiversity
		if err := rows.Scan(
			&row.Name,
			&row.Verified,
			&row.DivTrueSkill,
			&row.DivTrueSkillDelta,
			&row.DivNumRaces,
			&row.DivLowestTime,
			&row.DivLastRace,
			&row.StreamURL,
		); err != nil {
			return nil, err
		}

		// Append this row to the leaderboard
		leaderboard = append(leaderboard, row)
	}
	return leaderboard, nil
}
<commit_msg>whatever I hated this<commit_after>package models

import (
	"database/sql"
	"time"

	"github.com/go-sql-driver/mysql"
)

/*
	These are more functions for querying the "users" table,
	but these functions are only used for the website
*/

/*
	Data structures
*/

type StatsSeeded struct {
	TrueSkill float32
	Sigma     float32
	NumRaces  int
	LastRace  mysql.NullTime
}

type StatsUnseeded struct {
	AdjustedAverage int
	RealAverage     int
	NumRaces        int
	NumForfeits     int
	ForfeitPenalty  int
	LowestTime      int
	LastRace        mysql.NullTime
}

type StatsDiversity struct {
	TrueSkill    float64
	Sigma        float64
	Change       float64
	NumRaces     int
	LastRace     mysql.NullTime
	NewTrueSkill float64 // Only used when doing new TrueSkill calculation
}

// ProfilesRow gets each row for all profiles
type ProfilesRow struct {
	Username        string
	DatetimeCreated time.Time
	StreamURL       string
	NumAchievements int
	TotalRaces      int
}

// ProfileData has all data for each racer
type ProfileData struct {
	Username          string
	DatetimeCreated   time.Time
	DatetimeLastLogin time.Time
	Admin             int
	Verified          bool
	StatsSeeded       StatsSeeded
	StatsUnseeded     StatsUnseeded
	StatsDiversity    StatsDiversity
	StreamURL         string
	Banned            bool
}

/*
type LeaderboardRowMostPlayed struct {
	Name     string
	Total    int
	Verified int
}
*/

/*
	Functions
*/

/*
func (*Users) GetStatsSeeded(username string) (StatsSeeded, error) {
	var stats StatsSeeded
	if err := db.QueryRow(`
		SELECT
			seeded_trueskill,
			seeded_trueskill_sigma,
			seeded_num_races,
			seeded_last_race
		FROM
			users
		WHERE
			username = ?
	`, username).Scan(
		&stats.ELO,
		&stats.NumSeededRaces,
		&stats.LastSeededRace,
	); err != nil {
		return stats, err
	} else {
		return stats, nil
	}
}

func (*Users) GetStatsUnseeded(username string) (StatsUnseeded, error) {
	var stats StatsUnseeded
	if err := db.QueryRow(`
		SELECT
			unseeded_adjusted_average,
			unseeded_real_average,
			num_unseeded_races,
			num_forfeits,
			forfeit_penalty,
			lowest_unseeded_time,
			last_unseeded_race
		FROM
			users
		WHERE
			username = ?
	`, username).Scan(
		&stats.UnseededAdjustedAverage,
		&stats.UnseededRealAverage,
		&stats.NumUnseededRaces,
		&stats.NumForfeits,
		&stats.ForfeitPenalty,
		&stats.LowestUnseededTime,
		&stats.LastUnseededRace,
	); err != nil {
		return stats, err
	} else {
		return stats, nil
	}
}
*/

// GetProfileData gets player data to populate the player's profile page
func (*Users) GetProfileData(username string) (ProfileData, error) {
	var profileData ProfileData
	var rawVerified int
	if err := db.QueryRow(`
		SELECT
			u.username,
			u.datetime_created,
			u.datetime_last_login,
			u.admin,
			u.verified,
			u.seeded_trueskill,
			u.seeded_trueskill_sigma,
			u.seeded_num_races,
			u.seeded_last_race,
			u.unseeded_adjusted_average,
			u.unseeded_real_average,
			u.unseeded_num_races,
			u.unseeded_num_forfeits,
			u.unseeded_forfeit_penalty,
			u.unseeded_lowest_time,
			u.unseeded_last_race,
			u.stream_url,
			CASE WHEN u.id IN (SELECT user_id FROM banned_users) THEN 1 ELSE 0 END AS BIT

		FROM
			users u
		WHERE
			steam_id > 0 and
			username = ?
	`, username).Scan(
		&profileData.Username,
		&profileData.DatetimeCreated,
		&profileData.DatetimeLastLogin,
		&profileData.Admin,
		&rawVerified,
		&profileData.StatsSeeded.TrueSkill,
		&profileData.StatsSeeded.Sigma,
		&profileData.StatsSeeded.NumRaces,
		&profileData.StatsSeeded.LastRace,
		&profileData.StatsUnseeded.AdjustedAverage,
		&profileData.StatsUnseeded.RealAverage,
		&profileData.StatsUnseeded.NumRaces,
		&profileData.StatsUnseeded.NumForfeits,
		&profileData.StatsUnseeded.ForfeitPenalty,
		&profileData.StatsUnseeded.LowestTime,
		&profileData.StatsUnseeded.LastRace,
		&profileData.StreamURL,
		&profileData.Banned,
	); err == sql.ErrNoRows {
		return profileData, nil
	} else if err != nil {
		return profileData, err
	} else {
		// Convert the int to a bool
		if rawVerified == 1 {
			profileData.Verified = true
		}
		return profileData, nil
	}
}

// GetUserProfiles gets players data to populate the profiles page
func (*Users) GetUserProfiles(currentPage int, usersPerPage int) ([]ProfilesRow, int, error) {
	usersOffset := (currentPage - 1) * usersPerPage
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
			u.username,
			u.datetime_created,
			u.stream_url,
			count(ua.achievement_id),
			(
				SELECT COUNT(id)
				FROM race_participants
				WHERE user_id = u.id
			) AS num_total_race
		FROM
			users u
		LEFT JOIN
			user_achievements ua
			ON
				u.id = ua.user_id
		WHERE
			u.steam_id > 0
		GROUP BY
			u.username
		ORDER BY
			u.username ASC
		LIMIT
			?
		OFFSET
			?
	`, usersPerPage, usersOffset); err == sql.ErrNoRows {
		return nil, 0, nil
	} else if err != nil {
		return nil, 0, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the user profile results
	profiles := make([]ProfilesRow, 0)
	for rows.Next() {
		var row ProfilesRow
		if err := rows.Scan(
			&row.Username,
			&row.DatetimeCreated,
			&row.StreamURL,
			&row.NumAchievements,
			&row.TotalRaces,
		); err != nil {
			return nil, 0, err
		}

		profiles = append(profiles, row)
	}

	// Find total amount of users
	var allProfilesCount int
	if err := db.QueryRow(`
		SELECT count(id)
		FROM users
		WHERE steam_id > 0
	`).Scan(&allProfilesCount); err != nil {
		return nil, 0, err
	}

	return profiles, allProfilesCount, nil
}

// Make a leaderboard for the unseeded format based on all of the users
type LeaderboardRowUnseeded struct {
	Name            string
	AdjustedAverage int
	RealAverage     int
	NumRaces        int
	NumForfeits     int
	ForfeitPenalty  int
	LowestTime      int
	LastRace        time.Time
	LastRaceId      int
	Verified        int
	StreamURL       string
}

func (*Users) GetLeaderboardUnseeded(racesNeeded int, racesLimit int) ([]LeaderboardRowUnseeded, error) {
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
			u.username,
			u.verified,
			u.unseeded_adjusted_average,
			u.unseeded_real_average,
			u.unseeded_num_races,
			u.unseeded_num_forfeits,
			u.unseeded_forfeit_penalty,
			u.unseeded_lowest_time,
			u.unseeded_last_race,
			MAX(rp.race_id),
			u.stream_url
		FROM
			users u
			LEFT JOIN race_participants rp
				ON rp.user_id = u.id
			LEFT JOIN races r
				ON r.id = rp.race_id
		WHERE
			u.unseeded_num_races >= ?
			AND u.id NOT IN (SELECT user_id FROM banned_users)
		GROUP BY
			u.username
		ORDER BY
			unseeded_adjusted_average ASC
		LIMIT ?
	`, racesNeeded, racesLimit); err != nil {
		return nil, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the users
	leaderboard := make([]LeaderboardRowUnseeded, 0)
	for rows.Next() {
		var row LeaderboardRowUnseeded
		if err := rows.Scan(
			&row.Name,
			&row.AdjustedAverage,
			&row.RealAverage,
			&row.NumRaces,
			&row.NumForfeits,
			&row.ForfeitPenalty,
			&row.LowestTime,
			&row.LastRace,
			&row.LastRaceId,
			&row.Verified,
			&row.StreamURL,
		); err != nil {
			return nil, err
		}

		// Append this row to the leaderboard
		leaderboard = append(leaderboard, row)
	}
	return leaderboard, nil
}

// Make a leaderboard for the seeded format based on all of the users
type LeaderboardRowSeeded struct {
	Name      string
	TrueSkill float64
	NumRaces  int
	LastRace  time.Time
	Verified  int
}

func (*Users) GetLeaderboardSeeded(racesNeeded int, racesLimit int) ([]LeaderboardRowSeeded, error) {
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
			u.username,
			u.seeded_trueskill,
			u.seeded_trueskill_sigma,
			u.seeded_num_races,
			u.seeded_last_race,
			u.verified,
		FROM
			users u
		WHERE
			u.seeded_num_races > ?
			AND u.id NOT IN (SELECT user_id FROM banned_users)
		GROUP BY u.username
		LIMIT ?
	`, racesNeeded, racesLimit); err != nil {
		return nil, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the users
	leaderboard := make([]LeaderboardRowSeeded, 0)
	for rows.Next() {
		var row LeaderboardRowSeeded
		if err := rows.Scan(
			&row.Name,
			&row.TrueSkill,
			&row.NumRaces,
			&row.LastRace,
			&row.Verified,
		); err != nil {
			return nil, err
		}

		// Append this row to the leaderboard
		leaderboard = append(leaderboard, row)
	}

	return leaderboard, nil
}

type LeaderboardRowDiversity struct {
	Name              string
	DivTrueSkill      float64
	DivTrueSkillDelta float64
	DivNumRaces       sql.NullInt64
	DivLowestTime     sql.NullInt64
	DivLastRace       time.Time
	DivLastRaceId     int
	Verified          int
	StreamURL         string
}

func (*Users) GetLeaderboardDiversity(racesNeeded int, racesLimit int) ([]LeaderboardRowDiversity, error) {
	var rows *sql.Rows
	if v, err := db.Query(`
		SELECT
			u.username,
			u.verified,
			u.diversity_trueskill,
			ROUND(u.diversity_trueskill_change, 2),
			u.diversity_num_races,
			(SELECT
					MIN(run_time)
				FROM
					race_participants
				LEFT JOIN races
					ON race_participants.race_id = races.id
				WHERE
					place > 0
					AND u.id = user_id
					AND races.format = 'diversity') as r_time,
			u.diversity_last_race,
			MAX(rp.race_id),
			u.stream_url
		FROM
			users u
			LEFT JOIN
				race_participants rp ON rp.user_id = u.id
			LEFT JOIN
				races r ON r.id = rp.race_id
		WHERE
			diversity_num_races >= ?
				AND r.format = 'diversity'
				AND rp.place > 0
				AND u.id NOT IN (SELECT user_id FROM banned_users)
		GROUP BY u.username
		ORDER BY u.diversity_trueskill DESC
		LIMIT ?
	`, racesNeeded, racesLimit); err != nil {
		return nil, err
	} else {
		rows = v
	}
	defer rows.Close()

	// Iterate over the users
	leaderboard := make([]LeaderboardRowDiversity, 0)
	for rows.Next() {
		var row LeaderboardRowDiversity
		if err := rows.Scan(
			&row.Name,
			&row.DivTrueSkill,
			&row.DivTrueSkillDelta,
			&row.DivNumRaces,
			&row.DivLowestTime,
			&row.DivLastRace,
			&row.DivLastRaceId,
			&row.Verified,
			&row.StreamURL,
		); err != nil {
			return nil, err
		}

		// Append this row to the leaderboard
		leaderboard = append(leaderboard, row)
	}
	return leaderboard, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"syscall"
	"unicode/utf8"

	"github.com/mibk/syd/event"
	"github.com/mibk/syd/textutil"
	"github.com/mibk/syd/vi"
	"github.com/mibk/syd/view"
)

func doOnce(f func()) func(int) {
	return func(_ int) {
		f()
	}
}

func doNTimesAndCommit(f func()) func(int) {
	return func(n int) {
		vi.DoNTimes(f)(n)
		buffer.CommitChanges()
	}
}

func trans(cmd string) []event.KeyPress {
	events := make([]event.KeyPress, 0, len(cmd))
	for _, r := range []rune(cmd) {
		events = append(events, event.KeyPress{Key: event.Key(r)})
	}
	return events
}

func performMapping() {
	parser.AddCommand(trans("ZQ"), doOnce(quit))
	parser.AddCommand(trans("ZZ"), doOnce(saveAndQuit))
	parser.AddCommand([]event.KeyPress{{Key: 'z', Ctrl: true}}, doOnce(suspend))

	parser.AddCommand(trans("."), vi.DoNTimes(repeatLastAction))

	parser.AddMotion(trans("j"), vi.DoNTimes(down))
	parser.AddMotion(trans("k"), vi.DoNTimes(up))
	parser.AddMotion(trans("h"), vi.DoNTimes(left))
	parser.AddMotion(trans("l"), vi.DoNTimes(right))

	parser.AddMotion(trans("G"), gotoLine)
	parser.AddAlias(trans("gg"), trans("1G"))
	parser.AddMotion(trans("|"), gotoColumn)
	parser.AddAlias(trans("0"), trans("|"))
	parser.AddMotion(trans("^"), doOnce(gotoFirstNonBlank))
	parser.AddMotion(trans("$"), doOnce(gotoEOL))
	parser.AddMotion(trans("_"), underscore)

	parser.AddMotion(trans("H"), gotoScreenLineFromTop)
	parser.AddMotion(trans("L"), gotoScreenLineFromBotton)
	parser.AddMotion(trans("M"), doOnce(gotoMiddleScreenLine))
	parser.AddCommand(trans("zt"), doOnce(setScreenLineTop))
	parser.AddCommand(trans("zz"), doOnce(setScreenLineMiddle))
	parser.AddCommand(trans("zb"), doOnce(SetScreenLineBottom))

	parser.AddCommand([]event.KeyPress{{Key: 'f', Ctrl: true}}, vi.DoNTimes(pageDown))
	parser.AddCommand([]event.KeyPress{{Key: 'b', Ctrl: true}}, vi.DoNTimes(pageUp))

	parser.AddCommand(trans("i"), doOnce(insertMode))
	parser.AddCommand(trans("a"), doOnce(appendRight))
	parser.AddCommand(trans("o"), doOnce(openLineDown))
	parser.AddCommand(trans("O"), doOnce(openLineUp))
	parser.AddAlias(trans("I"), trans("^i"))
	parser.AddAlias(trans("A"), trans("$a"))

	parser.AddCommand(trans(":"), doOnce(commandMode))
	parser.AddCommand(trans("u"), vi.DoNTimes(undo))
	parser.AddCommand([]event.KeyPress{{Key: 'r', Ctrl: true}}, vi.DoNTimes(redo))

	parser.AddCommand(trans("d"), doNTimesAndCommit(delete), vi.RequiresMotion)
	parser.AddAlias(trans("dd"), trans("d_"))
	parser.AddAlias(trans("D"), trans("d$"))
	parser.AddAlias(trans("x"), trans("dl"))
	parser.AddAlias(trans("X"), trans("dh"))

	parser.AddCommand(trans("c"), doNTimesAndCommit(change), vi.RequiresMotion)
	parser.AddAlias(trans("cc"), trans("c_"))
	parser.AddAlias(trans("C"), trans("c$"))
	parser.AddAlias(trans("s"), trans("dli"))
	parser.AddAlias(trans("S"), trans("c_"))

	parser.AddCommand(trans("r"), replace)

	parser.AddCommand(trans("y"), doOnce(yank), vi.RequiresMotion)
	parser.AddAlias(trans("yy"), trans("y_"))
	parser.AddAlias(trans("Y"), trans("y$"))
	parser.AddCommand(trans("P"), doOnce(Paste))
	parser.AddCommand(trans("p"), doOnce(paste))
}

func quit()        { shouldQuit = true }
func saveAndQuit() { checkAndSave(); quit() }
func suspend() {
	ui.Close()
	defer ui.Reinit()
	pid, tid := syscall.Getpid(), syscall.Gettid()
	if err := syscall.Tgkill(pid, tid, syscall.SIGSTOP); err != nil {
		panic(err)
	}
}

func repeatLastAction() {
	doNotRemember()
	lastAction()
}

func down()  { viewport.GotoLine(viewport.Line() + 1); linewise() }
func up()    { viewport.GotoLine(viewport.Line() - 1); linewise() }
func right() { viewport.GotoColumn(viewport.Column() + 1); charwise() }
func left()  { viewport.GotoColumn(viewport.Column() - 1); charwise() }

func underscore(n int) {
	if n != 0 {
		n--
	}
	viewport.GotoLine(viewport.Line() + n)
	linewise()
}

func gotoLine(n int) {
	if n == 0 {
		n = view.Last
	} else {
		n--
	}
	viewport.GotoLine(n)
	linewise()
}
func gotoEOL()         { viewport.GotoColumn(view.Last); charwise() }
func gotoColumn(n int) { viewport.GotoColumn(n - 1); charwise() }
func gotoFirstNonBlank() {
	off := viewport.CurrentCell().Offset
	start := textutil.FindLineStart(buffer, int64(off))
	off = int(textutil.FindIndentOffset(buffer, start))
	viewport.SetCursor(off)
	charwise()
}

func gotoScreenLineFromTop(n int) {
	if n != 0 {
		n--
	}
	viewport.GotoLine(viewport.Line() - viewport.ScreenLine() + n)
}
func gotoScreenLineFromBotton(n int) {
	if n != 0 {
		n--
	}
	viewport.GotoLine(viewport.Line() - viewport.ScreenLine() +
		viewport.Height() - n - 1)
}
func gotoMiddleScreenLine() { gotoScreenLineFromTop(viewport.Height() / 2) }

func setScreenLineTop()    { setScreenLinePos(0) }
func setScreenLineMiddle() { setScreenLinePos(viewport.Height()/2 - 1) }
func SetScreenLineBottom() { setScreenLinePos(viewport.Height() - 1) }
func setScreenLinePos(pos int) {
	s := viewport.ScreenLine()
	f := viewport.FirstLine()
	l := viewport.Line()
	viewport.SetFirstLine(f - (pos - s))
	viewport.GotoLine(l)
}

func pageDown() { viewport.SetFirstLine(viewport.FirstLine() + viewport.Height()) }
func pageUp()   { viewport.SetFirstLine(viewport.FirstLine() - viewport.Height()) }

func undo() { buffer.Undo() }
func redo() { buffer.Redo() }

func delete() {
	start, end, desiredOffset := findBorders()
	buffer.Delete(start, end-start)
	viewport.SetCursor(desiredOffset)
	lastOffset = start
}

func findBorders() (off1, off2, desiredOffset int) {
	off1 = lastOffset
	off2 = viewport.CurrentCell().Offset
	if off1 > off2 {
		off1, off2 = off2, off1
	}
	desiredOffset = off1
	if isLinewise {
		off1 = int(textutil.FindLineStart(buffer, int64(off1)))
		off2 = int(textutil.FindLineEnd(buffer, int64(off2)))
	}
	return
}

func yank() {
	start, end, desiredOffset := findBorders()

	clipboard = make([]byte, end-start)
	buffer.ReadAt(clipboard, int64(start))
	wasCopiedLinewise = isLinewise

	viewport.SetCursor(desiredOffset)
	lastOffset = start
}
func paste() {
	if clipboard == nil {
		return
	}
	off := lastOffset
	if wasCopiedLinewise {
		off = int(textutil.FindLineEnd(buffer, int64(off)))
		down()
	} else {
		right()
	}
	buffer.Insert(off, clipboard)
}
func Paste() {
	if clipboard == nil {
		return
	}
	off := lastOffset
	if wasCopiedLinewise {
		off = int(textutil.FindLineStart(buffer, int64(off)))
	}
	buffer.Insert(off, clipboard)
}

func appendRight() {
	right()
	insertMode()
}

func openLineDown() {
	end := int(textutil.FindLineEnd(buffer, int64(lastOffset)))
	start := int(textutil.FindLineStart(buffer, int64(lastOffset)))
	openLine(end, start)
}

func openLineUp() {
	off := int(textutil.FindLineStart(buffer, int64(lastOffset)))
	openLine(off, off)
}

func openLine(off, start int) {
	ioffset := textutil.FindIndentOffset(buffer, int64(start))
	b := make([]byte, int(ioffset)-start+1)
	buffer.ReadAt(b[:len(b)-1], int64(start))
	b[len(b)-1] = '\n'
	buffer.Insert(off, b)
	viewport.SetCursor(off + len(b) - 1)
	insertMode()
}

func change() {
	delete()
	if isLinewise {
		openLineUp()
	} else {
		insertMode()
	}
}

func replace(n int) {
	if n == 0 {
		n = 1
	}
	ev := event.PollEvent()
	if ev, ok := ev.(event.KeyPress); ok {
		viewport.GotoColumn(viewport.Column() + n)
		delete()
		p := make([]byte, 4)
		length := utf8.EncodeRune(p, rune(ev.Key))
		p = bytes.Repeat(p[:length], n)
		buffer.Insert(lastOffset, p)
	}
}
<commit_msg>Paste n times<commit_after>package main

import (
	"bytes"
	"syscall"
	"unicode/utf8"

	"github.com/mibk/syd/event"
	"github.com/mibk/syd/textutil"
	"github.com/mibk/syd/vi"
	"github.com/mibk/syd/view"
)

func doOnce(f func()) func(int) {
	return func(_ int) {
		f()
	}
}

func doNTimesAndCommit(f func()) func(int) {
	return func(n int) {
		vi.DoNTimes(f)(n)
		buffer.CommitChanges()
	}
}

func trans(cmd string) []event.KeyPress {
	events := make([]event.KeyPress, 0, len(cmd))
	for _, r := range []rune(cmd) {
		events = append(events, event.KeyPress{Key: event.Key(r)})
	}
	return events
}

func performMapping() {
	parser.AddCommand(trans("ZQ"), doOnce(quit))
	parser.AddCommand(trans("ZZ"), doOnce(saveAndQuit))
	parser.AddCommand([]event.KeyPress{{Key: 'z', Ctrl: true}}, doOnce(suspend))

	parser.AddCommand(trans("."), vi.DoNTimes(repeatLastAction))

	parser.AddMotion(trans("j"), vi.DoNTimes(down))
	parser.AddMotion(trans("k"), vi.DoNTimes(up))
	parser.AddMotion(trans("h"), vi.DoNTimes(left))
	parser.AddMotion(trans("l"), vi.DoNTimes(right))

	parser.AddMotion(trans("G"), gotoLine)
	parser.AddAlias(trans("gg"), trans("1G"))
	parser.AddMotion(trans("|"), gotoColumn)
	parser.AddAlias(trans("0"), trans("|"))
	parser.AddMotion(trans("^"), doOnce(gotoFirstNonBlank))
	parser.AddMotion(trans("$"), doOnce(gotoEOL))
	parser.AddMotion(trans("_"), underscore)

	parser.AddMotion(trans("H"), gotoScreenLineFromTop)
	parser.AddMotion(trans("L"), gotoScreenLineFromBotton)
	parser.AddMotion(trans("M"), doOnce(gotoMiddleScreenLine))
	parser.AddCommand(trans("zt"), doOnce(setScreenLineTop))
	parser.AddCommand(trans("zz"), doOnce(setScreenLineMiddle))
	parser.AddCommand(trans("zb"), doOnce(SetScreenLineBottom))

	parser.AddCommand([]event.KeyPress{{Key: 'f', Ctrl: true}}, vi.DoNTimes(pageDown))
	parser.AddCommand([]event.KeyPress{{Key: 'b', Ctrl: true}}, vi.DoNTimes(pageUp))

	parser.AddCommand(trans("i"), doOnce(insertMode))
	parser.AddCommand(trans("a"), doOnce(appendRight))
	parser.AddCommand(trans("o"), doOnce(openLineDown))
	parser.AddCommand(trans("O"), doOnce(openLineUp))
	parser.AddAlias(trans("I"), trans("^i"))
	parser.AddAlias(trans("A"), trans("$a"))

	parser.AddCommand(trans(":"), doOnce(commandMode))
	parser.AddCommand(trans("u"), vi.DoNTimes(undo))
	parser.AddCommand([]event.KeyPress{{Key: 'r', Ctrl: true}}, vi.DoNTimes(redo))

	parser.AddCommand(trans("d"), doNTimesAndCommit(delete), vi.RequiresMotion)
	parser.AddAlias(trans("dd"), trans("d_"))
	parser.AddAlias(trans("D"), trans("d$"))
	parser.AddAlias(trans("x"), trans("dl"))
	parser.AddAlias(trans("X"), trans("dh"))

	parser.AddCommand(trans("c"), doNTimesAndCommit(change), vi.RequiresMotion)
	parser.AddAlias(trans("cc"), trans("c_"))
	parser.AddAlias(trans("C"), trans("c$"))
	parser.AddAlias(trans("s"), trans("dli"))
	parser.AddAlias(trans("S"), trans("c_"))

	parser.AddCommand(trans("r"), replace)

	parser.AddCommand(trans("y"), doOnce(yank), vi.RequiresMotion)
	parser.AddAlias(trans("yy"), trans("y_"))
	parser.AddAlias(trans("Y"), trans("y$"))
	parser.AddCommand(trans("P"), doNTimesAndCommit(Paste))
	parser.AddCommand(trans("p"), doNTimesAndCommit(paste))
}

func quit()        { shouldQuit = true }
func saveAndQuit() { checkAndSave(); quit() }
func suspend() {
	ui.Close()
	defer ui.Reinit()
	pid, tid := syscall.Getpid(), syscall.Gettid()
	if err := syscall.Tgkill(pid, tid, syscall.SIGSTOP); err != nil {
		panic(err)
	}
}

func repeatLastAction() {
	doNotRemember()
	lastAction()
}

func down()  { viewport.GotoLine(viewport.Line() + 1); linewise() }
func up()    { viewport.GotoLine(viewport.Line() - 1); linewise() }
func right() { viewport.GotoColumn(viewport.Column() + 1); charwise() }
func left()  { viewport.GotoColumn(viewport.Column() - 1); charwise() }

func underscore(n int) {
	if n != 0 {
		n--
	}
	viewport.GotoLine(viewport.Line() + n)
	linewise()
}

func gotoLine(n int) {
	if n == 0 {
		n = view.Last
	} else {
		n--
	}
	viewport.GotoLine(n)
	linewise()
}
func gotoEOL()         { viewport.GotoColumn(view.Last); charwise() }
func gotoColumn(n int) { viewport.GotoColumn(n - 1); charwise() }
func gotoFirstNonBlank() {
	off := viewport.CurrentCell().Offset
	start := textutil.FindLineStart(buffer, int64(off))
	off = int(textutil.FindIndentOffset(buffer, start))
	viewport.SetCursor(off)
	charwise()
}

func gotoScreenLineFromTop(n int) {
	if n != 0 {
		n--
	}
	viewport.GotoLine(viewport.Line() - viewport.ScreenLine() + n)
}
func gotoScreenLineFromBotton(n int) {
	if n != 0 {
		n--
	}
	viewport.GotoLine(viewport.Line() - viewport.ScreenLine() +
		viewport.Height() - n - 1)
}
func gotoMiddleScreenLine() { gotoScreenLineFromTop(viewport.Height() / 2) }

func setScreenLineTop()    { setScreenLinePos(0) }
func setScreenLineMiddle() { setScreenLinePos(viewport.Height()/2 - 1) }
func SetScreenLineBottom() { setScreenLinePos(viewport.Height() - 1) }
func setScreenLinePos(pos int) {
	s := viewport.ScreenLine()
	f := viewport.FirstLine()
	l := viewport.Line()
	viewport.SetFirstLine(f - (pos - s))
	viewport.GotoLine(l)
}

func pageDown() { viewport.SetFirstLine(viewport.FirstLine() + viewport.Height()) }
func pageUp()   { viewport.SetFirstLine(viewport.FirstLine() - viewport.Height()) }

func undo() { buffer.Undo() }
func redo() { buffer.Redo() }

func delete() {
	start, end, desiredOffset := findBorders()
	buffer.Delete(start, end-start)
	viewport.SetCursor(desiredOffset)
	lastOffset = start
}

func findBorders() (off1, off2, desiredOffset int) {
	off1 = lastOffset
	off2 = viewport.CurrentCell().Offset
	if off1 > off2 {
		off1, off2 = off2, off1
	}
	desiredOffset = off1
	if isLinewise {
		off1 = int(textutil.FindLineStart(buffer, int64(off1)))
		off2 = int(textutil.FindLineEnd(buffer, int64(off2)))
	}
	return
}

func yank() {
	start, end, desiredOffset := findBorders()

	clipboard = make([]byte, end-start)
	buffer.ReadAt(clipboard, int64(start))
	wasCopiedLinewise = isLinewise

	viewport.SetCursor(desiredOffset)
	lastOffset = start
}
func paste() {
	if clipboard == nil {
		return
	}
	off := lastOffset
	if wasCopiedLinewise {
		off = int(textutil.FindLineEnd(buffer, int64(off)))
		down()
	} else {
		right()
	}
	buffer.Insert(off, clipboard)
}
func Paste() {
	if clipboard == nil {
		return
	}
	off := lastOffset
	if wasCopiedLinewise {
		off = int(textutil.FindLineStart(buffer, int64(off)))
	}
	buffer.Insert(off, clipboard)
}

func appendRight() {
	right()
	insertMode()
}

func openLineDown() {
	end := int(textutil.FindLineEnd(buffer, int64(lastOffset)))
	start := int(textutil.FindLineStart(buffer, int64(lastOffset)))
	openLine(end, start)
}

func openLineUp() {
	off := int(textutil.FindLineStart(buffer, int64(lastOffset)))
	openLine(off, off)
}

func openLine(off, start int) {
	ioffset := textutil.FindIndentOffset(buffer, int64(start))
	b := make([]byte, int(ioffset)-start+1)
	buffer.ReadAt(b[:len(b)-1], int64(start))
	b[len(b)-1] = '\n'
	buffer.Insert(off, b)
	viewport.SetCursor(off + len(b) - 1)
	insertMode()
}

func change() {
	delete()
	if isLinewise {
		openLineUp()
	} else {
		insertMode()
	}
}

func replace(n int) {
	if n == 0 {
		n = 1
	}
	ev := event.PollEvent()
	if ev, ok := ev.(event.KeyPress); ok {
		viewport.GotoColumn(viewport.Column() + n)
		delete()
		p := make([]byte, 4)
		length := utf8.EncodeRune(p, rune(ev.Key))
		p = bytes.Repeat(p[:length], n)
		buffer.Insert(lastOffset, p)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vellum

import (
	"bytes"
	"io"
	"strconv"
)

var defaultBuilderOpts = &BuilderOpts{
	Encoder:           1,
	RegistryTableSize: 10000,
	RegistryMRUSize:   2,
}

// A Builder is used to build a new FST.  When possible data is
// streamed out to the underlying Writer as soon as possible.
type Builder struct {
	opts      *BuilderOpts
	root      *builderState
	nextID    int
	nodeCount uint
	registry  *registry
	lastVal   []byte
	encoder   encoder
	len       int
}

// NewBuilder returns a new Builder which will stream out the
// underlying representation to the provided Writer as the set is built.
func newBuilder(w io.Writer, opts *BuilderOpts) (*Builder, error) {
	if opts == nil {
		opts = defaultBuilderOpts
	}
	rv := &Builder{
		nextID:    1,
		registry:  newRegistry(opts.RegistryTableSize, opts.RegistryMRUSize),
		root:      &builderState{},
		nodeCount: 1,
		opts:      opts,
	}

	var err error
	rv.encoder, err = loadEncoder(opts.Encoder, w)
	if err != nil {
		return nil, err
	}
	err = rv.encoder.start(rv)
	if err != nil {
		return nil, err
	}
	return rv, nil
}

// Insert the provided value to the set being built.
// NOTE: values must be inserted in lexicographical order.
func (s *Builder) Insert(key []byte, val uint64) error {
	// ensure items are added in lexicographic order
	if bytes.Compare(key, s.lastVal) < 0 {
		return ErrOutOfOrder
	}
	// identify the common prefix between this val and the last
	commonLen := commonPrefixLen(s.lastVal, key)
	commonPrefix := key[:commonLen]
	// update last val
	s.lastVal = key
	// find the optimization point, the last common state
	optState, val := s.traverseInsert(commonPrefix, val)
	// optimize the portion that can be updated
	err := s.optimize(optState)
	if err != nil {
		return err
	}
	// add the remaining bytes
	s.addSuffix(optState, key[commonLen:], val)
	s.len++
	return nil
}

// Close MUST be called after inserting all values.
func (s *Builder) Close() error {
	s.lastVal = nil
	err := s.optimize(s.root)
	if err != nil {
		return err
	}
	err = s.encoder.encodeState(s.root)
	if err != nil {
		return err
	}
	err = s.encoder.finish(s)
	if err != nil {
		return err
	}
	return nil
}

func (s *Builder) traverseInsert(key []byte, val uint64) (*builderState, uint64) {
	state := s.root
	var next *transition
	for i := range key {
		var adjustment uint64
		next = state.transitionFor(key[i])
		if next != nil {
			if next.val > val {
				diff := next.val - val
				adjustment += diff
				next.val -= diff
				if next.dest.final {
					next.dest.finalVal += diff
				}
				val = 0
			} else {
				val = val - next.val
			}

			// push down adjustment to all descendants of the current dest
			for j := range next.dest.transitions {
				next.dest.transitions[j].val += adjustment
			}

			state = next.dest
		} else {
			// should never happen during insert, as we already established
			// the common prefix, look for way to eliminate this
			return nil, val
		}
	}
	return state, val
}

func (s *Builder) traverse(key []byte) (*builderState, uint64) {
	var next *transition
	state := s.root
	var val uint64
	for i := range key {
		next = state.transitionFor(key[i])
		if next != nil {
			val += next.val
			state = next.dest
		} else {
			return nil, 0
		}
	}
	if next != nil && next.dest.final {
		val += next.dest.finalVal
	}
	return state, val
}

func (s *Builder) optimize(state *builderState) error {
	if !state.hasTransitions() {
		return nil
	}

	lastTransition := state.lastTransition()
	err := s.optimize(lastTransition.dest)
	if err != nil {
		return err
	}

	if equiv := s.registry.entry(lastTransition.dest); equiv != nil {
		state.replaceTransition(&transition{key: lastTransition.key, dest: equiv, val: lastTransition.val})
		s.nodeCount--
	} else {
		err := s.encoder.encodeState(lastTransition.dest)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Builder) addSuffix(start *builderState, suffix []byte, val uint64) {
	node := start
	for i, char := range suffix {
		newNode := &builderState{
			id: s.nextID,
		}
		transition := &transition{key: char, dest: newNode}
		if i == 0 {
			transition.val = val
		}
		node.addTransition(transition)
		node = newNode
		s.nextID++
		s.nodeCount++
	}
	node.final = true
}

type builderState struct {
	transitions []*transition
	id          int
	final       bool
	finalVal    uint64
	offset      int
}

func (s *builderState) equiv(o *builderState) bool {
	if s.final != o.final {
		return false
	}
	if s.finalVal != o.finalVal {
		return false
	}
	if len(s.transitions) != len(o.transitions) {
		return false
	}
	for i := range s.transitions {
		if !s.transitions[i].equiv(o.transitions[i]) {
			return false
		}
	}
	return true
}

func (s *builderState) hasTransitions() bool {
	return len(s.transitions) > 0
}

func (s *builderState) findTransition(t byte) int {
	for i := range s.transitions {
		if t == s.transitions[i].key {
			return i
		}
	}
	return -1
}

func (s *builderState) transitionFor(t byte) *transition {
	pos := s.findTransition(t)
	if pos < 0 {
		return nil
	}
	return s.transitions[pos]
}

func (s *builderState) replaceTransition(replacement *transition) {
	pos := s.findTransition(replacement.key)
	if pos < 0 {
		return
	}
	s.transitions[pos] = replacement
}

func (s *builderState) lastTransition() *transition {
	if len(s.transitions) < 1 {
		return nil
	}
	return s.transitions[len(s.transitions)-1]
}

func (s *builderState) addTransition(transition *transition) {
	s.transitions = append(s.transitions, transition)
}

func (s *builderState) hash() string {
	var hash string
	if s.final {
		hash += "f"
	}

	for i := range s.transitions {
		transitionState := s.transitions[i].dest
		hash += string(s.transitions[i].key) + strconv.Itoa(transitionState.id)
	}

	return hash
}

type transition struct {
	key  byte
	dest *builderState
	val  uint64
}

func (t *transition) equiv(o *transition) bool {
	if t.key != o.key {
		return false
	}
	if t.dest != o.dest {
		return false
	}
	if t.val != o.val {
		return false
	}
	return true
}

// commonPrefixLen is a helper method used in several places to find
// the common prefix length of two byte slices
func commonPrefixLen(a, b []byte) int {
	prefixLen := 0
	lim := len(a)
	if len(b) < lim {
		lim = len(b)
	}
	for i := 0; i < lim; i++ {
		if a[i] != b[i] {
			break
		}
		prefixLen++
	}
	return prefixLen
}
<commit_msg>don't waste space in registry for implicit final state<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vellum

import (
	"bytes"
	"io"
	"strconv"
)

var defaultBuilderOpts = &BuilderOpts{
	Encoder:           1,
	RegistryTableSize: 10000,
	RegistryMRUSize:   2,
}

// A Builder is used to build a new FST.  When possible data is
// streamed out to the underlying Writer as soon as possible.
type Builder struct {
	opts      *BuilderOpts
	root      *builderState
	nextID    int
	nodeCount uint
	registry  *registry
	lastVal   []byte
	encoder   encoder
	len       int

	implicitFinal *builderState
}

// NewBuilder returns a new Builder which will stream out the
// underlying representation to the provided Writer as the set is built.
func newBuilder(w io.Writer, opts *BuilderOpts) (*Builder, error) {
	if opts == nil {
		opts = defaultBuilderOpts
	}
	rv := &Builder{
		nextID:    1,
		registry:  newRegistry(opts.RegistryTableSize, opts.RegistryMRUSize),
		root:      &builderState{},
		nodeCount: 1,
		opts:      opts,
	}

	var err error
	rv.encoder, err = loadEncoder(opts.Encoder, w)
	if err != nil {
		return nil, err
	}
	err = rv.encoder.start(rv)
	if err != nil {
		return nil, err
	}
	return rv, nil
}

// Insert the provided value to the set being built.
// NOTE: values must be inserted in lexicographical order.
func (s *Builder) Insert(key []byte, val uint64) error {
	// ensure items are added in lexicographic order
	if bytes.Compare(key, s.lastVal) < 0 {
		return ErrOutOfOrder
	}
	// identify the common prefix between this val and the last
	commonLen := commonPrefixLen(s.lastVal, key)
	commonPrefix := key[:commonLen]
	// update last val
	s.lastVal = key
	// find the optimization point, the last common state
	optState, val := s.traverseInsert(commonPrefix, val)
	// optimize the portion that can be updated
	err := s.optimize(optState)
	if err != nil {
		return err
	}
	// add the remaining bytes
	s.addSuffix(optState, key[commonLen:], val)
	s.len++
	return nil
}

// Close MUST be called after inserting all values.
func (s *Builder) Close() error {
	s.lastVal = nil
	err := s.optimize(s.root)
	if err != nil {
		return err
	}
	err = s.encoder.encodeState(s.root)
	if err != nil {
		return err
	}
	err = s.encoder.finish(s)
	if err != nil {
		return err
	}
	return nil
}

func (s *Builder) traverseInsert(key []byte, val uint64) (*builderState, uint64) {
	state := s.root
	var next *transition
	for i := range key {
		var adjustment uint64
		next = state.transitionFor(key[i])
		if next != nil {
			if next.val > val {
				diff := next.val - val
				adjustment += diff
				next.val -= diff
				if next.dest.final {
					next.dest.finalVal += diff
				}
				val = 0
			} else {
				val = val - next.val
			}

			// push down adjustment to all descendants of the current dest
			for j := range next.dest.transitions {
				next.dest.transitions[j].val += adjustment
			}

			state = next.dest
		} else {
			// should never happen during insert, as we already established
			// the common prefix, look for way to eliminate this
			return nil, val
		}
	}
	return state, val
}

func (s *Builder) traverse(key []byte) (*builderState, uint64) {
	var next *transition
	state := s.root
	var val uint64
	for i := range key {
		next = state.transitionFor(key[i])
		if next != nil {
			val += next.val
			state = next.dest
		} else {
			return nil, 0
		}
	}
	if next != nil && next.dest.final {
		val += next.dest.finalVal
	}
	return state, val
}

func (s *Builder) optimize(state *builderState) error {
	if !state.hasTransitions() {
		return nil
	}

	lastTransition := state.lastTransition()
	err := s.optimize(lastTransition.dest)
	if err != nil {
		return err
	}

	// we don't want to waste a slot in the cache for the implicit final state
	// instead, for now we track it explicitly.  this should be cleaned up
	// further in the future, as all we really care is that the file offset
	// becomes 0, but for now this is required.
	if lastTransition.dest.final && !lastTransition.dest.hasTransitions() &&
		lastTransition.dest.finalVal == 0 {

		// the first time we've encountered this situation, remember the state
		if s.implicitFinal == nil {
			s.implicitFinal = lastTransition.dest
			return nil
		}

		// replace ourselves with the implicit final state
		state.replaceTransition(&transition{key: lastTransition.key, dest: s.implicitFinal, val: lastTransition.val})
		s.nodeCount--
		return nil
	}

	if equiv := s.registry.entry(lastTransition.dest); equiv != nil {
		state.replaceTransition(&transition{key: lastTransition.key, dest: equiv, val: lastTransition.val})
		s.nodeCount--
	} else {
		err := s.encoder.encodeState(lastTransition.dest)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Builder) addSuffix(start *builderState, suffix []byte, val uint64) {
	node := start
	for i, char := range suffix {
		newNode := &builderState{
			id: s.nextID,
		}
		transition := &transition{key: char, dest: newNode}
		if i == 0 {
			transition.val = val
		}
		node.addTransition(transition)
		node = newNode
		s.nextID++
		s.nodeCount++
	}
	node.final = true
}

type builderState struct {
	transitions []*transition
	id          int
	final       bool
	finalVal    uint64
	offset      int
}

func (s *builderState) equiv(o *builderState) bool {
	if s.final != o.final {
		return false
	}
	if s.finalVal != o.finalVal {
		return false
	}
	if len(s.transitions) != len(o.transitions) {
		return false
	}
	for i := range s.transitions {
		if !s.transitions[i].equiv(o.transitions[i]) {
			return false
		}
	}
	return true
}

func (s *builderState) hasTransitions() bool {
	return len(s.transitions) > 0
}

func (s *builderState) findTransition(t byte) int {
	for i := range s.transitions {
		if t == s.transitions[i].key {
			return i
		}
	}
	return -1
}

func (s *builderState) transitionFor(t byte) *transition {
	pos := s.findTransition(t)
	if pos < 0 {
		return nil
	}
	return s.transitions[pos]
}

func (s *builderState) replaceTransition(replacement *transition) {
	pos := s.findTransition(replacement.key)
	if pos < 0 {
		return
	}
	s.transitions[pos] = replacement
}

func (s *builderState) lastTransition() *transition {
	if len(s.transitions) < 1 {
		return nil
	}
	return s.transitions[len(s.transitions)-1]
}

func (s *builderState) addTransition(transition *transition) {
	s.transitions = append(s.transitions, transition)
}

func (s *builderState) hash() string {
	var hash string
	if s.final {
		hash += "f"
	}

	for i := range s.transitions {
		transitionState := s.transitions[i].dest
		hash += string(s.transitions[i].key) + strconv.Itoa(transitionState.id)
	}

	return hash
}

type transition struct {
	key  byte
	dest *builderState
	val  uint64
}

func (t *transition) equiv(o *transition) bool {
	if t.key != o.key {
		return false
	}
	if t.dest != o.dest {
		return false
	}
	if t.val != o.val {
		return false
	}
	return true
}

// commonPrefixLen is a helper method used in several places to find
// the common prefix length of two byte slices
func commonPrefixLen(a, b []byte) int {
	prefixLen := 0
	lim := len(a)
	if len(b) < lim {
		lim = len(b)
	}
	for i := 0; i < lim; i++ {
		if a[i] != b[i] {
			break
		}
		prefixLen++
	}
	return prefixLen
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jobs

import (
	"fmt"
	"os"
	"regexp"
	"testing"
)

const testThis = "@k8s-bot test this"

// Make sure that our rerun commands match our triggers.
func TestJobTriggers(t *testing.T) {
	ja := &JobAgent{}
	if err := ja.load("../jobs.yaml"); err != nil {
		t.Fatalf("Could not load job configs: %v", err)
	}
	if len(ja.jobs) == 0 {
		t.Fatalf("No jobs found in jobs.yaml.")
	}
	for _, jobs := range ja.jobs {
		for i, job := range jobs {
			if job.Name == "" {
				t.Errorf("Job %v needs a name.", job)
				continue
			}
			if job.Context == "" {
				t.Errorf("Job %s needs a context.", job.Name)
			}
			if job.RerunCommand == "" || job.Trigger == "" {
				t.Errorf("Job %s needs a trigger and a rerun command.", job.Name)
				continue
			}
			// Check that the merge bot will run AlwaysRun jobs, otherwise it
			// will attempt to rerun forever.
			if job.AlwaysRun && !job.re.MatchString(testThis) {
				t.Errorf("AlwaysRun job %s: \"%s\" does not match regex \"%v\".", job.Name, testThis, job.Trigger)
			}
			// Check that the rerun command actually runs the job.
			if !job.re.MatchString(job.RerunCommand) {
				t.Errorf("For job %s: RerunCommand \"%s\" does not match regex \"%v\".", job.Name, job.RerunCommand, job.Trigger)
			}
			// Next check that the rerun command doesn't run any other jobs.
			for j, job2 := range jobs {
				if i == j {
					continue
				}
				if job2.re.MatchString(job.RerunCommand) {
					t.Errorf("RerunCommand \"%s\" from job %s matches \"%v\" from job %s but shouldn't.", job.RerunCommand, job.Name, job2.Trigger, job2.Name)
				}
			}
			// Ensure that jobs have a shell script of the same name.
			if s, err := os.Stat(fmt.Sprintf("../../jobs/%s.sh", job.Name)); err != nil {
				t.Errorf("Cannot find test-infra/jobs/%s.sh", job.Name)
			} else {
				if s.Mode()&0111 == 0 {
					t.Errorf("Not executable: %s.sh (%o)", job.Name, s.Mode()&0777)
				}
				if s.Mode()&0444 == 0 {
					t.Errorf("Not readable: %s.sh (%o)", job.Name, s.Mode()&0777)
				}
			}
		}
	}
}

func TestCommentBodyMatches(t *testing.T) {
	var testcases = []struct {
		repo         string
		body         string
		expectedJobs []string
	}{
		{
			"org/repo",
			"this is a random comment",
			[]string{},
		},
		{
			"org/repo",
			"ok to test",
			[]string{"gce", "unit"},
		},
		{
			"org/repo",
			"@k8s-bot test this",
			[]string{"gce", "unit", "gke"},
		},
		{
			"org/repo",
			"@k8s-bot unit test this",
			[]string{"unit"},
		},
		{
			"org/repo",
			"@k8s-bot federation test this",
			[]string{"federation"},
		},
		{
			"org/repo2",
			"@k8s-bot test this",
			[]string{"cadveapster"},
		},
		{
			"org/repo3",
			"@k8s-bot test this",
			[]string{},
		},
	}
	ja := &JobAgent{
		jobs: map[string][]JenkinsJob{
			"org/repo": {
				{
					Name:      "gce",
					re:        regexp.MustCompile(`@k8s-bot (gce )?test this`),
					AlwaysRun: true,
				},
				{
					Name:      "unit",
					re:        regexp.MustCompile(`@k8s-bot (unit )?test this`),
					AlwaysRun: true,
				},
				{
					Name:      "gke",
					re:        regexp.MustCompile(`@k8s-bot (gke )?test this`),
					AlwaysRun: false,
				},
				{
					Name:      "federation",
					re:        regexp.MustCompile(`@k8s-bot federation test this`),
					AlwaysRun: false,
				},
			},
			"org/repo2": {
				{
					Name:      "cadveapster",
					re:        regexp.MustCompile(`@k8s-bot test this`),
					AlwaysRun: true,
				},
			},
		},
	}
	for _, tc := range testcases {
		actualJobs := ja.MatchingJobs(tc.repo, tc.body, regexp.MustCompile(`ok to test`))
		match := true
		if len(actualJobs) != len(tc.expectedJobs) {
			match = false
		} else {
			for _, actualJob := range actualJobs {
				found := false
				for _, expectedJob := range tc.expectedJobs {
					if expectedJob == actualJob.Name {
						found = true
						break
					}
				}
				if !found {
					match = false
					break
				}
			}
		}
		if !match {
			t.Errorf("Wrong jobs for body %s. Got %v, expected %v.", tc.body, actualJobs, tc.expectedJobs)
		}
	}
}
<commit_msg>Check that jobs don't have matching contexts.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jobs

import (
	"fmt"
	"os"
	"regexp"
	"testing"
)

const testThis = "@k8s-bot test this"

// Make sure that our rerun commands match our triggers.
func TestJobTriggers(t *testing.T) {
	ja := &JobAgent{}
	if err := ja.load("../jobs.yaml"); err != nil {
		t.Fatalf("Could not load job configs: %v", err)
	}
	if len(ja.jobs) == 0 {
		t.Fatalf("No jobs found in jobs.yaml.")
	}
	for _, jobs := range ja.jobs {
		for i, job := range jobs {
			if job.Name == "" {
				t.Errorf("Job %v needs a name.", job)
				continue
			}
			if job.Context == "" {
				t.Errorf("Job %s needs a context.", job.Name)
			}
			if job.RerunCommand == "" || job.Trigger == "" {
				t.Errorf("Job %s needs a trigger and a rerun command.", job.Name)
				continue
			}
			// Check that the merge bot will run AlwaysRun jobs, otherwise it
			// will attempt to rerun forever.
			if job.AlwaysRun && !job.re.MatchString(testThis) {
				t.Errorf("AlwaysRun job %s: \"%s\" does not match regex \"%v\".", job.Name, testThis, job.Trigger)
			}
			// Check that the rerun command actually runs the job.
			if !job.re.MatchString(job.RerunCommand) {
				t.Errorf("For job %s: RerunCommand \"%s\" does not match regex \"%v\".", job.Name, job.RerunCommand, job.Trigger)
			}
			// Next check that the rerun command doesn't run any other jobs.
			for j, job2 := range jobs {
				if i == j {
					continue
				}
				if job.Context == job2.Context && i > j {
					t.Errorf("Jobs %s and %s have the same context: %s", job.Name, job2.Name, job.Context)
				}
				if job2.re.MatchString(job.RerunCommand) {
					t.Errorf("RerunCommand \"%s\" from job %s matches \"%v\" from job %s but shouldn't.", job.RerunCommand, job.Name, job2.Trigger, job2.Name)
				}
			}
			// Ensure that jobs have a shell script of the same name.
			if s, err := os.Stat(fmt.Sprintf("../../jobs/%s.sh", job.Name)); err != nil {
				t.Errorf("Cannot find test-infra/jobs/%s.sh", job.Name)
			} else {
				if s.Mode()&0111 == 0 {
					t.Errorf("Not executable: %s.sh (%o)", job.Name, s.Mode()&0777)
				}
				if s.Mode()&0444 == 0 {
					t.Errorf("Not readable: %s.sh (%o)", job.Name, s.Mode()&0777)
				}
			}
		}
	}
}

func TestCommentBodyMatches(t *testing.T) {
	var testcases = []struct {
		repo         string
		body         string
		expectedJobs []string
	}{
		{
			"org/repo",
			"this is a random comment",
			[]string{},
		},
		{
			"org/repo",
			"ok to test",
			[]string{"gce", "unit"},
		},
		{
			"org/repo",
			"@k8s-bot test this",
			[]string{"gce", "unit", "gke"},
		},
		{
			"org/repo",
			"@k8s-bot unit test this",
			[]string{"unit"},
		},
		{
			"org/repo",
			"@k8s-bot federation test this",
			[]string{"federation"},
		},
		{
			"org/repo2",
			"@k8s-bot test this",
			[]string{"cadveapster"},
		},
		{
			"org/repo3",
			"@k8s-bot test this",
			[]string{},
		},
	}
	ja := &JobAgent{
		jobs: map[string][]JenkinsJob{
			"org/repo": {
				{
					Name:      "gce",
					re:        regexp.MustCompile(`@k8s-bot (gce )?test this`),
					AlwaysRun: true,
				},
				{
					Name:      "unit",
					re:        regexp.MustCompile(`@k8s-bot (unit )?test this`),
					AlwaysRun: true,
				},
				{
					Name:      "gke",
					re:        regexp.MustCompile(`@k8s-bot (gke )?test this`),
					AlwaysRun: false,
				},
				{
					Name:      "federation",
					re:        regexp.MustCompile(`@k8s-bot federation test this`),
					AlwaysRun: false,
				},
			},
			"org/repo2": {
				{
					Name:      "cadveapster",
					re:        regexp.MustCompile(`@k8s-bot test this`),
					AlwaysRun: true,
				},
			},
		},
	}
	for _, tc := range testcases {
		actualJobs := ja.MatchingJobs(tc.repo, tc.body, regexp.MustCompile(`ok to test`))
		match := true
		if len(actualJobs) != len(tc.expectedJobs) {
			match = false
		} else {
			for _, actualJob := range actualJobs {
				found := false
				for _, expectedJob := range tc.expectedJobs {
					if expectedJob == actualJob.Name {
						found = true
						break
					}
				}
				if !found {
					match = false
					break
				}
			}
		}
		if !match {
			t.Errorf("Wrong jobs for body %s. Got %v, expected %v.", tc.body, actualJobs, tc.expectedJobs)
		}
	}
}
<|endoftext|>


<commit_before>package rpc

import (
	"errors"
	"koding/newkite/dnode"
	"net/url"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"
)

const redialDurationStart = 1 * time.Second
const redialDurationMax = 60 * time.Second

// Dial is a helper for creating a Client for just calling methods on the server.
// Do not use it if you want to handle methods on client side. Instead create a
// new Client, register your methods on Client.Dnode then call Client.Dial().
func Dial(url string, reconnect bool) (*Client, error) {
	c := NewClient()
	c.Reconnect = reconnect

	err := c.Dial(url)
	if err != nil {
		return nil, err
	}

	return c, nil
}

// Client is a dnode RPC client.
type Client struct {
	// Websocket connection
	Conn *websocket.Conn

	// Websocket connection options.
	Config *websocket.Config

	// Dnode message processor.
	dnode *dnode.Dnode

	// A space for saving/reading extra properties about this client.
	properties map[string]interface{}

	// Should we reconnect if disconnected?
	Reconnect bool

	// Time to wait before redial connection.
	redialDuration time.Duration

	// on connect/disconnect handlers are invoked after every
	// connect/disconnect.
	onConnectHandlers    []func()
	onDisconnectHandlers []func()

	// For protecting access over OnConnect and OnDisconnect handlers.
	m sync.RWMutex
}

// NewClient returns a pointer to new Client.
// You need to call Dial() before interacting with the Server.
func NewClient() *Client {
	// Must send an "Origin" header. Does not checked on server.
	origin, _ := url.Parse("")

	config := &websocket.Config{
		Version: websocket.ProtocolVersionHybi13,
		Origin:  origin,
		// Location will be set when dialing.
	}

	c := &Client{
		properties:     make(map[string]interface{}),
		redialDuration: redialDurationStart,
		Config:         config,
	}

	c.dnode = dnode.New(c)
	return c
}

func (c *Client) SetWrappers(wrapMethodArgs, wrapCallbackArgs dnode.Wrapper, runMethod, runCallback dnode.Runner, onError func(error)) {
	c.dnode.WrapMethodArgs = wrapMethodArgs
	c.dnode.WrapCallbackArgs = wrapCallbackArgs
	c.dnode.RunMethod = runMethod
	c.dnode.RunCallback = runCallback
	c.dnode.OnError = onError
}

// Dial connects to the dnode server on "url" and starts a goroutine
// that processes incoming messages.
//
// Do not forget to register your handlers on Client.Dnode
// before calling Dial() to prevent race conditions.
func (c *Client) Dial(serverURL string) error {
	var err error

	if c.Config.Location, err = url.Parse(serverURL); err != nil {
		return err
	}

	if err = c.dial(); err != nil {
		return err
	}

	go c.run()

	return nil
}

// dial makes a single Dial() and run onConnectHandlers if connects.
func (c *Client) dial() error {
	ws, err := websocket.DialConfig(c.Config)
	if err != nil {
		return err
	}

	// We are connected
	c.Conn = ws

	// Reset the wait time.
	c.redialDuration = redialDurationStart

	// Must be run in a goroutine because a handler may wait a response from
	// server.
	go c.callOnConnectHandlers()

	return nil
}

// DialForever connects to the server in background.
// If the connection drops, it reconnects again.
func (c *Client) DialForever(serverURL string) (err error) {
	if c.Config.Location, err = url.Parse(serverURL); err != nil {
		return
	}

	go c.dialForever()

	return
}

func (c *Client) dialForever() {
	for c.dial() != nil {
		if !c.Reconnect {
			return
		}

		c.sleep()
	}
	go c.run()
}

// run consumes incoming dnode messages. Reconnects if necessary.
func (c *Client) run() (err error) {
	for {
	running:
		err = c.dnode.Run()
		c.callOnDisconnectHandlers()
	dialAgain:
		if !c.Reconnect {
			break
		}

		err = c.dial()
		if err != nil {
			c.sleep()
			goto dialAgain
		}

		goto running
	}

	return err
}

// sleep is used to wait for a while between dial retries.
// Each time it is called the redialDuration is incremented.
func (c *Client) sleep() {
	time.Sleep(c.redialDuration)

	c.redialDuration *= 2
	if c.redialDuration > redialDurationMax {
		c.redialDuration = redialDurationMax
	}
}

// Close closes the underlying websocket connection.
func (c *Client) Close() {
	c.Conn.Close()
}

func (c *Client) Send(msg []byte) error {
	// println("\nSending...", string(msg))
	if c.Conn == nil {
		return errors.New("Not connected")
	}

	return websocket.Message.Send(c.Conn, string(msg))
}

func (c *Client) Receive() ([]byte, error) {
	// println("Receiving...")
	var msg []byte
	err := websocket.Message.Receive(c.Conn, &msg)
	// println("\nReceived:", string(msg))
	return msg, err
}

func (c *Client) RemoveCallback(id uint64) {
	c.dnode.RemoveCallback(id)
}

// RemoteAddr returns the host:port as string if server connection.
func (c *Client) RemoteAddr() string {
	if c.Conn.IsServerConn() {
		return c.Conn.Request().RemoteAddr
	}
	return ""
}

func (c *Client) Properties() map[string]interface{} {
	return c.properties
}

// Call calls a method with args on the dnode server.
func (c *Client) Call(method string, args ...interface{}) (map[string]dnode.Path, error) {
	return c.dnode.Call(method, args...)
}

// OnConnect registers a function to run on client connect.
func (c *Client) OnConnect(handler func()) {
	c.m.Lock()
	c.onConnectHandlers = append(c.onConnectHandlers, handler)
	c.m.Unlock()
}

// OnDisconnect registers a function to run on client disconnect.
func (c *Client) OnDisconnect(handler func()) {
	c.m.Lock()
	c.onDisconnectHandlers = append(c.onDisconnectHandlers, handler)
	c.m.Unlock()
}

// callOnConnectHandlers runs the registered connect handlers.
func (c *Client) callOnConnectHandlers() {
	c.m.RLock()
	for _, handler := range c.onConnectHandlers {
		func() {
			defer recover()
			handler()
		}()
	}
	c.m.RUnlock()
}

// callOnDisconnectHandlers runs the registered disconnect handlers.
func (c *Client) callOnDisconnectHandlers() {
	c.m.RLock()
	for _, handler := range c.onDisconnectHandlers {
		func() {
			defer recover()
			handler()
		}()
	}
	c.m.RUnlock()
}
<commit_msg>kite: add env flag for printing debug messages<commit_after>package rpc

import (
	"errors"
	"fmt"
	"koding/newkite/dnode"
	"net/url"
	"os"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"
)

const redialDurationStart = 1 * time.Second
const redialDurationMax = 60 * time.Second

// Dial is a helper for creating a Client for just calling methods on the server.
// Do not use it if you want to handle methods on client side. Instead create a
// new Client, register your methods on Client.Dnode then call Client.Dial().
func Dial(url string, reconnect bool) (*Client, error) {
	c := NewClient()
	c.Reconnect = reconnect

	err := c.Dial(url)
	if err != nil {
		return nil, err
	}

	return c, nil
}

// Client is a dnode RPC client.
type Client struct {
	// Websocket connection
	Conn *websocket.Conn

	// Websocket connection options.
	Config *websocket.Config

	// Dnode message processor.
	dnode *dnode.Dnode

	// A space for saving/reading extra properties about this client.
	properties map[string]interface{}

	// Should we reconnect if disconnected?
	Reconnect bool

	// Time to wait before redial connection.
	redialDuration time.Duration

	// on connect/disconnect handlers are invoked after every
	// connect/disconnect.
	onConnectHandlers    []func()
	onDisconnectHandlers []func()

	// For protecting access over OnConnect and OnDisconnect handlers.
	m sync.RWMutex
}

// NewClient returns a pointer to new Client.
// You need to call Dial() before interacting with the Server.
func NewClient() *Client {
	// Must send an "Origin" header. Does not checked on server.
	origin, _ := url.Parse("")

	config := &websocket.Config{
		Version: websocket.ProtocolVersionHybi13,
		Origin:  origin,
		// Location will be set when dialing.
	}

	c := &Client{
		properties:     make(map[string]interface{}),
		redialDuration: redialDurationStart,
		Config:         config,
	}

	c.dnode = dnode.New(c)
	return c
}

func (c *Client) SetWrappers(wrapMethodArgs, wrapCallbackArgs dnode.Wrapper, runMethod, runCallback dnode.Runner, onError func(error)) {
	c.dnode.WrapMethodArgs = wrapMethodArgs
	c.dnode.WrapCallbackArgs = wrapCallbackArgs
	c.dnode.RunMethod = runMethod
	c.dnode.RunCallback = runCallback
	c.dnode.OnError = onError
}

// Dial connects to the dnode server on "url" and starts a goroutine
// that processes incoming messages.
//
// Do not forget to register your handlers on Client.Dnode
// before calling Dial() to prevent race conditions.
func (c *Client) Dial(serverURL string) error {
	var err error

	if c.Config.Location, err = url.Parse(serverURL); err != nil {
		return err
	}

	if err = c.dial(); err != nil {
		return err
	}

	go c.run()

	return nil
}

// dial makes a single Dial() and run onConnectHandlers if connects.
func (c *Client) dial() error {
	ws, err := websocket.DialConfig(c.Config)
	if err != nil {
		return err
	}

	// We are connected
	c.Conn = ws

	// Reset the wait time.
	c.redialDuration = redialDurationStart

	// Must be run in a goroutine because a handler may wait a response from
	// server.
	go c.callOnConnectHandlers()

	return nil
}

// DialForever connects to the server in background.
// If the connection drops, it reconnects again.
func (c *Client) DialForever(serverURL string) (err error) {
	if c.Config.Location, err = url.Parse(serverURL); err != nil {
		return
	}

	go c.dialForever()

	return
}

func (c *Client) dialForever() {
	for c.dial() != nil {
		if !c.Reconnect {
			return
		}

		c.sleep()
	}
	go c.run()
}

// run consumes incoming dnode messages. Reconnects if necessary.
func (c *Client) run() (err error) {
	for {
	running:
		err = c.dnode.Run()
		c.callOnDisconnectHandlers()
	dialAgain:
		if !c.Reconnect {
			break
		}

		err = c.dial()
		if err != nil {
			c.sleep()
			goto dialAgain
		}

		goto running
	}

	return err
}

// sleep is used to wait for a while between dial retries.
// Each time it is called the redialDuration is incremented.
func (c *Client) sleep() {
	time.Sleep(c.redialDuration)

	c.redialDuration *= 2
	if c.redialDuration > redialDurationMax {
		c.redialDuration = redialDurationMax
	}
}

// Close closes the underlying websocket connection.
func (c *Client) Close() {
	c.Conn.Close()
}

func (c *Client) Send(msg []byte) error {
	if os.Getenv("DNODE_PRINT_SEND") != "" {
		fmt.Fprintf(os.Stderr, "\nSending: %s\n", string(msg))
	}

	if c.Conn == nil {
		return errors.New("Not connected")
	}

	return websocket.Message.Send(c.Conn, string(msg))
}

func (c *Client) Receive() ([]byte, error) {
	// println("Receiving...")
	var msg []byte
	err := websocket.Message.Receive(c.Conn, &msg)

	if os.Getenv("DNODE_PRINT_RECV") != "" {
		fmt.Fprintf(os.Stderr, "\nReceived: %s\n", string(msg))
	}

	return msg, err
}

func (c *Client) RemoveCallback(id uint64) {
	c.dnode.RemoveCallback(id)
}

// RemoteAddr returns the host:port as string if server connection.
func (c *Client) RemoteAddr() string {
	if c.Conn.IsServerConn() {
		return c.Conn.Request().RemoteAddr
	}
	return ""
}

func (c *Client) Properties() map[string]interface{} {
	return c.properties
}

// Call calls a method with args on the dnode server.
func (c *Client) Call(method string, args ...interface{}) (map[string]dnode.Path, error) {
	return c.dnode.Call(method, args...)
}

// OnConnect registers a function to run on client connect.
func (c *Client) OnConnect(handler func()) {
	c.m.Lock()
	c.onConnectHandlers = append(c.onConnectHandlers, handler)
	c.m.Unlock()
}

// OnDisconnect registers a function to run on client disconnect.
func (c *Client) OnDisconnect(handler func()) {
	c.m.Lock()
	c.onDisconnectHandlers = append(c.onDisconnectHandlers, handler)
	c.m.Unlock()
}

// callOnConnectHandlers runs the registered connect handlers.
func (c *Client) callOnConnectHandlers() {
	c.m.RLock()
	for _, handler := range c.onConnectHandlers {
		func() {
			defer recover()
			handler()
		}()
	}
	c.m.RUnlock()
}

// callOnDisconnectHandlers runs the registered disconnect handlers.
func (c *Client) callOnDisconnectHandlers() {
	c.m.RLock()
	for _, handler := range c.onDisconnectHandlers {
		func() {
			defer recover()
			handler()
		}()
	}
	c.m.RUnlock()
}
<|endoftext|>


<commit_before>package spvwallet

import (
	"bytes"
	"fmt"
	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/bloom"
	hd "github.com/btcsuite/btcutil/hdkeychain"
	"sync"
	"time"
)

const FlagPrefix = 0x00

type TxStore struct {
	Adrs           []btcutil.Address
	watchedScripts [][]byte
	addrMutex      *sync.Mutex
	cbMutex        *sync.Mutex

	Param *chaincfg.Params

	internalKey *hd.ExtendedKey
	externalKey *hd.ExtendedKey

	listeners []func(TransactionCallback)

	Datastore
}

func NewTxStore(p *chaincfg.Params, db Datastore, masterPrivKey *hd.ExtendedKey) (*TxStore, error) {
	// Derive keys using Bip44
	fourtyFour, err := masterPrivKey.Child(hd.HardenedKeyStart + 44)
	if err != nil {
		return nil, err
	}
	bitcoin, err := fourtyFour.Child(hd.HardenedKeyStart + 0)
	if err != nil {
		return nil, err
	}
	account, err := bitcoin.Child(hd.HardenedKeyStart + 0)
	if err != nil {
		return nil, err
	}
	external, err := account.Child(0)
	if err != nil {
		return nil, err
	}
	internal, err := account.Child(1)
	if err != nil {
		return nil, err
	}
	txs := &TxStore{
		Param:       p,
		externalKey: external,
		internalKey: internal,
		addrMutex:   new(sync.Mutex),
		cbMutex:     new(sync.Mutex),
		Datastore:   db,
	}
	err = txs.PopulateAdrs()
	if err != nil {
		return nil, err
	}
	return txs, nil
}

// ... or I'm gonna fade away
func (ts *TxStore) GimmeFilter() (*bloom.Filter, error) {
	ts.PopulateAdrs()

	// get all utxos to add outpoints to filter
	allUtxos, err := ts.Utxos().GetAll()
	if err != nil {
		return nil, err
	}

	allStxos, err := ts.Stxos().GetAll()
	if err != nil {
		return nil, err
	}
	ts.addrMutex.Lock()
	elem := uint32(len(ts.Adrs) + len(allUtxos) + len(allStxos))
	f := bloom.NewFilter(elem, 0, 0.0001, wire.BloomUpdateAll)

	// note there could be false positives since we're just looking
	// for the 20 byte PKH without the opcodes.
	for _, a := range ts.Adrs { // add 20-byte pubkeyhash
		f.Add(a.ScriptAddress())
	}
	ts.addrMutex.Unlock()
	for _, u := range allUtxos {
		f.AddOutPoint(&u.Op)
	}

	for _, s := range allStxos {
		f.AddOutPoint(&s.Utxo.Op)
	}
	for _, w := range ts.watchedScripts {
		_, addrs, _, err := txscript.ExtractPkScriptAddrs(w, ts.Param)
		if err != nil {
			continue
		}
		f.Add(addrs[0].ScriptAddress())
	}

	return f, nil
}

// GetDoubleSpends takes a transaction and compares it with
// all transactions in the db.  It returns a slice of all txids in the db
// which are double spent by the received tx.
func CheckDoubleSpends(
	argTx *wire.MsgTx, txs []*wire.MsgTx) ([]*chainhash.Hash, error) {

	var dubs []*chainhash.Hash // slice of all double-spent txs
	argTxid := argTx.TxHash()

	for _, compTx := range txs {
		compTxid := compTx.TxHash()
		// check if entire tx is dup
		if argTxid.IsEqual(&compTxid) {
			return nil, fmt.Errorf("tx %s is dup", argTxid.String())
		}
		// not dup, iterate through inputs of argTx
		for _, argIn := range argTx.TxIn {
			// iterate through inputs of compTx
			for _, compIn := range compTx.TxIn {
				if outPointsEqual(
					argIn.PreviousOutPoint, compIn.PreviousOutPoint) {
					// found double spend
					dubs = append(dubs, &compTxid)
					break // back to argIn loop
				}
			}
		}
	}
	return dubs, nil
}

// GetPendingInv returns an inv message containing all txs known to the
// db which are at height 0 (not known to be confirmed).
// This can be useful on startup or to rebroadcast unconfirmed txs.
func (ts *TxStore) GetPendingInv() (*wire.MsgInv, error) {
	// use a map (really a set) do avoid dupes
	txidMap := make(map[chainhash.Hash]struct{})

	utxos, err := ts.Utxos().GetAll() // get utxos from db
	if err != nil {
		return nil, err
	}
	stxos, err := ts.Stxos().GetAll() // get stxos from db
	if err != nil {
		return nil, err
	}

	// iterate through utxos, adding txids of anything with height 0
	for _, utxo := range utxos {
		if utxo.AtHeight == 0 {
			txidMap[utxo.Op.Hash] = struct{}{} // adds to map
		}
	}
	// do the same with stxos based on height at which spent
	for _, stxo := range stxos {
		if stxo.SpendHeight == 0 {
			txidMap[stxo.SpendTxid] = struct{}{}
		}
	}

	invMsg := wire.NewMsgInv()
	for txid := range txidMap {
		item := wire.NewInvVect(wire.InvTypeTx, &txid)
		err = invMsg.AddInvVect(item)
		if err != nil {
			if err != nil {
				return nil, err
			}
		}
	}

	// return inv message with all txids (maybe none)
	return invMsg, nil
}

// PopulateAdrs just puts a bunch of adrs in ram; it doesn't touch the DB
func (ts *TxStore) PopulateAdrs() error {
	ts.lookahead()
	keys := ts.GetKeys()
	ts.addrMutex.Lock()
	ts.Adrs = []btcutil.Address{}
	for _, k := range keys {
		addr, err := k.Address(ts.Param)
		if err != nil {
			continue
		}
		ts.Adrs = append(ts.Adrs, addr)
	}
	ts.watchedScripts, _ = ts.WatchedScripts().GetAll()
	ts.addrMutex.Unlock()
	return nil
}

// Ingest puts a tx into the DB atomically.  This can result in a
// gain, a loss, or no result.  Gain or loss in satoshis is returned.
func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
	var hits uint32
	var err error
	// Tx has been OK'd by SPV; check tx sanity
	utilTx := btcutil.NewTx(tx) // convert for validation
	// Checks basic stuff like there are inputs and ouputs
	err = blockchain.CheckTransactionSanity(utilTx)
	if err != nil {
		return hits, err
	}

	// Generate PKscripts for all addresses
	ts.addrMutex.Lock()
	PKscripts := make([][]byte, len(ts.Adrs))
	for i, _ := range ts.Adrs {
		// Iterate through all our addresses
		PKscripts[i], err = txscript.PayToAddrScript(ts.Adrs[i])
		if err != nil {
			return hits, err
		}
	}
	ts.addrMutex.Unlock()
	cachedSha := tx.TxHash()
	// Iterate through all outputs of this tx, see if we gain
	cb := TransactionCallback{Txid: cachedSha.CloneBytes(), Height: height}
	value := int64(0)
	matchesWatchOnly := false
	for i, txout := range tx.TxOut {
		out := TransactionOutput{ScriptPubKey: txout.PkScript, Value: txout.Value, Index: uint32(i)}
		for _, script := range PKscripts {
			if bytes.Equal(txout.PkScript, script) { // new utxo found
				ts.Keys().MarkKeyAsUsed(txout.PkScript)
				newop := wire.OutPoint{
					Hash:  cachedSha,
					Index: uint32(i),
				}
				newu := Utxo{
					AtHeight:     height,
					Value:        txout.Value,
					ScriptPubkey: txout.PkScript,
					Op:           newop,
					WatchOnly:    false,
				}
				value += newu.Value
				ts.Utxos().Put(newu)
				hits++
				break
			}
		}
		// Now check watched scripts
		for _, script := range ts.watchedScripts {
			if bytes.Equal(txout.PkScript, script) {
				newop := wire.OutPoint{
					Hash:  cachedSha,
					Index: uint32(i),
				}
				newu := Utxo{
					AtHeight:     height,
					Value:        txout.Value,
					ScriptPubkey: txout.PkScript,
					Op:           newop,
					WatchOnly:    true,
				}
				ts.Utxos().Put(newu)
				matchesWatchOnly = true
			}
		}
		cb.Outputs = append(cb.Outputs, out)
	}
	utxos, err := ts.Utxos().GetAll()
	if err != nil {
		return 0, err
	}
	for _, txin := range tx.TxIn {
		for i, u := range utxos {
			if outPointsEqual(txin.PreviousOutPoint, u.Op) {
				st := Stxo{
					Utxo:        u,
					SpendHeight: height,
					SpendTxid:   cachedSha,
				}
				ts.Stxos().Put(st)
				ts.Utxos().Delete(u)
				utxos = append(utxos[:i], utxos[i+1:]...)
				if !u.WatchOnly {
					value -= u.Value
					hits++
				} else {
					matchesWatchOnly = true
				}

				in := TransactionInput{
					OutpointHash:       u.Op.Hash.CloneBytes(),
					OutpointIndex:      u.Op.Index,
					LinkedScriptPubKey: u.ScriptPubkey,
					Value:              u.Value,
				}
				cb.Inputs = append(cb.Inputs, in)
				break
			}
		}
	}

	// If hits is nonzero it's a relevant tx and we should store it
	if hits > 0 || matchesWatchOnly {
		ts.cbMutex.Lock()
		_, txn, err := ts.Txns().Get(tx.TxHash())
		shouldCallback := false
		if err != nil {
			txn.Timestamp = time.Now()
			shouldCallback = true
		}
		// Let's check the height before committing so we don't allow rogue peers to send us a lose
		// tx that resets our height to zero.
		if txn.Height <= 0 {
			ts.Txns().Put(tx, int(value), int(height), txn.Timestamp, hits == 0)
			if height > 0 {
				shouldCallback = true
			}
		}
		if shouldCallback {
			// Callback on listeners
			for _, listener := range ts.listeners {
				listener(cb)
			}
		}
		ts.cbMutex.Unlock()
		ts.PopulateAdrs()
	}
	return hits, err
}

func (ts *TxStore) markAsDead(txid chainhash.Hash) error {
	stxos, err := ts.Stxos().GetAll()
	if err != nil {
		return err
	}
	// If an stxo is marked dead, move it back into the utxo table
	for _, s := range stxos {
		if txid.IsEqual(&s.SpendTxid) {
			err := ts.Stxos().Delete(s)
			if err != nil {
				return err
			}
			ts.Txns().MarkAsDead(txid)
			err = ts.Utxos().Put(s.Utxo)
			if err != nil {
				return err
			}
		}
	}
	utxos, err := ts.Utxos().GetAll()
	if err != nil {
		return err
	}
	// Dead utxos and just be deleted
	for _, u := range utxos {
		if txid.IsEqual(&u.Op.Hash) {
			err := ts.Utxos().Delete(u)
			if err != nil {
				return err
			}
			ts.Txns().MarkAsDead(txid)
		}
	}
	return nil
}

func (ts *TxStore) processReorg(lastGoodHeight uint32) error {
	txns, err := ts.Txns().GetAll(true)
	if err != nil {
		return err
	}
	for _, tx := range txns {
		if tx.Height > int32(lastGoodHeight) {
			txid, err := chainhash.NewHashFromStr(tx.Txid)
			if err != nil {
				log.Error(err)
				continue
			}
			err = ts.markAsDead(*txid)
			if err != nil {
				log.Error(err)
				continue
			}
		}
	}
	return nil
}

func outPointsEqual(a, b wire.OutPoint) bool {
	if !a.Hash.IsEqual(&b.Hash) {
		return false
	}
	return a.Index == b.Index
}
<commit_msg>Fix bugs marking txs as dead<commit_after>package spvwallet

import (
	"bytes"
	"fmt"
	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/bloom"
	hd "github.com/btcsuite/btcutil/hdkeychain"
	"sync"
	"time"
)

const FlagPrefix = 0x00

type TxStore struct {
	Adrs           []btcutil.Address
	watchedScripts [][]byte
	addrMutex      *sync.Mutex
	cbMutex        *sync.Mutex

	Param *chaincfg.Params

	internalKey *hd.ExtendedKey
	externalKey *hd.ExtendedKey

	listeners []func(TransactionCallback)

	Datastore
}

func NewTxStore(p *chaincfg.Params, db Datastore, masterPrivKey *hd.ExtendedKey) (*TxStore, error) {
	// Derive keys using Bip44
	fourtyFour, err := masterPrivKey.Child(hd.HardenedKeyStart + 44)
	if err != nil {
		return nil, err
	}
	bitcoin, err := fourtyFour.Child(hd.HardenedKeyStart + 0)
	if err != nil {
		return nil, err
	}
	account, err := bitcoin.Child(hd.HardenedKeyStart + 0)
	if err != nil {
		return nil, err
	}
	external, err := account.Child(0)
	if err != nil {
		return nil, err
	}
	internal, err := account.Child(1)
	if err != nil {
		return nil, err
	}
	txs := &TxStore{
		Param:       p,
		externalKey: external,
		internalKey: internal,
		addrMutex:   new(sync.Mutex),
		cbMutex:     new(sync.Mutex),
		Datastore:   db,
	}
	err = txs.PopulateAdrs()
	if err != nil {
		return nil, err
	}
	return txs, nil
}

// ... or I'm gonna fade away
func (ts *TxStore) GimmeFilter() (*bloom.Filter, error) {
	ts.PopulateAdrs()

	// get all utxos to add outpoints to filter
	allUtxos, err := ts.Utxos().GetAll()
	if err != nil {
		return nil, err
	}

	allStxos, err := ts.Stxos().GetAll()
	if err != nil {
		return nil, err
	}
	ts.addrMutex.Lock()
	elem := uint32(len(ts.Adrs) + len(allUtxos) + len(allStxos))
	f := bloom.NewFilter(elem, 0, 0.0001, wire.BloomUpdateAll)

	// note there could be false positives since we're just looking
	// for the 20 byte PKH without the opcodes.
	for _, a := range ts.Adrs { // add 20-byte pubkeyhash
		f.Add(a.ScriptAddress())
	}
	ts.addrMutex.Unlock()
	for _, u := range allUtxos {
		f.AddOutPoint(&u.Op)
	}

	for _, s := range allStxos {
		f.AddOutPoint(&s.Utxo.Op)
	}
	for _, w := range ts.watchedScripts {
		_, addrs, _, err := txscript.ExtractPkScriptAddrs(w, ts.Param)
		if err != nil {
			continue
		}
		f.Add(addrs[0].ScriptAddress())
	}

	return f, nil
}

// GetDoubleSpends takes a transaction and compares it with
// all transactions in the db.  It returns a slice of all txids in the db
// which are double spent by the received tx.
func CheckDoubleSpends(argTx *wire.MsgTx, txs []*wire.MsgTx) ([]*chainhash.Hash, error) {
	var dubs []*chainhash.Hash // slice of all double-spent txs
	argTxid := argTx.TxHash()
	for _, compTx := range txs {
		compTxid := compTx.TxHash()
		for _, argIn := range argTx.TxIn {
			// iterate through inputs of compTx
			for _, compIn := range compTx.TxIn {
				if outPointsEqual(argIn.PreviousOutPoint, compIn.PreviousOutPoint) && !compTxid.IsEqual(&argTxid) {
					// found double spend
					dubs = append(dubs, &compTxid)
					break // back to argIn loop
				}
			}
		}
	}
	return dubs, nil
}

// GetPendingInv returns an inv message containing all txs known to the
// db which are at height 0 (not known to be confirmed).
// This can be useful on startup or to rebroadcast unconfirmed txs.
func (ts *TxStore) GetPendingInv() (*wire.MsgInv, error) {
	// use a map (really a set) do avoid dupes
	txidMap := make(map[chainhash.Hash]struct{})

	utxos, err := ts.Utxos().GetAll() // get utxos from db
	if err != nil {
		return nil, err
	}
	stxos, err := ts.Stxos().GetAll() // get stxos from db
	if err != nil {
		return nil, err
	}

	// iterate through utxos, adding txids of anything with height 0
	for _, utxo := range utxos {
		if utxo.AtHeight == 0 {
			txidMap[utxo.Op.Hash] = struct{}{} // adds to map
		}
	}
	// do the same with stxos based on height at which spent
	for _, stxo := range stxos {
		if stxo.SpendHeight == 0 {
			txidMap[stxo.SpendTxid] = struct{}{}
		}
	}

	invMsg := wire.NewMsgInv()
	for txid := range txidMap {
		item := wire.NewInvVect(wire.InvTypeTx, &txid)
		err = invMsg.AddInvVect(item)
		if err != nil {
			if err != nil {
				return nil, err
			}
		}
	}

	// return inv message with all txids (maybe none)
	return invMsg, nil
}

// PopulateAdrs just puts a bunch of adrs in ram; it doesn't touch the DB
func (ts *TxStore) PopulateAdrs() error {
	ts.lookahead()
	keys := ts.GetKeys()
	ts.addrMutex.Lock()
	ts.Adrs = []btcutil.Address{}
	for _, k := range keys {
		addr, err := k.Address(ts.Param)
		if err != nil {
			continue
		}
		ts.Adrs = append(ts.Adrs, addr)
	}
	ts.watchedScripts, _ = ts.WatchedScripts().GetAll()
	ts.addrMutex.Unlock()
	return nil
}

// Ingest puts a tx into the DB atomically.  This can result in a
// gain, a loss, or no result.  Gain or loss in satoshis is returned.
func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
	var hits uint32
	var err error
	// Tx has been OK'd by SPV; check tx sanity
	utilTx := btcutil.NewTx(tx) // convert for validation
	// Checks basic stuff like there are inputs and ouputs
	err = blockchain.CheckTransactionSanity(utilTx)
	if err != nil {
		return hits, err
	}

	// Generate PKscripts for all addresses
	ts.addrMutex.Lock()
	PKscripts := make([][]byte, len(ts.Adrs))
	for i, _ := range ts.Adrs {
		// Iterate through all our addresses
		PKscripts[i], err = txscript.PayToAddrScript(ts.Adrs[i])
		if err != nil {
			return hits, err
		}
	}
	ts.addrMutex.Unlock()
	cachedSha := tx.TxHash()
	// Iterate through all outputs of this tx, see if we gain
	cb := TransactionCallback{Txid: cachedSha.CloneBytes(), Height: height}
	value := int64(0)
	matchesWatchOnly := false
	for i, txout := range tx.TxOut {
		out := TransactionOutput{ScriptPubKey: txout.PkScript, Value: txout.Value, Index: uint32(i)}
		for _, script := range PKscripts {
			if bytes.Equal(txout.PkScript, script) { // new utxo found
				ts.Keys().MarkKeyAsUsed(txout.PkScript)
				newop := wire.OutPoint{
					Hash:  cachedSha,
					Index: uint32(i),
				}
				newu := Utxo{
					AtHeight:     height,
					Value:        txout.Value,
					ScriptPubkey: txout.PkScript,
					Op:           newop,
					WatchOnly:    false,
				}
				value += newu.Value
				ts.Utxos().Put(newu)
				hits++
				break
			}
		}
		// Now check watched scripts
		for _, script := range ts.watchedScripts {
			if bytes.Equal(txout.PkScript, script) {
				newop := wire.OutPoint{
					Hash:  cachedSha,
					Index: uint32(i),
				}
				newu := Utxo{
					AtHeight:     height,
					Value:        txout.Value,
					ScriptPubkey: txout.PkScript,
					Op:           newop,
					WatchOnly:    true,
				}
				ts.Utxos().Put(newu)
				matchesWatchOnly = true
			}
		}
		cb.Outputs = append(cb.Outputs, out)
	}
	utxos, err := ts.Utxos().GetAll()
	if err != nil {
		return 0, err
	}
	for _, txin := range tx.TxIn {
		for i, u := range utxos {
			if outPointsEqual(txin.PreviousOutPoint, u.Op) {
				st := Stxo{
					Utxo:        u,
					SpendHeight: height,
					SpendTxid:   cachedSha,
				}
				ts.Stxos().Put(st)
				ts.Utxos().Delete(u)
				utxos = append(utxos[:i], utxos[i+1:]...)
				if !u.WatchOnly {
					value -= u.Value
					hits++
				} else {
					matchesWatchOnly = true
				}

				in := TransactionInput{
					OutpointHash:       u.Op.Hash.CloneBytes(),
					OutpointIndex:      u.Op.Index,
					LinkedScriptPubKey: u.ScriptPubkey,
					Value:              u.Value,
				}
				cb.Inputs = append(cb.Inputs, in)
				break
			}
		}
	}

	// If hits is nonzero it's a relevant tx and we should store it
	if hits > 0 || matchesWatchOnly {
		ts.cbMutex.Lock()
		_, txn, err := ts.Txns().Get(tx.TxHash())
		shouldCallback := false
		if err != nil {
			txn.Timestamp = time.Now()
			shouldCallback = true
		}
		// Let's check the height before committing so we don't allow rogue peers to send us a lose
		// tx that resets our height to zero.
		if txn.Height <= 0 {
			ts.Txns().Put(tx, int(value), int(height), txn.Timestamp, hits == 0)
			if height > 0 {
				shouldCallback = true
			}
		}
		if shouldCallback {
			// Callback on listeners
			for _, listener := range ts.listeners {
				listener(cb)
			}
		}
		ts.cbMutex.Unlock()
		ts.PopulateAdrs()
	}
	return hits, err
}

func (ts *TxStore) markAsDead(txid chainhash.Hash) error {
	stxos, err := ts.Stxos().GetAll()
	if err != nil {
		return err
	}
	markStxoAsDead := func(s Stxo) error {
		err := ts.Stxos().Delete(s)
		if err != nil {
			return err
		}
		err = ts.Txns().MarkAsDead(txid)
		if err != nil {
			return err
		}
		return nil
	}
	for _, s := range stxos {
		// If an stxo is marked dead, move it back into the utxo table
		if txid.IsEqual(&s.SpendTxid) {
			if err := markStxoAsDead(s); err != nil {
				return err
			}
		}
		// If an dependency of the spend is dead then mark the spend as dead
		if txid.IsEqual(&s.Utxo.Op.Hash) {
			if err := markStxoAsDead(s); err != nil {
				return err
			}
			if err := ts.markAsDead(s.SpendTxid); err != nil {
				return err
			}
		}
	}
	utxos, err := ts.Utxos().GetAll()
	if err != nil {
		return err
	}
	// Dead utxos should just be deleted
	for _, u := range utxos {
		if txid.IsEqual(&u.Op.Hash) {
			err := ts.Utxos().Delete(u)
			if err != nil {
				return err
			}
			ts.Txns().MarkAsDead(txid)
		}
	}
	return nil
}

func (ts *TxStore) processReorg(lastGoodHeight uint32) error {
	txns, err := ts.Txns().GetAll(true)
	if err != nil {
		return err
	}
	for i := len(txns) - 1; i >= 0; i-- {
		if txns[i].Height > int32(lastGoodHeight) {
			txid, err := chainhash.NewHashFromStr(txns[i].Txid)
			if err != nil {
				log.Error(err)
				continue
			}
			err = ts.markAsDead(*txid)
			if err != nil {
				log.Error(err)
				continue
			}
		}
	}
	return nil
}

func outPointsEqual(a, b wire.OutPoint) bool {
	if !a.Hash.IsEqual(&b.Hash) {
		return false
	}
	return a.Index == b.Index
}
<|endoftext|>