<commit_before>package storage

import (
	"context"
	"time"

	"github.com/brocaar/lorawan"

	"github.com/brocaar/lora-app-server/internal/common"
	"github.com/brocaar/loraserver/api/ns"

	"github.com/jmoiron/sqlx"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

// NetworkServer defines the information to connect to a network-server.
type NetworkServer struct {
	ID        int64     `db:"id"`
	CreatedAt time.Time `db:"created_at"`
	UpdatedAt time.Time `db:"updated_at"`
	Name      string    `db:"name"`
	Server    string    `db:"server"`
}

// Validate validates the network-server data.
func (ns NetworkServer) Validate() error {
	return nil
}

// CreateNetworkServer creates the given network-server.
func CreateNetworkServer(db sqlx.Queryer, n *NetworkServer) error {
	if err := n.Validate(); err != nil {
		return errors.Wrap(err, "validation error")
	}

	now := time.Now()
	n.CreatedAt = now
	n.UpdatedAt = now

	err := sqlx.Get(db, &n.ID, `
		insert into network_server (
			created_at,
			updated_at,
			name,
			server
		) values ($1, $2, $3, $4)
		returning id`,
		n.CreatedAt,
		n.UpdatedAt,
		n.Name,
		n.Server,
	)
	if err != nil {
		return handlePSQLError(Insert, err, "insert error")
	}

	nsClient, err := common.NetworkServerPool.Get(n.Server)
	if err != nil {
		return errors.Wrap(err, "get network-server client error")
	}

	_, err = nsClient.CreateRoutingProfile(context.Background(), &ns.CreateRoutingProfileRequest{
		RoutingProfile: &ns.RoutingProfile{
			RoutingProfileID: common.ApplicationServerID,
			AsID:             common.ApplicationServerServer,
		},
	})
	if err != nil {
		log.WithError(err).Error("network-server create routing-profile api error")
		return handleGrpcError(err, "create routing-profile error")
	}

	log.WithFields(log.Fields{
		"id":     n.ID,
		"name":   n.Name,
		"server": n.Server,
	}).Info("network-server created")
	return nil
}

// GetNetworkServer returns the network-server matching the given id.
func GetNetworkServer(db sqlx.Queryer, id int64) (NetworkServer, error) {
	var ns NetworkServer
	err := sqlx.Get(db, &ns, "select * from network_server where id = $1", id)
	if err != nil {
		return ns, handlePSQLError(Select, err, "select error")
	}

	return ns, nil
}

// UpdateNetworkServer updates the given network-server.
func UpdateNetworkServer(db sqlx.Execer, n *NetworkServer) error {
	if err := n.Validate(); err != nil {
		return errors.Wrap(err, "validation error")
	}

	n.UpdatedAt = time.Now()

	res, err := db.Exec(`
		update network_server
		set
			updated_at = $2,
			name = $3,
			server = $4
		where id = $1`,
		n.ID,
		n.UpdatedAt,
		n.Name,
		n.Server,
	)
	if err != nil {
		return handlePSQLError(Update, err, "update error")
	}

	ra, err := res.RowsAffected()
	if err != nil {
		return errors.Wrap(err, "get rows affected error")
	}
	if ra == 0 {
		return ErrDoesNotExist
	}

	nsClient, err := common.NetworkServerPool.Get(n.Server)
	if err != nil {
		return errors.Wrap(err, "get network-server client error")
	}

	_, err = nsClient.UpdateRoutingProfile(context.Background(), &ns.UpdateRoutingProfileRequest{
		RoutingProfile: &ns.RoutingProfile{
			RoutingProfileID: common.ApplicationServerID,
			AsID:             common.ApplicationServerServer,
		},
	})
	if err != nil {
		log.WithError(err).Error("network-server update routing-profile api error")
		return handleGrpcError(err, "update routing-profile error")
	}

	log.WithFields(log.Fields{
		"id":     n.ID,
		"name":   n.Name,
		"server": n.Server,
	}).Info("network-server updated")
	return nil
}

// DeleteNetworkServer deletes the network-server matching the given id.
func DeleteNetworkServer(db sqlx.Ext, id int64) error {
	n, err := GetNetworkServer(db, id)
	if err != nil {
		return errors.Wrap(err, "get network-server error")
	}

	res, err := db.Exec("delete from network_server where id = $1", id)
	if err != nil {
		return handlePSQLError(Delete, err, "delete error")
	}
	ra, err := res.RowsAffected()
	if err != nil {
		return errors.Wrap(err, "get rows affected error")
	}
	if ra == 0 {
		return ErrDoesNotExist
	}

	nsClient, err := common.NetworkServerPool.Get(n.Server)
	if err != nil {
		return errors.Wrap(err, "get network-server client error")
	}

	_, err = nsClient.DeleteRoutingProfile(context.Background(), &ns.DeleteRoutingProfileRequest{
		RoutingProfileID: common.ApplicationServerID,
	})
	if err != nil {
		log.WithError(err).Error("network-server delete routing-profile api error")
		return handleGrpcError(err, "delete routing-profile error")
	}

	log.WithField("id", id).Info("network-server deleted")
	return nil
}

// GetNetworkServerCount returns the total number of network-servers.
func GetNetworkServerCount(db sqlx.Queryer) (int, error) {
	var count int
	err := sqlx.Get(db, &count, "select count(*) from network_server")
	if err != nil {
		return 0, handlePSQLError(Select, err, "select error")
	}

	return count, nil
}

// GetNetworkServerCountForOrganizationID returns the total number of
// network-servers accessible for the given organization id.
// A network-server is accessible for an organization when it is used by one
// of its service-profiles.
func GetNetworkServerCountForOrganizationID(db sqlx.Queryer, organizationID int64) (int, error) {
	var count int
	err := sqlx.Get(db, &count, `
		select
			count (ns.*)
		from
			network_server ns
		inner join service_profile sp
			on sp.network_server_id = ns.id
		where
			sp.organization_id = $1`,
		organizationID,
	)
	if err != nil {
		return 0, handlePSQLError(Select, err, "select error")
	}
	return count, nil
}

// GetNetworkServers returns a slice of network-servers.
func GetNetworkServers(db sqlx.Queryer, limit, offset int) ([]NetworkServer, error) {
	var nss []NetworkServer
	err := sqlx.Select(db, &nss, `
		select *
		from network_server
		order by name
		limit $1 offset $2`,
		limit,
		offset,
	)
	if err != nil {
		return nil, handlePSQLError(Select, err, "select error")
	}

	return nss, nil
}

// GetNetworkServersForOrganizationID returns a slice of network-server
// accessible for the given organization id.
// A network-server is accessible for an organization when it is used by one
// of its service-profiles.
func GetNetworkServersForOrganizationID(db sqlx.Queryer, organizationID int64, limit, offset int) ([]NetworkServer, error) {
	var nss []NetworkServer
	err := sqlx.Select(db, &nss, `
		select
			ns.*
		from
			network_server ns
		inner join service_profile sp
			on sp.network_server_id = ns.id
		where
			sp.organization_id = $1
		order by name
		limit $2 offset $3`,
		organizationID,
		limit,
		offset,
	)
	if err != nil {
		return nil, handlePSQLError(Select, err, "select error")
	}

	return nss, nil
}

// GetNetworkServerForDevEUI returns the network-server for the given DevEUI.
func GetNetworkServerForDevEUI(db sqlx.Queryer, devEUI lorawan.EUI64) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from
			network_server ns
		inner join device_profile dp
			on dp.network_server_id = ns.id
		inner join device d
			on d.device_profile_id = dp.device_profile_id
		where
			d.dev_eui = $1`,
		devEUI,
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}

// GetNetworkServerForDeviceProfileID returns the network-server for the given
// device-profile id.
func GetNetworkServerForDeviceProfileID(db sqlx.Queryer, id string) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from
			network_server ns
		inner join device_profile dp
			on dp.network_server_id = ns.id
		where
			dp.device_profile_id = $1`,
		id,
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}

// GetNetworkServerForServiceProfileID returns the network-server for the given
// service-profile id.
func GetNetworkServerForServiceProfileID(db sqlx.Queryer, id string) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from
			network_server ns
		inner join service_profile sp
			on sp.network_server_id = ns.id
		where
			sp.service_profile_id = $1`,
		id,
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}

// GetNetworkServerForGatewayMAC returns the network-server for a given
// gateway mac.
func GetNetworkServerForGatewayMAC(db sqlx.Queryer, mac lorawan.EUI64) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from network_server ns
		inner join gateway gw
			on gw.network_server_id = ns.id
		where
			gw.mac = $1`,
		mac[:],
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}
<commit_msg>Fix network-server duplication.<commit_after>package storage

import (
	"context"
	"time"

	"github.com/brocaar/lorawan"

	"github.com/brocaar/lora-app-server/internal/common"
	"github.com/brocaar/loraserver/api/ns"

	"github.com/jmoiron/sqlx"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

// NetworkServer defines the information to connect to a network-server.
type NetworkServer struct {
	ID        int64     `db:"id"`
	CreatedAt time.Time `db:"created_at"`
	UpdatedAt time.Time `db:"updated_at"`
	Name      string    `db:"name"`
	Server    string    `db:"server"`
}

// Validate validates the network-server data.
func (ns NetworkServer) Validate() error {
	return nil
}

// CreateNetworkServer creates the given network-server.
func CreateNetworkServer(db sqlx.Queryer, n *NetworkServer) error {
	if err := n.Validate(); err != nil {
		return errors.Wrap(err, "validation error")
	}

	now := time.Now()
	n.CreatedAt = now
	n.UpdatedAt = now

	err := sqlx.Get(db, &n.ID, `
		insert into network_server (
			created_at,
			updated_at,
			name,
			server
		) values ($1, $2, $3, $4)
		returning id`,
		n.CreatedAt,
		n.UpdatedAt,
		n.Name,
		n.Server,
	)
	if err != nil {
		return handlePSQLError(Insert, err, "insert error")
	}

	nsClient, err := common.NetworkServerPool.Get(n.Server)
	if err != nil {
		return errors.Wrap(err, "get network-server client error")
	}

	_, err = nsClient.CreateRoutingProfile(context.Background(), &ns.CreateRoutingProfileRequest{
		RoutingProfile: &ns.RoutingProfile{
			RoutingProfileID: common.ApplicationServerID,
			AsID:             common.ApplicationServerServer,
		},
	})
	if err != nil {
		log.WithError(err).Error("network-server create routing-profile api error")
		return handleGrpcError(err, "create routing-profile error")
	}

	log.WithFields(log.Fields{
		"id":     n.ID,
		"name":   n.Name,
		"server": n.Server,
	}).Info("network-server created")
	return nil
}

// GetNetworkServer returns the network-server matching the given id.
func GetNetworkServer(db sqlx.Queryer, id int64) (NetworkServer, error) {
	var ns NetworkServer
	err := sqlx.Get(db, &ns, "select * from network_server where id = $1", id)
	if err != nil {
		return ns, handlePSQLError(Select, err, "select error")
	}

	return ns, nil
}

// UpdateNetworkServer updates the given network-server.
func UpdateNetworkServer(db sqlx.Execer, n *NetworkServer) error {
	if err := n.Validate(); err != nil {
		return errors.Wrap(err, "validation error")
	}

	n.UpdatedAt = time.Now()

	res, err := db.Exec(`
		update network_server
		set
			updated_at = $2,
			name = $3,
			server = $4
		where id = $1`,
		n.ID,
		n.UpdatedAt,
		n.Name,
		n.Server,
	)
	if err != nil {
		return handlePSQLError(Update, err, "update error")
	}

	ra, err := res.RowsAffected()
	if err != nil {
		return errors.Wrap(err, "get rows affected error")
	}
	if ra == 0 {
		return ErrDoesNotExist
	}

	nsClient, err := common.NetworkServerPool.Get(n.Server)
	if err != nil {
		return errors.Wrap(err, "get network-server client error")
	}

	_, err = nsClient.UpdateRoutingProfile(context.Background(), &ns.UpdateRoutingProfileRequest{
		RoutingProfile: &ns.RoutingProfile{
			RoutingProfileID: common.ApplicationServerID,
			AsID:             common.ApplicationServerServer,
		},
	})
	if err != nil {
		log.WithError(err).Error("network-server update routing-profile api error")
		return handleGrpcError(err, "update routing-profile error")
	}

	log.WithFields(log.Fields{
		"id":     n.ID,
		"name":   n.Name,
		"server": n.Server,
	}).Info("network-server updated")
	return nil
}

// DeleteNetworkServer deletes the network-server matching the given id.
func DeleteNetworkServer(db sqlx.Ext, id int64) error {
	n, err := GetNetworkServer(db, id)
	if err != nil {
		return errors.Wrap(err, "get network-server error")
	}

	res, err := db.Exec("delete from network_server where id = $1", id)
	if err != nil {
		return handlePSQLError(Delete, err, "delete error")
	}
	ra, err := res.RowsAffected()
	if err != nil {
		return errors.Wrap(err, "get rows affected error")
	}
	if ra == 0 {
		return ErrDoesNotExist
	}

	nsClient, err := common.NetworkServerPool.Get(n.Server)
	if err != nil {
		return errors.Wrap(err, "get network-server client error")
	}

	_, err = nsClient.DeleteRoutingProfile(context.Background(), &ns.DeleteRoutingProfileRequest{
		RoutingProfileID: common.ApplicationServerID,
	})
	if err != nil {
		log.WithError(err).Error("network-server delete routing-profile api error")
		return handleGrpcError(err, "delete routing-profile error")
	}

	log.WithField("id", id).Info("network-server deleted")
	return nil
}

// GetNetworkServerCount returns the total number of network-servers.
func GetNetworkServerCount(db sqlx.Queryer) (int, error) {
	var count int
	err := sqlx.Get(db, &count, "select count(*) from network_server")
	if err != nil {
		return 0, handlePSQLError(Select, err, "select error")
	}

	return count, nil
}

// GetNetworkServerCountForOrganizationID returns the total number of
// network-servers accessible for the given organization id.
// A network-server is accessible for an organization when it is used by one
// of its service-profiles.
func GetNetworkServerCountForOrganizationID(db sqlx.Queryer, organizationID int64) (int, error) {
	var count int
	err := sqlx.Get(db, &count, `
		select
			count (distinct ns.id)
		from
			network_server ns
		inner join service_profile sp
			on sp.network_server_id = ns.id
		where
			sp.organization_id = $1`,
		organizationID,
	)
	if err != nil {
		return 0, handlePSQLError(Select, err, "select error")
	}
	return count, nil
}

// GetNetworkServers returns a slice of network-servers.
func GetNetworkServers(db sqlx.Queryer, limit, offset int) ([]NetworkServer, error) {
	var nss []NetworkServer
	err := sqlx.Select(db, &nss, `
		select *
		from network_server
		order by name
		limit $1 offset $2`,
		limit,
		offset,
	)
	if err != nil {
		return nil, handlePSQLError(Select, err, "select error")
	}

	return nss, nil
}

// GetNetworkServersForOrganizationID returns a slice of network-server
// accessible for the given organization id.
// A network-server is accessible for an organization when it is used by one
// of its service-profiles.
func GetNetworkServersForOrganizationID(db sqlx.Queryer, organizationID int64, limit, offset int) ([]NetworkServer, error) {
	var nss []NetworkServer
	err := sqlx.Select(db, &nss, `
		select
			ns.*
		from
			network_server ns
		inner join service_profile sp
			on sp.network_server_id = ns.id
		where
			sp.organization_id = $1
		group by ns.id
		order by name
		limit $2 offset $3`,
		organizationID,
		limit,
		offset,
	)
	if err != nil {
		return nil, handlePSQLError(Select, err, "select error")
	}

	return nss, nil
}

// GetNetworkServerForDevEUI returns the network-server for the given DevEUI.
func GetNetworkServerForDevEUI(db sqlx.Queryer, devEUI lorawan.EUI64) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from
			network_server ns
		inner join device_profile dp
			on dp.network_server_id = ns.id
		inner join device d
			on d.device_profile_id = dp.device_profile_id
		where
			d.dev_eui = $1`,
		devEUI,
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}

// GetNetworkServerForDeviceProfileID returns the network-server for the given
// device-profile id.
func GetNetworkServerForDeviceProfileID(db sqlx.Queryer, id string) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from
			network_server ns
		inner join device_profile dp
			on dp.network_server_id = ns.id
		where
			dp.device_profile_id = $1`,
		id,
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}

// GetNetworkServerForServiceProfileID returns the network-server for the given
// service-profile id.
func GetNetworkServerForServiceProfileID(db sqlx.Queryer, id string) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from
			network_server ns
		inner join service_profile sp
			on sp.network_server_id = ns.id
		where
			sp.service_profile_id = $1`,
		id,
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}

// GetNetworkServerForGatewayMAC returns the network-server for a given
// gateway mac.
func GetNetworkServerForGatewayMAC(db sqlx.Queryer, mac lorawan.EUI64) (NetworkServer, error) {
	var n NetworkServer
	err := sqlx.Get(db, &n, `
		select
			ns.*
		from network_server ns
		inner join gateway gw
			on gw.network_server_id = ns.id
		where
			gw.mac = $1`,
		mac[:],
	)
	if err != nil {
		return n, handlePSQLError(Select, err, "select error")
	}
	return n, nil
}
<|endoftext|>


<commit_before>package cssrefs

import (
	"strings"
	"testing"
)

func TestAll(t *testing.T) {

	// the required resources to be found in the example
	reqs := [...]Reference{
		{URI: "fineprint.css", Token: "css"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.eot", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.eot?#iefix", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.woff", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.ttf", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.svg#glyphicons_halflingsregular", Token: "font"},
		{URI: "../img/light_honeycomb_@2X.png", Token: "img"},
		{URI: "../img/light_honeycomb.png", Token: "img"},
	}

	// example HTML reader
	reader := strings.NewReader(`

        @import url("fineprint.css") print;

        @font-face {
          font-family: 'Glyphicons Halflings';
          src: url('../fonts/bootstrap/glyphicons-halflings-regular.eot');
          src: url('../fonts/bootstrap/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'),
            url('../fonts/bootstrap/glyphicons-halflings-regular.woff') format('woff'),
            url('../fonts/bootstrap/glyphicons-halflings-regular.ttf') format('truetype'),
            url('../fonts/bootstrap/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg');
        }

        body {
            background: url('../img/light_honeycomb_@2X.png');
            background-image: url('../img/light_honeycomb.png');
            background-size: 270px 289px;
            background-repeat: repeat;
        }
	`)

	// get the refs from the implementation
	refs := All(reader)

	need, have := len(reqs), len(refs)
	if need != have {
		t.Errorf("Wrong number of refs returned. need: %d , have: %d", need, have)
	}

	// loop through and verify URI and Token names
	for i, req := range reqs {
		ref := refs[i]

		if req.URI != ref.URI {
			t.Errorf("Mismatch URI detected. need: %s , have: %s", req.URI, ref.URI)
		}

		if req.Token != ref.Token {
			t.Errorf("Mismatch Token detected. need: %s , have: %s", req.Token, ref.Token)
		}
	}
}
<commit_msg>expanding the test to include a non-url bearing background<commit_after>package cssrefs

import (
	"strings"
	"testing"
)

func TestAll(t *testing.T) {

	// the required resources to be found in the example
	reqs := [...]Reference{
		{URI: "fineprint.css", Token: "css"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.eot", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.eot?#iefix", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.woff", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.ttf", Token: "font"},
		{URI: "../fonts/bootstrap/glyphicons-halflings-regular.svg#glyphicons_halflingsregular", Token: "font"},
		{URI: "../img/light_honeycomb_@2X.png", Token: "img"},
		{URI: "../img/light_honeycomb.png", Token: "img"},
	}

	// example HTML reader
	reader := strings.NewReader(`

        @import url("fineprint.css") print;

        @font-face {
          font-family: 'Glyphicons Halflings';
          src: url('../fonts/bootstrap/glyphicons-halflings-regular.eot');
          src: url('../fonts/bootstrap/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'),
            url('../fonts/bootstrap/glyphicons-halflings-regular.woff') format('woff'),
            url('../fonts/bootstrap/glyphicons-halflings-regular.ttf') format('truetype'),
            url('../fonts/bootstrap/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg');
        }

        .foo {
            background: red;
        }

        body {
            background: url('../img/light_honeycomb_@2X.png');
            background-image: url('../img/light_honeycomb.png');
            background-size: 270px 289px;
            background-repeat: repeat;
        }
	`)

	// get the refs from the implementation
	refs := All(reader)

	need, have := len(reqs), len(refs)
	if need != have {
		t.Errorf("Wrong number of refs returned. need: %d , have: %d", need, have)
	}

	// loop through and verify URI and Token names
	for i, req := range reqs {
		ref := refs[i]

		if req.URI != ref.URI {
			t.Errorf("Mismatch URI detected. need: %s , have: %s", req.URI, ref.URI)
		}

		if req.Token != ref.Token {
			t.Errorf("Mismatch Token detected. need: %s , have: %s", req.Token, ref.Token)
		}
	}
}
<|endoftext|>


<commit_before>// Package radar loads crime data from the City of Portland and makes it
// available to search using WGS84 coordinates.

package radar

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"strconv"

	"github.com/unit3/kdtree"
)

// One half mile in the WGS84 coordinate system.
//
// This is a rough estimate of distance derived from the fact that one degree
// of latitude or longitude is 70 miles wide at the equator. This distance is
// constant outside of the equator for latitude, but not for longitude.
const HALF_MILE = 0.00714

// A Point represents a latitude and longitude coordinate pair.
type Point struct {
	Lat float64
	Lng float64
}

type Points []*Point

type CsvRow []string

type CsvRows []CsvRow

type Coordinates []float64

// We store a slice of all the types of crime in the CSV data. This isn't used
// anywhere in the code but is something we make available to clients.
type CrimeTypes []string

func (types CrimeTypes) Contains(crimeType string) bool {
	for _, t := range types {
		if crimeType == t {
			return true
		}
	}
	return false
}

// Data for a single crime in the City's CSV data (one row).
type Crime struct {
	Id   int64
	Date string
	Time string
	Type string
}

// String formats a string version of a Crime.
func (c Crime) String() string {
	return fmt.Sprintf("(%v, %v, %v, %v)", c.Id, c.Date, c.Time, c.Type)
}

type Crimes []*Crime

// A location in the City's data with a coordinate at which crimes occurred.
type CrimeLocation struct {
	Point  *Point
	Crimes []*Crime
}

// This will help us find the CrimeLocation that a kd-tree node refers to.
type LocationLookup map[string]*CrimeLocation

// getOrCreateFromCsvRow gets an existing CrimeLocation for the coordinate
// stored in "row", or creates a CrimeLocation for that coordinate if one does
// not exist.
func (locs LocationLookup) getOrCreateFromCsvRow(row CsvRow) (*CrimeLocation, error) {
	var location *CrimeLocation
	var pointExists bool
	coords, err := floatCoordsFromRow(row)
	if err != nil {
		return nil, err
	}
	key := GetCoordinateKey(coords[0], coords[1])
	location, pointExists = locs[key]
	if !pointExists {
		point := Point{coords[0], coords[1]}
		location = &CrimeLocation{&point, make([]*Crime, 0)}
		locs[key] = location
	}
	return location, nil
}

// The result of a search for crimes near a location.
type SearchResult struct {
	Query     *Point
	Locations []*CrimeLocation
}

// Points returns all of the coordinates of a SearchResult's LocationLookup.
func (r SearchResult) Points() Points {
	points := make(Points, len(r.Locations))
	for _, loc := range r.Locations {
		points = append(points, loc.Point)
	}
	return points
}

// Crimes returns all of the Crimes in a SearchResult.
func (r SearchResult) Crimes() Crimes {
	crimes := make(Crimes, 0)
	for _, loc := range r.Locations {
		for _, crime := range loc.Crimes {
			crimes = append(crimes, crime)
		}
	}
	return crimes
}

// ToJson returns a SearchResult marshalled to JSON bytes.
// XXX: This is terrible but gained several hundred requests/sec over json.Marshall.
func (r SearchResult) ToJson() ([]byte, error) {
	buf := new(bytes.Buffer)
	buf.WriteString(fmt.Sprintf(`{"query":{"lat":%v,"lng":%v},"locations":[`, r.Query.Lat, r.Query.Lng))
	totalLocations := len(r.Locations)

	for x, location := range r.Locations {
		total := len(location.Crimes)
		buf.WriteString(fmt.Sprintf(`{"point":{"lat":%v,"lng":%v},`, location.Point.Lat, location.Point.Lng))
		buf.WriteString(`"crimes":[`)
		line := `{"id":%v,"date":"%v","time":"%v","type":"%v"}`
		for i, crime := range location.Crimes {
			isLast := i == total-1
			buf.WriteString(fmt.Sprintf(line, crime.Id, crime.Date, crime.Time, crime.Type))
			if (total > 1) && !isLast {
				buf.WriteString(",")
			}
		}
		buf.WriteString("]}")
		isLast :=  x == totalLocations-1
		if (totalLocations > 1) && !isLast {
			buf.WriteString(",")
		}
	}
	buf.WriteString("]}")
	return buf.Bytes(), nil
}

// An object that can find crimes near a WGS84 coordinate.
type CrimeFinder struct {
	LocationLookup LocationLookup
	CrimeTypes     CrimeTypes
	Tree           *kdtree.Tree
}

// Locations returned a slice of all the CrimeLocations in this CrimeFinder
func (finder *CrimeFinder) Locations() []*CrimeLocation {
	locations := make([]*CrimeLocation, 0)
	for _, location := range finder.LocationLookup {
		locations = append(locations, location)
	}
	return locations
}

// FindNear returns a SearchResult containing LocationLookup within a half-mile of ``query``
func (finder *CrimeFinder) FindNear(query Point) (SearchResult, error) {
	nearby := SearchResult{}
	nearby.Query = &query
	nearby.Locations = make([]*CrimeLocation, 0)
	ranges := map[int]kdtree.Range{
		0: {query.Lat - HALF_MILE, query.Lat + HALF_MILE},
		1: {query.Lng - HALF_MILE, query.Lng + HALF_MILE}}
	results, err := finder.Tree.FindRange(ranges)
	if err != nil {
		return nearby, err
	}
	for i := 0; i < len(results); i++ {
		node := results[i]
		// If we have a record for this coordinate, add it to ``nearby``.
		key := GetCoordinateKey(node.Coordinates[0], node.Coordinates[1])
		location, exists := finder.LocationLookup[key]
		if exists {
			nearby.Locations = append(nearby.Locations, location)
		}
	}
	return nearby, nil
}

// All returns a SearchResult containing all LocationLookup in the CrimeFinder.
func (finder *CrimeFinder) All() SearchResult {
	all := SearchResult{}
	all.Locations = finder.Locations()
	return all
}

// loadFromCsv hydrates a CrimeFinder from CSV data.
func (finder *CrimeFinder) loadFromCsv(rows CsvRows) error {
	locations := make(LocationLookup)
	numCrimes := 0
	for _, row := range rows {
		location, err := locations.getOrCreateFromCsvRow(row)
		if err != nil {
			continue
		}
		// Parse the "id" column as an int64
		id, err := strconv.ParseInt(row[0], 0, 64)
		if err != nil {
			continue
		}
		crimeType := string(row[3])
		if !finder.CrimeTypes.Contains(crimeType) {
			finder.CrimeTypes = append(finder.CrimeTypes, crimeType)
		}
		location.Crimes = append(location.Crimes, &Crime{id, row[1], row[2], crimeType})
		numCrimes += 1
	}
	log.Printf("Loaded %v crimes and %v locations", numCrimes, len(locations))
	finder.LocationLookup = locations
	return nil
}

// NewCrimeFinder creates a new CrimeFinder loaded from CSV data.
func NewCrimeFinder(filename string) (CrimeFinder, error) {
	var err error
	finder := CrimeFinder{}
	rows, err := readCrimes(filename)
	if err != nil {
		return finder, err
	}
	err = finder.loadFromCsv(rows)
	if err != nil {
		return finder, err
	}
	nodes := make([]*kdtree.Node, 0)
	for _, location := range finder.LocationLookup {
		node := kdtree.Node{}
		node.Coordinates = Coordinates{location.Point.Lat, location.Point.Lng}
		nodes = append(nodes, &node)
	}
	finder.Tree = kdtree.BuildTree(nodes)
	return finder, nil
}

// GetCoordinateKey returns a pair of float64 coordinates as strings.
func GetCoordinateKey(x float64, y float64) string {
	return fmt.Sprintf("%v,%v", x, y)
}

// isFloat checks if a string is coercible to a float.
func isFloat(s string) bool {
	_, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return false
	}
	return true
}

// readCrimes reads CSV data from a file identified by filename.
func readCrimes(filename string) (CsvRows, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	reader := csv.NewReader(f)
	reader.TrailingComma = true
	rows, err := reader.ReadAll()
	if err != nil {
		return nil, err
	}

	filteredRows := make(CsvRows, 0)
	for _, row := range rows {
		if row[8] == "" || row[9] == "" {
			continue
		}
		if !isFloat(row[8]) || !isFloat(row[9]) {
			continue
		}
		filteredRows = append(filteredRows, row)
	}

	return filteredRows, nil
}

// floatForCol tries to coerce a specific column of a CSV file into float64.
func floatForCol(col int, row CsvRow) (float64, error) {
	val := row[col]
	id := row[0]
	f, err := strconv.ParseFloat(val, 64)
	if err != nil {
		log.Printf("Could not parse column %v of row %v. Bad value: %v", col, id, val)
		return 0, err
	}
	return f, nil
}

// floatCoordsFromRow tries to return a coordinate pair for a row of CSV data.
func floatCoordsFromRow(row CsvRow) (Coordinates, error) {
	coords := make(Coordinates, 2)
	latitudeColumn := 8
	longitudeColumn := 9
	lat, err := floatForCol(latitudeColumn, row)
	if err != nil {
		return nil, err
	}
	coords[0] = lat
	lng, err := floatForCol(longitudeColumn, row)
	if err != nil {
		return nil, err
	}
	coords[1] = lng
	return coords, nil
}
<commit_msg>Use a separate constant for 1/2 mile of longitude<commit_after>// Package radar loads crime data from the City of Portland and makes it
// available to search using WGS84 coordinates.

package radar

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"strconv"

	"github.com/unit3/kdtree"
)

// One half mile of latitude in the WGS84 coordinate system.
const HALF_MILE_LAT = 0.00714

// One half mile of longitude in the WGS84 coordinate system.
const HALF_MILE_LNG = 0.00724

// A Point represents a latitude and longitude coordinate pair.
type Point struct {
	Lat float64
	Lng float64
}

type Points []*Point

type CsvRow []string

type CsvRows []CsvRow

type Coordinates []float64

// We store a slice of all the types of crime in the CSV data. This isn't used
// anywhere in the code but is something we make available to clients.
type CrimeTypes []string

func (types CrimeTypes) Contains(crimeType string) bool {
	for _, t := range types {
		if crimeType == t {
			return true
		}
	}
	return false
}

// Data for a single crime in the City's CSV data (one row).
type Crime struct {
	Id   int64
	Date string
	Time string
	Type string
}

// String formats a string version of a Crime.
func (c Crime) String() string {
	return fmt.Sprintf("(%v, %v, %v, %v)", c.Id, c.Date, c.Time, c.Type)
}

type Crimes []*Crime

// A location in the City's data with a coordinate at which crimes occurred.
type CrimeLocation struct {
	Point  *Point
	Crimes []*Crime
}

// This will help us find the CrimeLocation that a kd-tree node refers to.
type LocationLookup map[string]*CrimeLocation

// getOrCreateFromCsvRow gets an existing CrimeLocation for the coordinate
// stored in "row", or creates a CrimeLocation for that coordinate if one does
// not exist.
func (locs LocationLookup) getOrCreateFromCsvRow(row CsvRow) (*CrimeLocation, error) {
	var location *CrimeLocation
	var pointExists bool
	coords, err := floatCoordsFromRow(row)
	if err != nil {
		return nil, err
	}
	key := GetCoordinateKey(coords[0], coords[1])
	location, pointExists = locs[key]
	if !pointExists {
		point := Point{coords[0], coords[1]}
		location = &CrimeLocation{&point, make([]*Crime, 0)}
		locs[key] = location
	}
	return location, nil
}

// The result of a search for crimes near a location.
type SearchResult struct {
	Query     *Point
	Locations []*CrimeLocation
}

// Points returns all of the coordinates of a SearchResult's LocationLookup.
func (r SearchResult) Points() Points {
	points := make(Points, len(r.Locations))
	for _, loc := range r.Locations {
		points = append(points, loc.Point)
	}
	return points
}

// Crimes returns all of the Crimes in a SearchResult.
func (r SearchResult) Crimes() Crimes {
	crimes := make(Crimes, 0)
	for _, loc := range r.Locations {
		for _, crime := range loc.Crimes {
			crimes = append(crimes, crime)
		}
	}
	return crimes
}

// ToJson returns a SearchResult marshalled to JSON bytes.
// XXX: This is terrible but gained several hundred requests/sec over json.Marshall.
func (r SearchResult) ToJson() ([]byte, error) {
	buf := new(bytes.Buffer)
	buf.WriteString(fmt.Sprintf(`{"query":{"lat":%v,"lng":%v},"locations":[`, r.Query.Lat, r.Query.Lng))
	totalLocations := len(r.Locations)

	for x, location := range r.Locations {
		total := len(location.Crimes)
		buf.WriteString(fmt.Sprintf(`{"point":{"lat":%v,"lng":%v},`, location.Point.Lat, location.Point.Lng))
		buf.WriteString(`"crimes":[`)
		line := `{"id":%v,"date":"%v","time":"%v","type":"%v"}`
		for i, crime := range location.Crimes {
			isLast := i == total-1
			buf.WriteString(fmt.Sprintf(line, crime.Id, crime.Date, crime.Time, crime.Type))
			if (total > 1) && !isLast {
				buf.WriteString(",")
			}
		}
		buf.WriteString("]}")
		isLast :=  x == totalLocations-1
		if (totalLocations > 1) && !isLast {
			buf.WriteString(",")
		}
	}
	buf.WriteString("]}")
	return buf.Bytes(), nil
}

// An object that can find crimes near a WGS84 coordinate.
type CrimeFinder struct {
	LocationLookup LocationLookup
	CrimeTypes     CrimeTypes
	Tree           *kdtree.Tree
}

// Locations returned a slice of all the CrimeLocations in this CrimeFinder
func (finder *CrimeFinder) Locations() []*CrimeLocation {
	locations := make([]*CrimeLocation, 0)
	for _, location := range finder.LocationLookup {
		locations = append(locations, location)
	}
	return locations
}

// FindNear returns a SearchResult containing LocationLookup within a half-mile of ``query``
func (finder *CrimeFinder) FindNear(query Point) (SearchResult, error) {
	nearby := SearchResult{}
	nearby.Query = &query
	nearby.Locations = make([]*CrimeLocation, 0)
	ranges := map[int]kdtree.Range{
		0: {query.Lat - HALF_MILE_LAT, query.Lat + HALF_MILE_LAT},
		1: {query.Lng - HALF_MILE_LNG, query.Lng + HALF_MILE_LNG}}
	results, err := finder.Tree.FindRange(ranges)
	if err != nil {
		return nearby, err
	}
	for i := 0; i < len(results); i++ {
		node := results[i]
		// If we have a record for this coordinate, add it to ``nearby``.
		key := GetCoordinateKey(node.Coordinates[0], node.Coordinates[1])
		location, exists := finder.LocationLookup[key]
		if exists {
			nearby.Locations = append(nearby.Locations, location)
		}
	}
	return nearby, nil
}

// All returns a SearchResult containing all LocationLookup in the CrimeFinder.
func (finder *CrimeFinder) All() SearchResult {
	all := SearchResult{}
	all.Locations = finder.Locations()
	return all
}

// loadFromCsv hydrates a CrimeFinder from CSV data.
func (finder *CrimeFinder) loadFromCsv(rows CsvRows) error {
	locations := make(LocationLookup)
	numCrimes := 0
	for _, row := range rows {
		location, err := locations.getOrCreateFromCsvRow(row)
		if err != nil {
			continue
		}
		// Parse the "id" column as an int64
		id, err := strconv.ParseInt(row[0], 0, 64)
		if err != nil {
			continue
		}
		crimeType := string(row[3])
		if !finder.CrimeTypes.Contains(crimeType) {
			finder.CrimeTypes = append(finder.CrimeTypes, crimeType)
		}
		location.Crimes = append(location.Crimes, &Crime{id, row[1], row[2], crimeType})
		numCrimes += 1
	}
	log.Printf("Loaded %v crimes and %v locations", numCrimes, len(locations))
	finder.LocationLookup = locations
	return nil
}

// NewCrimeFinder creates a new CrimeFinder loaded from CSV data.
func NewCrimeFinder(filename string) (CrimeFinder, error) {
	var err error
	finder := CrimeFinder{}
	rows, err := readCrimes(filename)
	if err != nil {
		return finder, err
	}
	err = finder.loadFromCsv(rows)
	if err != nil {
		return finder, err
	}
	nodes := make([]*kdtree.Node, 0)
	for _, location := range finder.LocationLookup {
		node := kdtree.Node{}
		node.Coordinates = Coordinates{location.Point.Lat, location.Point.Lng}
		nodes = append(nodes, &node)
	}
	finder.Tree = kdtree.BuildTree(nodes)
	return finder, nil
}

// GetCoordinateKey returns a pair of float64 coordinates as strings.
func GetCoordinateKey(x float64, y float64) string {
	return fmt.Sprintf("%v,%v", x, y)
}

// isFloat checks if a string is coercible to a float.
func isFloat(s string) bool {
	_, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return false
	}
	return true
}

// readCrimes reads CSV data from a file identified by filename.
func readCrimes(filename string) (CsvRows, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	reader := csv.NewReader(f)
	reader.TrailingComma = true
	rows, err := reader.ReadAll()
	if err != nil {
		return nil, err
	}

	filteredRows := make(CsvRows, 0)
	for _, row := range rows {
		if row[8] == "" || row[9] == "" {
			continue
		}
		if !isFloat(row[8]) || !isFloat(row[9]) {
			continue
		}
		filteredRows = append(filteredRows, row)
	}

	return filteredRows, nil
}

// floatForCol tries to coerce a specific column of a CSV file into float64.
func floatForCol(col int, row CsvRow) (float64, error) {
	val := row[col]
	id := row[0]
	f, err := strconv.ParseFloat(val, 64)
	if err != nil {
		log.Printf("Could not parse column %v of row %v. Bad value: %v", col, id, val)
		return 0, err
	}
	return f, nil
}

// floatCoordsFromRow tries to return a coordinate pair for a row of CSV data.
func floatCoordsFromRow(row CsvRow) (Coordinates, error) {
	coords := make(Coordinates, 2)
	latitudeColumn := 8
	longitudeColumn := 9
	lat, err := floatForCol(latitudeColumn, row)
	if err != nil {
		return nil, err
	}
	coords[0] = lat
	lng, err := floatForCol(longitudeColumn, row)
	if err != nil {
		return nil, err
	}
	coords[1] = lng
	return coords, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The Fractal Team Authors
// This file is part of the fractal project.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package dpos

import (
	"fmt"
	"math/big"
	"strings"
	"testing"
)

var (
	big0              = big.NewInt(0)
	big1              = big.NewInt(1)
	big2              = big.NewInt(2)
	big3              = big.NewInt(3)
	big10             = big.NewInt(10)
	minStakeCandidate = new(big.Int).Mul(DefaultConfig.CandidateMinQuantity, DefaultConfig.unitStake())
	minStakeVote      = new(big.Int).Mul(DefaultConfig.VoterMinQuantity, DefaultConfig.unitStake())
)

func TestCandiate(t *testing.T) {
	ldb, function := newTestLDB()
	db, err := NewLDB(ldb)
	defer function()
	if err != nil {
		panic(fmt.Errorf("create db failed --- %v", err))
	}
	sys := &System{
		config: DefaultConfig,
		IDB:    db,
	}
	_ = sys
	for index, candidate := range candidates {
		if err := db.SetState(&GlobalState{
			Epoch:         uint64(index),
			PreEpoch:      uint64(index),
			TotalQuantity: big.NewInt(0),
		}); err != nil {
			panic(fmt.Errorf("SetState --- %v", err))
		}
		if err := sys.IDB.SetAvailableQuantity(uint64(index), candidate, new(big.Int).Mul(big10, minStakeCandidate)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, strings.Repeat(fmt.Sprintf("www.%v.com", candidate), int(DefaultConfig.MaxURLLen)), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "invalid url") {
			panic(fmt.Sprintf("RegCandidate invalid url %v mismatch", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), big1, uint64(index), 0); !strings.Contains(err.Error(), "non divisibility") {
			panic(fmt.Sprintf("RegCandidate invalid stake %v mismatch", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big0, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "insufficient") {
			panic(fmt.Sprintf("RegCandidate invalid stake %v mismatch", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("RegCandidate %v", err))
		}

		if ncandidates, _ := sys.GetCandidates(uint64(index)); len(ncandidates) != 1 {
			panic(fmt.Sprintf("GetCandidates mismatch"))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidate); candidateInfo.Quantity.Cmp(DefaultConfig.CandidateMinQuantity) != 0 || candidateInfo.TotalQuantity.Cmp(DefaultConfig.CandidateMinQuantity) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		}

		if gstate, _ := sys.GetState(uint64(index)); gstate.TotalQuantity.Cmp(DefaultConfig.CandidateMinQuantity) != 0 {
			panic(fmt.Sprintf("GetState mismatch"))
		}

		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("RegCandidate invalid name %v mismatch", err))
		}

		if err := sys.UpdateCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big2, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("UpdateCandidate %v", err))
		}

		if ncandidates, _ := sys.GetCandidates(uint64(index)); len(ncandidates) != 1 {
			panic(fmt.Sprintf("GetCandidates num mismatch"))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidate); candidateInfo.Quantity.Cmp(new(big.Int).Mul(big3, DefaultConfig.CandidateMinQuantity)) != 0 || candidateInfo.TotalQuantity.Cmp(new(big.Int).Mul(big3, DefaultConfig.CandidateMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate quantity mismatch"))
		}

		if gstate, _ := sys.GetState(uint64(index)); gstate.TotalQuantity.Cmp(new(big.Int).Mul(big3, DefaultConfig.CandidateMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetState mismatch"))
		}

		if err := sys.UnregCandidate(uint64(index), candidate, uint64(index), 0); err != nil {
			panic(fmt.Sprintf("UnregCandidate %v", err))
		}

		// if err := sys.RefundCandidate(uint64(index), candidate, uint64(index)); err == nil {
		// 	panic(fmt.Sprintf("RefundCandidate %v", err))
		// }

		if ncandidates, _ := sys.GetCandidates(uint64(index)); len(ncandidates) != 1 {
			panic(fmt.Sprintf("UnregCandidate %v mismatch", err))
		}
	}

}

func TestVote(t *testing.T) {
	ldb, function := newTestLDB()
	db, err := NewLDB(ldb)
	defer function()
	if err != nil {
		panic(fmt.Errorf("create db failed --- %v", err))
	}
	sys := &System{
		config: DefaultConfig,
		IDB:    db,
	}
	_ = sys

	for index, candidate := range candidates {
		if err := db.SetState(&GlobalState{
			Epoch:         uint64(index),
			PreEpoch:      uint64(index),
			TotalQuantity: big.NewInt(0),
		}); err != nil {
			panic(fmt.Errorf("SetState --- %v", err))
		}
		if err := sys.IDB.SetAvailableQuantity(uint64(index), candidate, new(big.Int).Mul(big10, minStakeCandidate)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("RegCandidate %v", err))
		}
		if err := sys.RefundCandidate(uint64(index), candidate, uint64(index), 0); err == nil {
			panic(fmt.Sprintf("RefundCandidate %v", err))
		}
	}

	for index, voter := range voters {
		if err := sys.IDB.SetAvailableQuantity(uint64(index), voter, new(big.Int).Mul(big10, minStakeVote)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.VoteCandidate(uint64(index), voter, "test", new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("VoteCandidate invalid candidate %v mismatch", err))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], big1, uint64(index), 0); !strings.Contains(err.Error(), "non divisibility") {
			panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big0, minStakeVote), uint64(index), 0); !strings.Contains(err.Error(), "insufficient") {
			panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big1, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big2, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		} else if candidateInfo.Type != Normal {
			panic(fmt.Sprintf("GetCandidate  type mismatch"))
		}
	}
}

func TestCandidateVote(t *testing.T) {
	ldb, function := newTestLDB()
	db, err := NewLDB(ldb)
	defer function()
	if err != nil {
		panic(fmt.Errorf("create db failed --- %v", err))
	}
	sys := &System{
		config: DefaultConfig,
		IDB:    db,
	}
	_ = sys

	for index, candidate := range candidates {
		if err := db.SetState(&GlobalState{
			Epoch:         uint64(index),
			PreEpoch:      uint64(index),
			TotalQuantity: big.NewInt(0),
		}); err != nil {
			panic(fmt.Errorf("SetState --- %v", err))
		}
		if err := sys.IDB.SetAvailableQuantity(uint64(index), candidate, new(big.Int).Mul(big10, minStakeCandidate)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("RegCandidate %v", err))
		}
		// if err := sys.UnregCandidate(uint64(index), candidate, uint64(index)); err != nil {
		// 	panic(fmt.Sprintf("UnregCandidate %v", err))
		// }
	}

	for index, voter := range voters {
		if err := sys.IDB.SetAvailableQuantity(uint64(index), voter, new(big.Int).Mul(big10, minStakeVote)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.VoteCandidate(uint64(index), voter, "test", new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("VoteCandidate invalid candidate %v mismatch", err))
		}

		// if err := sys.VoteCandidate(uint64(index), voter, candidates[index], big1, uint64(index)); !strings.Contains(err.Error(), "not in normal") {
		// 	panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		// }

		// if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index)); !strings.Contains(err.Error(), "insufficient") {
		// 	panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		// }

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big1, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big2, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		} else if candidateInfo.Type != Normal {
			panic(fmt.Sprintf("GetCandidate  type mismatch"))
		}
	}
}
<commit_msg>increase dpos unit test (#482)<commit_after>// Copyright 2018 The Fractal Team Authors
// This file is part of the fractal project.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package dpos

import (
	"fmt"
	"math/big"
	"strings"
	"testing"
)

var (
	big0              = big.NewInt(0)
	big1              = big.NewInt(1)
	big2              = big.NewInt(2)
	big3              = big.NewInt(3)
	big10             = big.NewInt(10)
	minStakeCandidate = new(big.Int).Mul(DefaultConfig.CandidateMinQuantity, DefaultConfig.unitStake())
	minStakeVote      = new(big.Int).Mul(DefaultConfig.VoterMinQuantity, DefaultConfig.unitStake())
)

func TestCandiate(t *testing.T) {
	ldb, function := newTestLDB()
	db, err := NewLDB(ldb)
	defer function()
	if err != nil {
		panic(fmt.Errorf("create db failed --- %v", err))
	}
	sys := &System{
		config: DefaultConfig,
		IDB:    db,
	}
	_ = sys
	for index, candidate := range candidates {
		if err := db.SetState(&GlobalState{
			Epoch:         uint64(index),
			PreEpoch:      uint64(index),
			TotalQuantity: big.NewInt(0),
		}); err != nil {
			panic(fmt.Errorf("SetState --- %v", err))
		}
		if err := sys.IDB.SetAvailableQuantity(uint64(index), candidate, new(big.Int).Mul(big10, minStakeCandidate)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}

		if err := sys.RefundCandidate(uint64(index), candidate, uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("RefundCandidate %v", err))
		}

		if err := sys.UpdateCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big2, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("UpdateCandidate %v", err))
		}

		if err := sys.UnregCandidate(uint64(index), candidate, uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("UnregCandidate %v", err))
		}

		if err := sys.RegCandidate(uint64(index), candidate, strings.Repeat(fmt.Sprintf("www.%v.com", candidate), int(DefaultConfig.MaxURLLen)), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "invalid url") {
			panic(fmt.Sprintf("RegCandidate invalid url %v mismatch", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), big1, uint64(index), 0); !strings.Contains(err.Error(), "non divisibility") {
			panic(fmt.Sprintf("RegCandidate invalid stake %v mismatch", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big0, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "insufficient") {
			panic(fmt.Sprintf("RegCandidate invalid stake %v mismatch", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("RegCandidate %v", err))
		}

		if ncandidates, _ := sys.GetCandidates(uint64(index)); len(ncandidates) != 1 {
			panic(fmt.Sprintf("GetCandidates mismatch"))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidate); candidateInfo.Quantity.Cmp(DefaultConfig.CandidateMinQuantity) != 0 || candidateInfo.TotalQuantity.Cmp(DefaultConfig.CandidateMinQuantity) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		}

		if gstate, _ := sys.GetState(uint64(index)); gstate.TotalQuantity.Cmp(DefaultConfig.CandidateMinQuantity) != 0 {
			panic(fmt.Sprintf("GetState mismatch"))
		}

		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("RegCandidate %v", err))
		}

		if err := sys.UpdateCandidate(uint64(index), candidate, strings.Repeat(fmt.Sprintf("www.%v.com", candidate), int(DefaultConfig.MaxURLLen)), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "invalid url") {
			panic(fmt.Sprintf("UpdateCandidate invalid url %v mismatch", err))
		}

		if err := sys.UpdateCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), big1, uint64(index), 0); !strings.Contains(err.Error(), "non divisibility") {
			panic(fmt.Sprintf("UpdateCandidate invalid stake %v mismatch", err))
		}

		if err := sys.UpdateCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big2, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("UpdateCandidate %v", err))
		}

		if ncandidates, _ := sys.GetCandidates(uint64(index)); len(ncandidates) != 1 {
			panic(fmt.Sprintf("GetCandidates num mismatch"))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidate); candidateInfo.Quantity.Cmp(new(big.Int).Mul(big3, DefaultConfig.CandidateMinQuantity)) != 0 || candidateInfo.TotalQuantity.Cmp(new(big.Int).Mul(big3, DefaultConfig.CandidateMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate quantity mismatch"))
		}

		if gstate, _ := sys.GetState(uint64(index)); gstate.TotalQuantity.Cmp(new(big.Int).Mul(big3, DefaultConfig.CandidateMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetState mismatch"))
		}

		if err := sys.RefundCandidate(uint64(index), candidate, uint64(index), 0); !strings.Contains(err.Error(), "not in freeze") {
			panic(fmt.Sprintf("RefundCandidate %v", err))
		}

		if err := sys.UnregCandidate(uint64(index), candidate, uint64(index), 0); err != nil {
			panic(fmt.Sprintf("UnregCandidate %v", err))
		}

		if err := sys.UnregCandidate(uint64(index), candidate, uint64(index), 0); !strings.Contains(err.Error(), "not in normal") {
			panic(fmt.Sprintf("UnregCandidate %v", err))
		}

		if err := sys.UpdateCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big2, minStakeCandidate), uint64(index), 0); !strings.Contains(err.Error(), "not in normal") {
			panic(fmt.Sprintf("UpdateCandidate %v", err))
		}

		if err := sys.RefundCandidate(uint64(index), candidate, uint64(index), 0); !strings.Contains(err.Error(), "not arrived") {
			panic(fmt.Sprintf("RefundCandidate %v", err))
		}

		if ncandidates, _ := sys.GetCandidates(uint64(index)); len(ncandidates) != 1 {
			panic(fmt.Sprintf("UnregCandidate %v mismatch", err))
		}
	}

}

func TestVote(t *testing.T) {
	ldb, function := newTestLDB()
	db, err := NewLDB(ldb)
	defer function()
	if err != nil {
		panic(fmt.Errorf("create db failed --- %v", err))
	}
	sys := &System{
		config: DefaultConfig,
		IDB:    db,
	}
	_ = sys

	for index, candidate := range candidates {
		if err := db.SetState(&GlobalState{
			Epoch:         uint64(index),
			PreEpoch:      uint64(index),
			TotalQuantity: big.NewInt(0),
		}); err != nil {
			panic(fmt.Errorf("SetState --- %v", err))
		}
		if err := sys.IDB.SetAvailableQuantity(uint64(index), candidate, new(big.Int).Mul(big10, minStakeCandidate)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("RegCandidate %v", err))
		}
		if err := sys.RefundCandidate(uint64(index), candidate, uint64(index), 0); err == nil {
			panic(fmt.Sprintf("RefundCandidate %v", err))
		}
	}

	for index, voter := range voters {
		if err := sys.IDB.SetAvailableQuantity(uint64(index), voter, new(big.Int).Mul(big10, minStakeVote)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.VoteCandidate(uint64(index), voter, "test", new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("VoteCandidate invalid candidate %v mismatch", err))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], big1, uint64(index), 0); !strings.Contains(err.Error(), "non divisibility") {
			panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big0, minStakeVote), uint64(index), 0); !strings.Contains(err.Error(), "insufficient") {
			panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big1, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big2, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		} else if candidateInfo.Type != Normal {
			panic(fmt.Sprintf("GetCandidate  type mismatch"))
		}
	}
}

func TestCandidateVote(t *testing.T) {
	ldb, function := newTestLDB()
	db, err := NewLDB(ldb)
	defer function()
	if err != nil {
		panic(fmt.Errorf("create db failed --- %v", err))
	}
	sys := &System{
		config: DefaultConfig,
		IDB:    db,
	}
	_ = sys

	for index, candidate := range candidates {
		if err := db.SetState(&GlobalState{
			Epoch:         uint64(index),
			PreEpoch:      uint64(index),
			TotalQuantity: big.NewInt(0),
		}); err != nil {
			panic(fmt.Errorf("SetState --- %v", err))
		}
		if err := sys.IDB.SetAvailableQuantity(uint64(index), candidate, new(big.Int).Mul(big10, minStakeCandidate)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.RegCandidate(uint64(index), candidate, fmt.Sprintf("www.%v.com", candidate), new(big.Int).Mul(big1, minStakeCandidate), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("RegCandidate %v", err))
		}
		// if err := sys.UnregCandidate(uint64(index), candidate, uint64(index)); err != nil {
		// 	panic(fmt.Sprintf("UnregCandidate %v", err))
		// }
	}

	for index, voter := range voters {
		if err := sys.IDB.SetAvailableQuantity(uint64(index), voter, new(big.Int).Mul(big10, minStakeVote)); err != nil {
			panic(fmt.Errorf("SetAvailableQuantity --- %v", err))
		}
		if err := sys.VoteCandidate(uint64(index), voter, "test", new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); !strings.Contains(err.Error(), "invalid candidate") {
			panic(fmt.Sprintf("VoteCandidate invalid candidate %v mismatch", err))
		}

		// if err := sys.VoteCandidate(uint64(index), voter, candidates[index], big1, uint64(index)); !strings.Contains(err.Error(), "not in normal") {
		// 	panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		// }

		// if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index)); !strings.Contains(err.Error(), "insufficient") {
		// 	panic(fmt.Sprintf("VoteCandidate invalid stake %v mismatch", err))
		// }

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big1, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		}

		if err := sys.VoteCandidate(uint64(index), voter, candidates[index], new(big.Int).Mul(big1, minStakeVote), uint64(index), 0); err != nil {
			panic(fmt.Sprintf("VoteCandidate --- %v", err))
		}

		if candidateInfo, _ := sys.GetCandidate(uint64(index), candidates[index]); new(big.Int).Sub(candidateInfo.TotalQuantity, candidateInfo.Quantity).Cmp(new(big.Int).Mul(big2, DefaultConfig.VoterMinQuantity)) != 0 {
			panic(fmt.Sprintf("GetCandidate mismatch"))
		} else if candidateInfo.Type != Normal {
			panic(fmt.Sprintf("GetCandidate  type mismatch"))
		}
	}
}
<|endoftext|>


<commit_before>package daemon

import (
	"net/http"
	// FIXME: importing this package has the side-effect of poisoning the http.DefaultServeMux
	// FIXME: with the /debug/pprof endpoints
	"context"
	"net"
	"net/http/pprof"
)

type PProfServer struct {
	cc       chan PprofServerControlMsg
	state    PprofServerControlMsg
	listener net.Listener
}

type PprofServerControlMsg struct {
	// Whether the server should listen for requests on the given address
	Run bool
	// Must be set if Run is true, undefined otherwise
	HttpListenAddress string
}

func NewPProfServer(ctx context.Context) *PProfServer {

	s := &PProfServer{
		cc: make(chan PprofServerControlMsg),
	}

	go s.controlLoop(ctx)
	return s
}

func (s *PProfServer) controlLoop(ctx context.Context) {
outer:
	for {

		var msg PprofServerControlMsg
		select {
		case <-ctx.Done():
			if s.listener != nil {
				s.listener.Close()
			}
			break outer
		case msg = <-s.cc:
			// proceed
		}

		var err error
		if msg.Run && s.listener == nil {

			s.listener, err = net.Listen("tcp", msg.HttpListenAddress)
			if err != nil {
				s.listener = nil
				continue
			}

			// FIXME: because net/http/pprof does not provide a mux,
			mux := http.NewServeMux()
			mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
			mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
			go http.Serve(s.listener, mux)
			continue
		}

		if !msg.Run && s.listener != nil {
			s.listener.Close()
			s.listener = nil
			continue
		}
	}
}

func (s *PProfServer) Control(msg PprofServerControlMsg) {
	s.cc <- msg
}
<commit_msg>privatize pprofServer<commit_after>package daemon

import (
	"net/http"
	// FIXME: importing this package has the side-effect of poisoning the http.DefaultServeMux
	// FIXME: with the /debug/pprof endpoints
	"context"
	"net"
	"net/http/pprof"
)

type pprofServer struct {
	cc       chan PprofServerControlMsg
	state    PprofServerControlMsg
	listener net.Listener
}

type PprofServerControlMsg struct {
	// Whether the server should listen for requests on the given address
	Run bool
	// Must be set if Run is true, undefined otherwise
	HttpListenAddress string
}

func NewPProfServer(ctx context.Context) *pprofServer {

	s := &pprofServer{
		cc: make(chan PprofServerControlMsg),
	}

	go s.controlLoop(ctx)
	return s
}

func (s *pprofServer) controlLoop(ctx context.Context) {
outer:
	for {

		var msg PprofServerControlMsg
		select {
		case <-ctx.Done():
			if s.listener != nil {
				s.listener.Close()
			}
			break outer
		case msg = <-s.cc:
			// proceed
		}

		var err error
		if msg.Run && s.listener == nil {

			s.listener, err = net.Listen("tcp", msg.HttpListenAddress)
			if err != nil {
				s.listener = nil
				continue
			}

			// FIXME: because net/http/pprof does not provide a mux,
			mux := http.NewServeMux()
			mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
			mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
			go http.Serve(s.listener, mux)
			continue
		}

		if !msg.Run && s.listener != nil {
			s.listener.Close()
			s.listener = nil
			continue
		}
	}
}

func (s *pprofServer) Control(msg PprofServerControlMsg) {
	s.cc <- msg
}
<|endoftext|>


<commit_before>package textbackend

// TextRow to store the level and text of one row.
type TextRow struct {
	Level int    `json:"level"`
	Text  string `json:"text"`
}

// GetLevelWhitespace returns the whitespace for the row.
func (t *TextRow) GetLevelWhitespace(whitespace string) string {
	tmp := ""
	if t.Level > 0 {
		for i := 0; i < t.Level; i++ {
			tmp += whitespace
		}
	}
	return tmp
}

// GetString returns the row with whitespace and text
func (t *TextRow) GetString(whitespace string) string {
	return t.GetLevelWhitespace(whitespace) + t.Text
}
<commit_msg>Create NewTextRow func<commit_after>package textbackend

// TextRow to store the level and text of one row.
type TextRow struct {
	Level int    `json:"level"`
	Text  string `json:"text"`
}

// NewTextRow initialize and return a TextRow object
func NewTextRow(l int, t string) *TextRow {
	return &TextRow{l, t}
}

// GetLevelWhitespace returns the whitespace for the row.
func (t *TextRow) GetLevelWhitespace(whitespace string) string {
	tmp := ""
	if t.Level > 0 {
		for i := 0; i < t.Level; i++ {
			tmp += whitespace
		}
	}
	return tmp
}

// GetString returns the row with whitespace and text
func (t *TextRow) GetString(whitespace string) string {
	return t.GetLevelWhitespace(whitespace) + t.Text
}
<|endoftext|>


<commit_before>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package sqlbuilder

// CreateViewStmt 创建视图的语句
type CreateViewStmt struct {
	*ddlStmt

	name        string
	selectStmt  *SelectStmt
	columns     []string
	checkOption string
	temporary   bool
	replace     bool
}

// CreateViewStmtHooker CreateViewStmt.DDLSQL 的钩子函数
type CreateViewStmtHooker interface {
	CreateViewStmtHook(*CreateViewStmt) ([]string, error)
}

// CreateView 创建视图
func CreateView(e Engine, d Dialect) *CreateViewStmt {
	stmt := &CreateViewStmt{}
	stmt.ddlStmt = newDDLStmt(e, d, stmt)

	return stmt
}

// View 将当前查询语句转换为视图
func (stmt *SelectStmt) View(name string) *CreateViewStmt {
	return CreateView(stmt.Engine(), stmt.Dialect()).
		From(stmt)
}

// Reset 重置对象
func (stmt *CreateViewStmt) Reset() {
	stmt.name = ""
	stmt.selectStmt = nil
	stmt.columns = stmt.columns[:0]
	stmt.temporary = false
}

// Name 指定视图名称
func (stmt *CreateViewStmt) Name(name string) *CreateViewStmt {
	stmt.name = name
	return stmt
}

// Temporary 临时视图
func (stmt *CreateViewStmt) Temporary() *CreateViewStmt {
	stmt.temporary = true
	return stmt
}

// Replace 如果已经存在，则更新视图内容
func (stmt *CreateViewStmt) Replace() *CreateViewStmt {
	stmt.replace = true
	return stmt
}

// CheckOption 指定 CHECK OPTION 选项
func (stmt *CreateViewStmt) CheckOption(opt string) *CreateViewStmt {
	stmt.checkOption = opt
	return stmt
}

// From 指定 Select 语句
func (stmt *CreateViewStmt) From(sel *SelectStmt) *CreateViewStmt {
	stmt.selectStmt = sel
	return stmt
}

// DDLSQL 返回创建视图的 SQL 语句
func (stmt *CreateViewStmt) DDLSQL() ([]string, error) {
	if stmt.name == "" {
		return nil, ErrTableIsEmpty
	}

	if stmt.selectStmt == nil {
		return nil, ErrValueIsEmpty
	}

	if hook, ok := stmt.Dialect().(CreateViewStmtHooker); ok {
		return hook.CreateViewStmtHook(stmt)
	}

	builder := New("CREATE ")

	if stmt.replace {
		builder.WriteString(" OR REPLACE ")
	}

	if stmt.temporary {
		builder.WriteString(" TEMPORARY ")
	}

	builder.WriteString(" VIEW ").WriteString(stmt.name)

	if len(stmt.columns) > 0 {
		builder.WriteByte('(')
		for _, col := range stmt.columns {
			builder.WriteString(col).WriteByte(',')
		}
		builder.TruncateLast(1)
		builder.WriteByte(')')
	}

	q, args, err := stmt.selectStmt.SQL()
	if err != nil {
		return nil, err
	}
	if len(args) > 0 {
		return nil, ErrViewSelectNotAllowArgs
	}
	builder.WriteString(q)

	return []string{builder.String()}, nil
}
<commit_msg>[sqlbuilder] 根据 master 分支内容作调节<commit_after>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package sqlbuilder

// CreateViewStmt 创建视图的语句
type CreateViewStmt struct {
	*ddlStmt

	name        string
	selectStmt  *SelectStmt
	columns     []string
	checkOption string
	temporary   bool
	replace     bool
}

// CreateViewStmtHooker CreateViewStmt.DDLSQL 的钩子函数
type CreateViewStmtHooker interface {
	CreateViewStmtHook(*CreateViewStmt) ([]string, error)
}

// CreateView 创建视图
func CreateView(e Engine, d Dialect) *CreateViewStmt {
	stmt := &CreateViewStmt{}
	stmt.ddlStmt = newDDLStmt(e, d, stmt)

	return stmt
}

// View 将当前查询语句转换为视图
func (stmt *SelectStmt) View(name string) *CreateViewStmt {
	return CreateView(stmt.Engine(), stmt.Dialect()).
		From(stmt)
}

// Reset 重置对象
func (stmt *CreateViewStmt) Reset() {
	stmt.name = ""
	stmt.selectStmt = nil
	stmt.columns = stmt.columns[:0]
	stmt.temporary = false
}

// Name 指定视图名称
func (stmt *CreateViewStmt) Name(name string) *CreateViewStmt {
	stmt.name = name
	return stmt
}

// Temporary 临时视图
func (stmt *CreateViewStmt) Temporary() *CreateViewStmt {
	stmt.temporary = true
	return stmt
}

// Replace 如果已经存在，则更新视图内容
func (stmt *CreateViewStmt) Replace() *CreateViewStmt {
	stmt.replace = true
	return stmt
}

// CheckOption 指定 CHECK OPTION 选项
func (stmt *CreateViewStmt) CheckOption(opt string) *CreateViewStmt {
	stmt.checkOption = opt
	return stmt
}

// From 指定 Select 语句
func (stmt *CreateViewStmt) From(sel *SelectStmt) *CreateViewStmt {
	stmt.selectStmt = sel
	return stmt
}

// DDLSQL 返回创建视图的 SQL 语句
func (stmt *CreateViewStmt) DDLSQL() ([]string, error) {
	if stmt.name == "" {
		return nil, ErrTableIsEmpty
	}

	if stmt.selectStmt == nil {
		return nil, ErrValueIsEmpty
	}

	if hook, ok := stmt.Dialect().(CreateViewStmtHooker); ok {
		return hook.CreateViewStmtHook(stmt)
	}

	builder := New("CREATE ")

	if stmt.replace {
		builder.WriteString(" OR REPLACE ")
	}

	if stmt.temporary {
		builder.WriteString(" TEMPORARY ")
	}

	builder.WriteString(" VIEW ").
		WriteBytes(stmt.l).
		WriteString(stmt.name).
		WriteBytes(stmt.r)

	if len(stmt.columns) > 0 {
		builder.WriteBytes('(')
		for _, col := range stmt.columns {
			builder.WriteBytes(stmt.l).
				WriteString(col).
				WriteBytes(stmt.r, ',')
		}
		builder.TruncateLast(1)
		builder.WriteBytes(')')
	}

	q, args, err := stmt.selectStmt.SQL()
	if err != nil {
		return nil, err
	}
	if len(args) > 0 {
		return nil, ErrViewSelectNotAllowArgs
	}
	builder.WriteString(q)

	return []string{builder.String()}, nil
}
<|endoftext|>


<commit_before>package gf

const VERSION = "v1.13.3"
const AUTHORS = "john<john@goframe.org>"
<commit_msg>version updates<commit_after>package gf

const VERSION = "v1.13.4"
const AUTHORS = "john<john@goframe.org>"
<|endoftext|>


<commit_before>package openweathermap

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

const (
	baseUrl string = "http://api.openweathermap.org/data/2.5/weather?%s"
)

type Coordinates struct {
	Longitude float64 `json:"lon"`
	Latitude  float64 `json:"lat"`
}

type Sys struct {
	Type    int     `json:"type"`
	Id      int     `json:"id"`
	Message float64 `json:"message"`
	Country string  `json:"country"`
	Sunrise int     `json:"sunrise"`
	Sunset  int     `json:"sunset"`
}

type Wind struct {
	Speed float64 `json:"speed"`
	Deg   int     `json:"deg"`
}

type Weather struct {
	Id          int    `json:"id"`
	Main        string `json:"main"`
	Description string `json:"description"`
	Icon        string `json:"icon"`
}

type Main struct {
	Temp     float64 `json:"temp"`
	TempMin  float64 `json:"temp_min"`
	TempMax  float64 `json:"temp_max"`
	Pressure int     `json:"pressure"`
	Humidity int     `json:"humidity"`
}

type Clouds struct {
	All int `json:"all"`
}

type WeatherData struct {
	GeoPos  Coordinates `json:"coord"`
	Sys     Sys         `json:"sys"`
	Base    string      `json:"base"`
	Weather []Weather   `json:"weather"`
	Main    Main        `json:"main"`
	Wind    Wind        `json:"wind"`
	Clouds  Clouds      `json:"clouds"`
	Dt      int         `json:"dt"`
	Id      int         `json:"id"`
	Name    string      `json:"name"`
	Cod     int         `json:"cod"`
}

const (
	baseUrl  string = "http://api.openweathermap.org/data/2.5/weather?%s"
)

func (w *WeatherData) GetByName(location string) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "q=%s"), location))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *WeatherData) GetByCoordinates(location *Coordinates) {
	response, err := http.Get(fmt.Sprintf(
		fmt.Sprintf(
			baseUrl, "lat=%f&lon=%f"), location.Latitude, location.Longitude))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *WeatherData) GetByID(id int) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "id=%d"), id))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *WeatherData) GetByArea() {}
<commit_msg>fix dup url bug<commit_after>package openweathermap

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

const (
	baseUrl string = "http://api.openweathermap.org/data/2.5/weather?%s"
)

type Coordinates struct {
	Longitude float64 `json:"lon"`
	Latitude  float64 `json:"lat"`
}

type Sys struct {
	Type    int     `json:"type"`
	Id      int     `json:"id"`
	Message float64 `json:"message"`
	Country string  `json:"country"`
	Sunrise int     `json:"sunrise"`
	Sunset  int     `json:"sunset"`
}

type Wind struct {
	Speed float64 `json:"speed"`
	Deg   int     `json:"deg"`
}

type Weather struct {
	Id          int    `json:"id"`
	Main        string `json:"main"`
	Description string `json:"description"`
	Icon        string `json:"icon"`
}

type Main struct {
	Temp     float64 `json:"temp"`
	TempMin  float64 `json:"temp_min"`
	TempMax  float64 `json:"temp_max"`
	Pressure int     `json:"pressure"`
	Humidity int     `json:"humidity"`
}

type Clouds struct {
	All int `json:"all"`
}

type WeatherData struct {
	GeoPos  Coordinates `json:"coord"`
	Sys     Sys         `json:"sys"`
	Base    string      `json:"base"`
	Weather []Weather   `json:"weather"`
	Main    Main        `json:"main"`
	Wind    Wind        `json:"wind"`
	Clouds  Clouds      `json:"clouds"`
	Dt      int         `json:"dt"`
	Id      int         `json:"id"`
	Name    string      `json:"name"`
	Cod     int         `json:"cod"`
}

func (w *WeatherData) GetByName(location string) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "q=%s"), location))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *WeatherData) GetByCoordinates(location *Coordinates) {
	response, err := http.Get(fmt.Sprintf(
		fmt.Sprintf(
			baseUrl, "lat=%f&lon=%f"), location.Latitude, location.Longitude))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *WeatherData) GetByID(id int) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "id=%d"), id))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *WeatherData) GetByArea() {}
<|endoftext|>


<commit_before>package youtube

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
)

func SetLogOutput(w io.Writer) {
	log.SetOutput(w)
}

func NewYoutube() *Youtube {
	return &Youtube{DownloadPercent: make(chan int64, 100)}
}

type stream map[string]string

type Youtube struct {
	StreamList        []stream
	VideoID           string
	videoInfo         string
	DownloadPercent   chan int64
	contentLength     float64
	totalWrittenBytes float64
	downloadLevel     float64
}

func (y *Youtube) DecodeURL(url string) error {
	err := y.findVideoId(url)
	if err != nil {
		return fmt.Errorf("findvideoID error=%s", err)
	}

	err = y.getVideoInfo()
	if err != nil {
		return fmt.Errorf("getVideoInfo error=%s", err)
	}

	err = y.parseVideoInfo()
	if err != nil {
		return fmt.Errorf("parse video info failed, err=%s", err)
	}

	return nil
}

func (y *Youtube) StartDownload(dstDir string) error {
	// y.DownloadPercent = make(chan int64, 100)
	//download highest resolution on [0]
	targetStream := y.StreamList[0]
	url := targetStream["url"] + "&signature=" + targetStream["sig"]
	log.Println("Download url=", url)

	targetFile := fmt.Sprintf("%s%c%s.%s", dstDir, os.PathSeparator, targetStream["title"], "mp4")
	//targetStream["title"], targetStream["author"])
	log.Println("Download to file=", targetFile)
	err := y.videoDLWorker(targetFile, url)
	return err
}

func (y *Youtube) parseVideoInfo() error {
	answer, err := url.ParseQuery(y.videoInfo)
	if err != nil {
		return err
	}

	status, ok := answer["status"]
	if !ok {
		err = fmt.Errorf("no response status found in the server's answer")
		return err
	}
	if status[0] == "fail" {
		reason, ok := answer["reason"]
		if ok {
			err = fmt.Errorf("'fail' response status found in the server's answer, reason: '%s'", reason[0])
		} else {
			err = errors.New(fmt.Sprint("'fail' response status found in the server's answer, no reason given"))
		}
		return err
	}
	if status[0] != "ok" {
		err = fmt.Errorf("non-success response status found in the server's answer (status: '%s')", status)
		return err
	}

	// read the streams map
	stream_map, ok := answer["url_encoded_fmt_stream_map"]
	if !ok {
		err = errors.New(fmt.Sprint("no stream map found in the server's answer"))
		return err
	}

	// read each stream

	streams_list := strings.Split(stream_map[0], ",")

	var streams []stream
	for stream_pos, stream_raw := range streams_list {
		stream_qry, err := url.ParseQuery(stream_raw)
		if err != nil {
			log.Println(fmt.Sprintf("An error occured while decoding one of the video's stream's information: stream %d: %s\n", stream_pos, err))
			continue
		}
		var sig string
		if _, exist := stream_qry["sig"]; exist {
			sig = stream_qry["sig"][0]
		}

		stream := stream{
			"quality": stream_qry["quality"][0],
			"type":    stream_qry["type"][0],
			"url":     stream_qry["url"][0],
			"sig":     sig,
			"title":   answer["title"][0],
			"author":  answer["author"][0],
		}
		streams = append(streams, stream)
		log.Printf("Stream found: quality '%s', format '%s'", stream_qry["quality"][0], stream_qry["type"][0])
	}

	y.StreamList = streams
	return nil
}

func (y *Youtube) getVideoInfo() error {
	url := "http://youtube.com/get_video_info?video_id=" + y.VideoID
	log.Printf("url: %s", url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	y.videoInfo = string(body)
	return nil
}

func (y *Youtube) findVideoId(url string) error {
	videoId := url
	if strings.Contains(videoId, "youtu") || strings.ContainsAny(videoId, "\"?&/<%=") {
		re_list := []*regexp.Regexp{
			regexp.MustCompile(`(?:v|embed|watch\?v)(?:=|/)([^"&?/=%]{11})`),
			regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`),
			regexp.MustCompile(`([^"&?/=%]{11})`),
		}
		for _, re := range re_list {
			if is_match := re.MatchString(videoId); is_match {
				subs := re.FindStringSubmatch(videoId)
				videoId = subs[1]
			}
		}
	}
	log.Printf("Found video id: '%s'", videoId)
	y.VideoID = videoId
	if strings.ContainsAny(videoId, "?&/<%=") {
		return errors.New("invalid characters in video id")
	}
	if len(videoId) < 10 {
		return errors.New("the video id must be at least 10 characters long")
	}
	return nil
}

func (y *Youtube) Write(p []byte) (n int, err error) {
	n = len(p)
	y.totalWrittenBytes = y.totalWrittenBytes + float64(n)
	currentPercent := ((y.totalWrittenBytes / y.contentLength) * 100)
	if (y.downloadLevel <= currentPercent) && (y.downloadLevel < 100) {
		y.downloadLevel++
		y.DownloadPercent <- int64(y.downloadLevel)
	}
	return
}
func (y *Youtube) videoDLWorker(destFile string, target string) error {
	resp, err := http.Get(target)
	if err != nil {
		log.Printf("Http.Get\nerror: %s\ntarget: %s\n", err, target)
		return err
	}
	defer resp.Body.Close()
	y.contentLength = float64(resp.ContentLength)

	if resp.StatusCode != 200 {
		log.Printf("reading answer: non 200 status code received: '%s'", err)
		return errors.New("non 200 status code received")
	}
	out, err := os.Create(destFile)
	if err != nil {
		return err
	}
	mw := io.MultiWriter(out, y)
	_, err = io.Copy(mw, resp.Body)
	if err != nil {
		log.Println("download video err=", err)
		return err
	}
	return nil
}
<commit_msg>File path is passed instead of Directory path to func StartDownload()<commit_after>package youtube

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

func SetLogOutput(w io.Writer) {
	log.SetOutput(w)
}

func NewYoutube() *Youtube {
	return &Youtube{DownloadPercent: make(chan int64, 100)}
}

type stream map[string]string

type Youtube struct {
	StreamList        []stream
	VideoID           string
	videoInfo         string
	DownloadPercent   chan int64
	contentLength     float64
	totalWrittenBytes float64
	downloadLevel     float64
}

func (y *Youtube) DecodeURL(url string) error {
	err := y.findVideoId(url)
	if err != nil {
		return fmt.Errorf("findvideoID error=%s", err)
	}

	err = y.getVideoInfo()
	if err != nil {
		return fmt.Errorf("getVideoInfo error=%s", err)
	}

	err = y.parseVideoInfo()
	if err != nil {
		return fmt.Errorf("parse video info failed, err=%s", err)
	}

	return nil
}

func (y *Youtube) StartDownload(destFile string) error {
	//download highest resolution on [0]
	targetStream := y.StreamList[0]
	url := targetStream["url"] + "&signature=" + targetStream["sig"]
	log.Println("Download url=", url)

	log.Println("Download to file=", destFile)
	err := y.videoDLWorker(destFile, url)
	return err
}

func (y *Youtube) parseVideoInfo() error {
	answer, err := url.ParseQuery(y.videoInfo)
	if err != nil {
		return err
	}

	status, ok := answer["status"]
	if !ok {
		err = fmt.Errorf("no response status found in the server's answer")
		return err
	}
	if status[0] == "fail" {
		reason, ok := answer["reason"]
		if ok {
			err = fmt.Errorf("'fail' response status found in the server's answer, reason: '%s'", reason[0])
		} else {
			err = errors.New(fmt.Sprint("'fail' response status found in the server's answer, no reason given"))
		}
		return err
	}
	if status[0] != "ok" {
		err = fmt.Errorf("non-success response status found in the server's answer (status: '%s')", status)
		return err
	}

	// read the streams map
	stream_map, ok := answer["url_encoded_fmt_stream_map"]
	if !ok {
		err = errors.New(fmt.Sprint("no stream map found in the server's answer"))
		return err
	}

	// read each stream

	streams_list := strings.Split(stream_map[0], ",")

	var streams []stream
	for stream_pos, stream_raw := range streams_list {
		stream_qry, err := url.ParseQuery(stream_raw)
		if err != nil {
			log.Println(fmt.Sprintf("An error occured while decoding one of the video's stream's information: stream %d: %s\n", stream_pos, err))
			continue
		}
		var sig string
		if _, exist := stream_qry["sig"]; exist {
			sig = stream_qry["sig"][0]
		}

		stream := stream{
			"quality": stream_qry["quality"][0],
			"type":    stream_qry["type"][0],
			"url":     stream_qry["url"][0],
			"sig":     sig,
			"title":   answer["title"][0],
			"author":  answer["author"][0],
		}
		streams = append(streams, stream)
		log.Printf("Stream found: quality '%s', format '%s'", stream_qry["quality"][0], stream_qry["type"][0])
	}

	y.StreamList = streams
	return nil
}

func (y *Youtube) getVideoInfo() error {
	url := "http://youtube.com/get_video_info?video_id=" + y.VideoID
	log.Printf("url: %s", url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	y.videoInfo = string(body)
	return nil
}

func (y *Youtube) findVideoId(url string) error {
	videoId := url
	if strings.Contains(videoId, "youtu") || strings.ContainsAny(videoId, "\"?&/<%=") {
		re_list := []*regexp.Regexp{
			regexp.MustCompile(`(?:v|embed|watch\?v)(?:=|/)([^"&?/=%]{11})`),
			regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`),
			regexp.MustCompile(`([^"&?/=%]{11})`),
		}
		for _, re := range re_list {
			if is_match := re.MatchString(videoId); is_match {
				subs := re.FindStringSubmatch(videoId)
				videoId = subs[1]
			}
		}
	}
	log.Printf("Found video id: '%s'", videoId)
	y.VideoID = videoId
	if strings.ContainsAny(videoId, "?&/<%=") {
		return errors.New("invalid characters in video id")
	}
	if len(videoId) < 10 {
		return errors.New("the video id must be at least 10 characters long")
	}
	return nil
}

func (y *Youtube) Write(p []byte) (n int, err error) {
	n = len(p)
	y.totalWrittenBytes = y.totalWrittenBytes + float64(n)
	currentPercent := ((y.totalWrittenBytes / y.contentLength) * 100)
	if (y.downloadLevel <= currentPercent) && (y.downloadLevel < 100) {
		y.downloadLevel++
		y.DownloadPercent <- int64(y.downloadLevel)
	}
	return
}
func (y *Youtube) videoDLWorker(destFile string, target string) error {
	resp, err := http.Get(target)
	if err != nil {
		log.Printf("Http.Get\nerror: %s\ntarget: %s\n", err, target)
		return err
	}
	defer resp.Body.Close()
	y.contentLength = float64(resp.ContentLength)

	if resp.StatusCode != 200 {
		log.Printf("reading answer: non 200[code=%v] status code received: '%v'", resp.StatusCode, err)
		return errors.New("non 200 status code received")
	}
	err = os.MkdirAll(filepath.Dir(destFile), 666)
	if err != nil {
		return err
	}
	out, err := os.Create(destFile)
	if err != nil {
		return err
	}
	mw := io.MultiWriter(out, y)
	_, err = io.Copy(mw, resp.Body)
	if err != nil {
		log.Println("download video err=", err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libdokan

import (
	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

// File represents KBFS files.
type File struct {
	FSO
}

func newFile(folder *Folder, node libkbfs.Node, name string, parent libkbfs.Node) *File {
	f := &File{FSO{
		name:   name,
		parent: parent,
		folder: folder,
		node:   node,
	}}
	f.refcount.Increase()
	return f
}

// GetFileInformation for dokan.
func (f *File) GetFileInformation(ctx context.Context, fi *dokan.FileInfo) (a *dokan.Stat, err error) {
	f.folder.fs.logEnter(ctx, "File GetFileInformation")
	defer func() { f.folder.reportErr(ctx, libkbfs.ReadMode, err) }()

	a, err = eiToStat(f.folder.fs.config.KBFSOps().Stat(ctx, f.node))
	if a != nil {
		f.folder.fs.log.CDebugf(ctx, "File GetFileInformation node=%v => %v", f.node, *a)
	} else {
		f.folder.fs.log.CDebugf(ctx, "File GetFileInformation node=%v => Error %T %v", f.node, err, err)
	}
	return a, err
}

// CanDeleteFile - return just nil
// TODO check for permissions here.
func (f *File) CanDeleteFile(ctx context.Context, fi *dokan.FileInfo) error {
	f.folder.fs.logEnterf(ctx, "File CanDeleteFile for %q", f.name)
	return nil
}

// Cleanup - for dokan, remember to handle deletions.
// If Cleanup is called with non-nil FileInfo that has IsDeleteOnClose()
// no libdokan locks should be held prior to the call.
func (f *File) Cleanup(ctx context.Context, fi *dokan.FileInfo) {
	var err error
	f.folder.fs.logEnter(ctx, "File Cleanup")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	f.folder.fs.log.CDebugf(ctx, "Cleanup %v", *f)
	if fi != nil && fi.IsDeleteOnClose() {
		// renameAndDeletionLock should be the first lock to be grabbed in libdokan.
		f.folder.fs.renameAndDeletionLock.Lock()
		defer f.folder.fs.renameAndDeletionLock.Unlock()
		f.folder.fs.log.CDebugf(ctx, "Removing (Delete) file in cleanup %s", f.name)

		err = f.folder.fs.config.KBFSOps().RemoveEntry(ctx, f.parent, f.name)
	}

	if f.refcount.Decrease() {
		f.folder.fs.log.CDebugf(ctx, "Forgetting file node")
		f.folder.forgetNode(ctx, f.node)
	}
}

// FlushFileBuffers performs a (f)sync.
func (f *File) FlushFileBuffers(ctx context.Context, fi *dokan.FileInfo) (err error) {
	f.folder.fs.logEnter(ctx, "File FlushFileBuffers")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	return f.folder.fs.config.KBFSOps().SyncAll(ctx, f.node.GetFolderBranch())
}

// ReadFile for dokan reads.
func (f *File) ReadFile(ctx context.Context, fi *dokan.FileInfo, bs []byte, offset int64) (n int, err error) {
	f.folder.fs.logEnter(ctx, "ReadFile")
	defer func() { f.folder.reportErr(ctx, libkbfs.ReadMode, err) }()

	var nlarge int64
	nlarge, err = f.folder.fs.config.KBFSOps().Read(ctx, f.node, bs, offset)

	// This is safe since length of slices always fits into an int
	return int(nlarge), err
}

// WriteFile for dokan writes.
func (f *File) WriteFile(ctx context.Context, fi *dokan.FileInfo, bs []byte, offset int64) (n int, err error) {
	f.folder.fs.logEnter(ctx, "WriteFile")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	if offset == -1 {
		ei, err := f.folder.fs.config.KBFSOps().Stat(ctx, f.node)
		if err != nil {
			return 0, err
		}
		offset = int64(ei.Size)
	}

	err = f.folder.fs.config.KBFSOps().Write(ctx, f.node, bs, offset)
	return len(bs), err
}

// SetEndOfFile for dokan (f)truncates.
func (f *File) SetEndOfFile(ctx context.Context, fi *dokan.FileInfo, length int64) (err error) {
	f.folder.fs.logEnter(ctx, "File SetEndOfFile")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	return f.folder.fs.config.KBFSOps().Truncate(ctx, f.node, uint64(length))
}

// SetAllocationSize for dokan (f)truncates but does not grow
// file size (it may fallocate, but that is not done at the
// moment).
func (f *File) SetAllocationSize(ctx context.Context, fi *dokan.FileInfo, newSize int64) (err error) {
	f.folder.fs.logEnter(ctx, "File SetAllocationSize")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	ei, err := f.folder.fs.config.KBFSOps().Stat(ctx, f.node)
	if err != nil {
		return err
	}

	// Refuse to grow the file.
	if int64(ei.Size) <= newSize {
		return nil
	}

	return f.folder.fs.config.KBFSOps().Truncate(ctx, f.node, uint64(newSize))
}

// SetFileAttributes for Dokan.
func (f *File) SetFileAttributes(ctx context.Context, fi *dokan.FileInfo, fileAttributes dokan.FileAttribute) error {
	f.folder.fs.logEnter(ctx, "File SetFileAttributes")
	// TODO handle attributes for real.
	return nil
}
<commit_msg>libdokan: log SetFileAttributes better<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libdokan

import (
	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

// File represents KBFS files.
type File struct {
	FSO
}

func newFile(folder *Folder, node libkbfs.Node, name string, parent libkbfs.Node) *File {
	f := &File{FSO{
		name:   name,
		parent: parent,
		folder: folder,
		node:   node,
	}}
	f.refcount.Increase()
	return f
}

// GetFileInformation for dokan.
func (f *File) GetFileInformation(ctx context.Context, fi *dokan.FileInfo) (a *dokan.Stat, err error) {
	f.folder.fs.logEnter(ctx, "File GetFileInformation")
	defer func() { f.folder.reportErr(ctx, libkbfs.ReadMode, err) }()

	a, err = eiToStat(f.folder.fs.config.KBFSOps().Stat(ctx, f.node))
	if a != nil {
		f.folder.fs.log.CDebugf(ctx, "File GetFileInformation node=%v => %v", f.node, *a)
	} else {
		f.folder.fs.log.CDebugf(ctx, "File GetFileInformation node=%v => Error %T %v", f.node, err, err)
	}
	return a, err
}

// CanDeleteFile - return just nil
// TODO check for permissions here.
func (f *File) CanDeleteFile(ctx context.Context, fi *dokan.FileInfo) error {
	f.folder.fs.logEnterf(ctx, "File CanDeleteFile for %q", f.name)
	return nil
}

// Cleanup - for dokan, remember to handle deletions.
// If Cleanup is called with non-nil FileInfo that has IsDeleteOnClose()
// no libdokan locks should be held prior to the call.
func (f *File) Cleanup(ctx context.Context, fi *dokan.FileInfo) {
	var err error
	f.folder.fs.logEnter(ctx, "File Cleanup")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	f.folder.fs.log.CDebugf(ctx, "Cleanup %v", *f)
	if fi != nil && fi.IsDeleteOnClose() {
		// renameAndDeletionLock should be the first lock to be grabbed in libdokan.
		f.folder.fs.renameAndDeletionLock.Lock()
		defer f.folder.fs.renameAndDeletionLock.Unlock()
		f.folder.fs.log.CDebugf(ctx, "Removing (Delete) file in cleanup %s", f.name)

		err = f.folder.fs.config.KBFSOps().RemoveEntry(ctx, f.parent, f.name)
	}

	if f.refcount.Decrease() {
		f.folder.fs.log.CDebugf(ctx, "Forgetting file node")
		f.folder.forgetNode(ctx, f.node)
	}
}

// FlushFileBuffers performs a (f)sync.
func (f *File) FlushFileBuffers(ctx context.Context, fi *dokan.FileInfo) (err error) {
	f.folder.fs.logEnter(ctx, "File FlushFileBuffers")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	return f.folder.fs.config.KBFSOps().SyncAll(ctx, f.node.GetFolderBranch())
}

// ReadFile for dokan reads.
func (f *File) ReadFile(ctx context.Context, fi *dokan.FileInfo, bs []byte, offset int64) (n int, err error) {
	f.folder.fs.logEnter(ctx, "ReadFile")
	defer func() { f.folder.reportErr(ctx, libkbfs.ReadMode, err) }()

	var nlarge int64
	nlarge, err = f.folder.fs.config.KBFSOps().Read(ctx, f.node, bs, offset)

	// This is safe since length of slices always fits into an int
	return int(nlarge), err
}

// WriteFile for dokan writes.
func (f *File) WriteFile(ctx context.Context, fi *dokan.FileInfo, bs []byte, offset int64) (n int, err error) {
	f.folder.fs.logEnter(ctx, "WriteFile")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	if offset == -1 {
		ei, err := f.folder.fs.config.KBFSOps().Stat(ctx, f.node)
		if err != nil {
			return 0, err
		}
		offset = int64(ei.Size)
	}

	err = f.folder.fs.config.KBFSOps().Write(ctx, f.node, bs, offset)
	return len(bs), err
}

// SetEndOfFile for dokan (f)truncates.
func (f *File) SetEndOfFile(ctx context.Context, fi *dokan.FileInfo, length int64) (err error) {
	f.folder.fs.logEnter(ctx, "File SetEndOfFile")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	return f.folder.fs.config.KBFSOps().Truncate(ctx, f.node, uint64(length))
}

// SetAllocationSize for dokan (f)truncates but does not grow
// file size (it may fallocate, but that is not done at the
// moment).
func (f *File) SetAllocationSize(ctx context.Context, fi *dokan.FileInfo, newSize int64) (err error) {
	f.folder.fs.logEnter(ctx, "File SetAllocationSize")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()

	ei, err := f.folder.fs.config.KBFSOps().Stat(ctx, f.node)
	if err != nil {
		return err
	}

	// Refuse to grow the file.
	if int64(ei.Size) <= newSize {
		return nil
	}

	return f.folder.fs.config.KBFSOps().Truncate(ctx, f.node, uint64(newSize))
}

// SetFileAttributes for Dokan.
func (f *File) SetFileAttributes(ctx context.Context, fi *dokan.FileInfo, fileAttributes dokan.FileAttribute) error {
	f.folder.fs.logEnterf(ctx, "File SetFileAttributes %X", fileAttributes)
	// TODO handle attributes for real.
	return nil
}
<|endoftext|>