<commit_before>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package models

import (
	"fmt"
	"strings"
	"time"

	"github.com/topfreegames/khan/util"

	"gopkg.in/gorp.v1"
)

// ClanByName allows sorting clans by name
type ClanByName []*Clan

func (a ClanByName) Len() int           { return len(a) }
func (a ClanByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ClanByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// Clan identifies uniquely one clan in a given game
type Clan struct {
	ID               int       `db:"id"`
	GameID           string    `db:"game_id"`
	PublicID         string    `db:"public_id"`
	Name             string    `db:"name"`
	OwnerID          int       `db:"owner_id"`
	Metadata         util.JSON `db:"metadata"`
	AllowApplication bool      `db:"allow_application"`
	AutoJoin         bool      `db:"auto_join"`
	CreatedAt        int64     `db:"created_at"`
	UpdatedAt        int64     `db:"updated_at"`
	DeletedAt        int64     `db:"deleted_at"`
}

// PreInsert populates fields before inserting a new clan
func (c *Clan) PreInsert(s gorp.SqlExecutor) error {
	c.CreatedAt = time.Now().UnixNano() / 1000000
	c.UpdatedAt = c.CreatedAt
	return nil
}

// PreUpdate populates fields before updating a clan
func (c *Clan) PreUpdate(s gorp.SqlExecutor) error {
	c.UpdatedAt = time.Now().UnixNano() / 1000000
	return nil
}

// GetPlayerClansCount returns the count of clans the player is the owner or a member.
func GetPlayerClansCount(db DB, gameID, playerPublicID string) (int, error) {
	query := `
	SELECT
		Coalesce(m.count, 0) + Coalesce(c.count, 0) AS count
	FROM players p
		LEFT OUTER JOIN (
			SELECT
				COUNT(*) AS count, m.player_id
			FROM memberships m
			WHERE
				m.deleted_at=0 AND m.approved=true AND m.game_id=$1
			GROUP BY m.player_id
		) m ON m.player_id=p.id

	  LEFT OUTER JOIN (
			SELECT
			  COUNT(*) count, c.owner_id
			FROM clans c
			WHERE
				c.game_id=$1
			GROUP BY c.owner_id
		) c ON c.owner_id=p.id
	WHERE
		p.public_id=$2`

	count, err := db.SelectInt(query, gameID, playerPublicID)
	if err != nil {
		return 0, err
	}
	return int(count), nil
}

// GetClanByID returns a clan by id
func GetClanByID(db DB, id int) (*Clan, error) {
	obj, err := db.Get(Clan{}, id)
	if err != nil || obj == nil {
		return nil, &ModelNotFoundError{"Clan", id}
	}
	return obj.(*Clan), nil
}

// GetClanByPublicID returns a clan by its public id
func GetClanByPublicID(db DB, gameID, publicID string) (*Clan, error) {
	var clan Clan
	err := db.SelectOne(&clan, "SELECT * FROM clans WHERE game_id=$1 AND public_id=$2", gameID, publicID)
	if err != nil || &clan == nil {
		return nil, &ModelNotFoundError{"Clan", publicID}
	}
	return &clan, nil
}

// GetClanByPublicIDAndOwnerPublicID returns a clan by its public id and the owner public id
func GetClanByPublicIDAndOwnerPublicID(db DB, gameID, publicID, ownerPublicID string) (*Clan, error) {
	var clan Clan
	err := db.SelectOne(&clan, "SELECT clans.* FROM clans, players WHERE clans.game_id=$1 AND clans.public_id=$2 AND clans.owner_id=players.id AND players.public_id=$3", gameID, publicID, ownerPublicID)
	if err != nil || &clan == nil {
		return nil, &ModelNotFoundError{"Clan", publicID}
	}
	return &clan, nil
}

// CreateClan creates a new clan
func CreateClan(db DB, gameID, publicID, name, ownerPublicID string, metadata util.JSON, allowApplication, autoJoin bool, maxClansPerPlayer int) (*Clan, error) {
	player, err := GetPlayerByPublicID(db, gameID, ownerPublicID)
	if err != nil {
		return nil, err
	}

	if player.MembershipCount+player.OwnershipCount >= maxClansPerPlayer {
		return nil, &PlayerReachedMaxClansError{ownerPublicID}
	}

	clan := &Clan{
		GameID:           gameID,
		PublicID:         publicID,
		Name:             name,
		OwnerID:          player.ID,
		Metadata:         metadata,
		AllowApplication: allowApplication,
		AutoJoin:         autoJoin,
	}

	err = db.Insert(clan)
	if err != nil {
		return nil, err
	}

	err = IncrementPlayerOwnershipCount(db, player.ID, 1)
	if err != nil {
		return nil, err
	}
	return clan, nil
}

// LeaveClan allows the clan owner to leave the clan and transfer the clan ownership to the next player in line
func LeaveClan(db DB, gameID, publicID, ownerPublicID string) error {
	clan, err := GetClanByPublicIDAndOwnerPublicID(db, gameID, publicID, ownerPublicID)

	if err != nil {
		return err
	}

	newOwnerMembership, err := GetOldestMemberWithHighestLevel(db, gameID, publicID)
	if err != nil {
		// Clan has no members, delete it
		_, err = db.Delete(clan)
		if err != nil {
			return err
		}
		return nil
	}

	oldOwnerID := clan.OwnerID
	clan.OwnerID = newOwnerMembership.PlayerID

	_, err = db.Update(clan)
	if err != nil {
		return err
	}

	err = deleteMembershipHelper(db, newOwnerMembership, oldOwnerID)
	if err != nil {
		return err
	}

	return nil
}

// TransferClanOwnership allows the clan owner to transfer the clan ownership to the a clan member
func TransferClanOwnership(db DB, gameID, clanPublicID, ownerPublicID, playerPublicID string, levels util.JSON, maxLevel int) error {
	clan, err := GetClanByPublicIDAndOwnerPublicID(db, gameID, clanPublicID, ownerPublicID)
	if err != nil {
		return err
	}

	newOwnerMembership, err := GetMembershipByClanAndPlayerPublicID(db, gameID, clanPublicID, playerPublicID)
	if err != nil {
		return err
	}

	oldOwnerID := clan.OwnerID
	clan.OwnerID = newOwnerMembership.PlayerID
	_, err = db.Update(clan)
	if err != nil {
		return err
	}

	level := GetLevelByLevelInt(maxLevel, levels)
	if level == "" {
		return &InvalidLevelForGameError{gameID, level}
	}
	oldOwnerMembership, err := GetDeletedMembershipByClanAndPlayerPublicID(db, gameID, clanPublicID, ownerPublicID)
	if err != nil {
		err = db.Insert(&Membership{
			GameID:      gameID,
			ClanID:      clan.ID,
			PlayerID:    oldOwnerID,
			RequestorID: oldOwnerID,
			Level:       level,
			Approved:    true,
			Denied:      false,
			CreatedAt:   clan.CreatedAt,
			UpdatedAt:   time.Now().UnixNano() / 1000000,
		})
		if err != nil {
			return err
		}
	} else {
		oldOwnerMembership.Approved = true
		oldOwnerMembership.Denied = false
		oldOwnerMembership.Level = level
		oldOwnerMembership.RequestorID = oldOwnerID
		_, err = db.Update(oldOwnerMembership)
	}

	err = deleteMembershipHelper(db, newOwnerMembership, oldOwnerID)
	if err != nil {
		return err
	}

	return nil
}

// UpdateClan updates an existing clan
func UpdateClan(db DB, gameID, publicID, name, ownerPublicID string, metadata util.JSON, allowApplication, autoJoin bool) (*Clan, error) {
	clan, err := GetClanByPublicIDAndOwnerPublicID(db, gameID, publicID, ownerPublicID)

	if err != nil {
		return nil, err
	}

	clan.Name = name
	clan.Metadata = metadata
	clan.AllowApplication = allowApplication
	clan.AutoJoin = autoJoin

	_, err = db.Update(clan)

	if err != nil {
		return nil, err
	}

	return clan, nil
}

// GetAllClans returns a list of all clans in a given game
func GetAllClans(db DB, gameID string) ([]Clan, error) {
	if gameID == "" {
		return nil, &EmptyGameIDError{"Clan"}
	}

	var clans []Clan
	_, err := db.Select(&clans, "select * from clans where game_id=$1 order by name", gameID)
	if err != nil {
		return nil, err
	}

	return clans, nil
}

// GetClanDetails returns all details for a given clan by its game id and public id
func GetClanDetails(db DB, gameID, publicID string, maxClansPerPlayer int) (util.JSON, error) {
	query := `
	SELECT
		c.game_id GameID,
		c.public_id ClanPublicID, c.name ClanName, c.metadata ClanMetadata,
		m.membership_level MembershipLevel, m.approved MembershipApproved, m.denied MembershipDenied,
		m.Banned MembershipBanned,
		m.created_at MembershipCreatedAt, m.updated_at MembershipUpdatedAt,
		o.public_id OwnerPublicID, o.name OwnerName, o.metadata OwnerMetadata,
		p.public_id PlayerPublicID, p.name PlayerName, p.metadata DBPlayerMetadata,
		r.public_id RequestorPublicID, r.name RequestorName,
		Coalesce(apm.membership_count, 0) MembershipCount,
		Coalesce(com.ownership_count, 0) OwnershipCount
	FROM clans c
		INNER JOIN players o ON c.owner_id=o.id
		LEFT OUTER JOIN memberships m ON m.clan_id=c.id AND m.deleted_at=0
		LEFT OUTER JOIN players r ON m.requestor_id=r.id
		LEFT OUTER JOIN players p ON m.player_id=p.id
		LEFT OUTER JOIN (
			SELECT
				memberships.clan_id, memberships.player_id, count(*) membership_count
			FROM memberships
			WHERE memberships.approved=true
			GROUP BY memberships.clan_id, memberships.player_id
		) apm ON apm.clan_id=c.id AND apm.player_id=p.id
		LEFT OUTER JOIN (
			SELECT
				clans.owner_id, count(*) ownership_count
			FROM clans
			GROUP BY clans.owner_id
		) com ON com.owner_id=p.id
	WHERE
		c.game_id=$1 AND c.public_id=$2
	`
	var details []clanDetailsDAO
	_, err := db.Select(&details, query, gameID, publicID)
	if err != nil {
		return nil, err
	}

	if len(details) == 0 {
		return nil, &ModelNotFoundError{"Clan", publicID}
	}

	result := make(util.JSON)
	result["name"] = details[0].ClanName
	result["metadata"] = details[0].ClanMetadata
	result["allowApplication"] = details[0].ClanAllowApplication
	result["autoJoin"] = details[0].ClanAutoJoin

	result["owner"] = util.JSON{
		"publicID": details[0].OwnerPublicID,
		"name":     details[0].OwnerName,
		"metadata": details[0].OwnerMetadata,
	}

	if details[0].PlayerPublicID.Valid {
		// First row player public id is not null, meaning we found players!
		player := details[0]

		result["roster"] = make([]util.JSON, 0)
		result["memberships"] = util.JSON{
			"pending": []util.JSON{},
			"banned":  []util.JSON{},
			"denied":  []util.JSON{},
		}
		memberships := result["memberships"].(util.JSON)

		for _, member := range details {
			approved := nullOrBool(player.MembershipApproved)
			denied := nullOrBool(player.MembershipDenied)
			banned := nullOrBool(player.MembershipBanned)
			pending := !approved && !denied && !banned

			memberData := member.Serialize()

			switch {
			case pending:
				if member.MembershipCount+member.OwnershipCount < maxClansPerPlayer {
					memberships["pending"] = append(memberships["pending"].([]util.JSON), memberData)
				}
			case banned:
				memberships["banned"] = append(memberships["banned"].([]util.JSON), memberData)
			case denied:
				memberships["denied"] = append(memberships["denied"].([]util.JSON), memberData)
			case approved:
				result["roster"] = append(result["roster"].([]util.JSON), memberData)
			}
		}
	} else {
		//Otherwise return empty array of object
		result["roster"] = []util.JSON{}
		result["memberships"] = util.JSON{
			"pending": []util.JSON{},
			"banned":  []util.JSON{},
			"denied":  []util.JSON{},
		}
	}

	return result, nil
}

// SearchClan returns a list of clans for a given term (by name or publicID)
func SearchClan(db DB, gameID, term string) ([]Clan, error) {
	if term == "" {
		return nil, &EmptySearchTermError{}
	}

	query := `
	SELECT * FROM clans WHERE game_id=$1 AND (
		(lower(name) like $2) OR
		(lower(public_id) like $2)
	)`

	termStmt := fmt.Sprintf("%%%s%%", strings.ToLower(term))

	var clans []Clan
	_, err := db.Select(&clans, query, gameID, termStmt)
	if err != nil {
		return nil, err
	}

	return clans, nil
}
<commit_msg>Getting rid of GetPlayerClansCount.<commit_after>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package models

import (
	"fmt"
	"strings"
	"time"

	"github.com/topfreegames/khan/util"

	"gopkg.in/gorp.v1"
)

// ClanByName allows sorting clans by name
type ClanByName []*Clan

func (a ClanByName) Len() int           { return len(a) }
func (a ClanByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ClanByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// Clan identifies uniquely one clan in a given game
type Clan struct {
	ID               int       `db:"id"`
	GameID           string    `db:"game_id"`
	PublicID         string    `db:"public_id"`
	Name             string    `db:"name"`
	OwnerID          int       `db:"owner_id"`
	Metadata         util.JSON `db:"metadata"`
	AllowApplication bool      `db:"allow_application"`
	AutoJoin         bool      `db:"auto_join"`
	CreatedAt        int64     `db:"created_at"`
	UpdatedAt        int64     `db:"updated_at"`
	DeletedAt        int64     `db:"deleted_at"`
}

// PreInsert populates fields before inserting a new clan
func (c *Clan) PreInsert(s gorp.SqlExecutor) error {
	c.CreatedAt = time.Now().UnixNano() / 1000000
	c.UpdatedAt = c.CreatedAt
	return nil
}

// PreUpdate populates fields before updating a clan
func (c *Clan) PreUpdate(s gorp.SqlExecutor) error {
	c.UpdatedAt = time.Now().UnixNano() / 1000000
	return nil
}

// GetClanByID returns a clan by id
func GetClanByID(db DB, id int) (*Clan, error) {
	obj, err := db.Get(Clan{}, id)
	if err != nil || obj == nil {
		return nil, &ModelNotFoundError{"Clan", id}
	}
	return obj.(*Clan), nil
}

// GetClanByPublicID returns a clan by its public id
func GetClanByPublicID(db DB, gameID, publicID string) (*Clan, error) {
	var clan Clan
	err := db.SelectOne(&clan, "SELECT * FROM clans WHERE game_id=$1 AND public_id=$2", gameID, publicID)
	if err != nil || &clan == nil {
		return nil, &ModelNotFoundError{"Clan", publicID}
	}
	return &clan, nil
}

// GetClanByPublicIDAndOwnerPublicID returns a clan by its public id and the owner public id
func GetClanByPublicIDAndOwnerPublicID(db DB, gameID, publicID, ownerPublicID string) (*Clan, error) {
	var clan Clan
	err := db.SelectOne(&clan, "SELECT clans.* FROM clans, players WHERE clans.game_id=$1 AND clans.public_id=$2 AND clans.owner_id=players.id AND players.public_id=$3", gameID, publicID, ownerPublicID)
	if err != nil || &clan == nil {
		return nil, &ModelNotFoundError{"Clan", publicID}
	}
	return &clan, nil
}

// CreateClan creates a new clan
func CreateClan(db DB, gameID, publicID, name, ownerPublicID string, metadata util.JSON, allowApplication, autoJoin bool, maxClansPerPlayer int) (*Clan, error) {
	player, err := GetPlayerByPublicID(db, gameID, ownerPublicID)
	if err != nil {
		return nil, err
	}

	if player.MembershipCount+player.OwnershipCount >= maxClansPerPlayer {
		return nil, &PlayerReachedMaxClansError{ownerPublicID}
	}

	clan := &Clan{
		GameID:           gameID,
		PublicID:         publicID,
		Name:             name,
		OwnerID:          player.ID,
		Metadata:         metadata,
		AllowApplication: allowApplication,
		AutoJoin:         autoJoin,
	}

	err = db.Insert(clan)
	if err != nil {
		return nil, err
	}

	err = IncrementPlayerOwnershipCount(db, player.ID, 1)
	if err != nil {
		return nil, err
	}
	return clan, nil
}

// LeaveClan allows the clan owner to leave the clan and transfer the clan ownership to the next player in line
func LeaveClan(db DB, gameID, publicID, ownerPublicID string) error {
	clan, err := GetClanByPublicIDAndOwnerPublicID(db, gameID, publicID, ownerPublicID)

	if err != nil {
		return err
	}

	newOwnerMembership, err := GetOldestMemberWithHighestLevel(db, gameID, publicID)
	if err != nil {
		// Clan has no members, delete it
		_, err = db.Delete(clan)
		if err != nil {
			return err
		}
		return nil
	}

	oldOwnerID := clan.OwnerID
	clan.OwnerID = newOwnerMembership.PlayerID

	_, err = db.Update(clan)
	if err != nil {
		return err
	}

	err = deleteMembershipHelper(db, newOwnerMembership, oldOwnerID)
	if err != nil {
		return err
	}

	return nil
}

// TransferClanOwnership allows the clan owner to transfer the clan ownership to the a clan member
func TransferClanOwnership(db DB, gameID, clanPublicID, ownerPublicID, playerPublicID string, levels util.JSON, maxLevel int) error {
	clan, err := GetClanByPublicIDAndOwnerPublicID(db, gameID, clanPublicID, ownerPublicID)
	if err != nil {
		return err
	}

	newOwnerMembership, err := GetMembershipByClanAndPlayerPublicID(db, gameID, clanPublicID, playerPublicID)
	if err != nil {
		return err
	}

	oldOwnerID := clan.OwnerID
	clan.OwnerID = newOwnerMembership.PlayerID
	_, err = db.Update(clan)
	if err != nil {
		return err
	}

	level := GetLevelByLevelInt(maxLevel, levels)
	if level == "" {
		return &InvalidLevelForGameError{gameID, level}
	}
	oldOwnerMembership, err := GetDeletedMembershipByClanAndPlayerPublicID(db, gameID, clanPublicID, ownerPublicID)
	if err != nil {
		err = db.Insert(&Membership{
			GameID:      gameID,
			ClanID:      clan.ID,
			PlayerID:    oldOwnerID,
			RequestorID: oldOwnerID,
			Level:       level,
			Approved:    true,
			Denied:      false,
			CreatedAt:   clan.CreatedAt,
			UpdatedAt:   time.Now().UnixNano() / 1000000,
		})
		if err != nil {
			return err
		}
	} else {
		oldOwnerMembership.Approved = true
		oldOwnerMembership.Denied = false
		oldOwnerMembership.Level = level
		oldOwnerMembership.RequestorID = oldOwnerID
		_, err = db.Update(oldOwnerMembership)
	}

	err = deleteMembershipHelper(db, newOwnerMembership, oldOwnerID)
	if err != nil {
		return err
	}

	return nil
}

// UpdateClan updates an existing clan
func UpdateClan(db DB, gameID, publicID, name, ownerPublicID string, metadata util.JSON, allowApplication, autoJoin bool) (*Clan, error) {
	clan, err := GetClanByPublicIDAndOwnerPublicID(db, gameID, publicID, ownerPublicID)

	if err != nil {
		return nil, err
	}

	clan.Name = name
	clan.Metadata = metadata
	clan.AllowApplication = allowApplication
	clan.AutoJoin = autoJoin

	_, err = db.Update(clan)

	if err != nil {
		return nil, err
	}

	return clan, nil
}

// GetAllClans returns a list of all clans in a given game
func GetAllClans(db DB, gameID string) ([]Clan, error) {
	if gameID == "" {
		return nil, &EmptyGameIDError{"Clan"}
	}

	var clans []Clan
	_, err := db.Select(&clans, "select * from clans where game_id=$1 order by name", gameID)
	if err != nil {
		return nil, err
	}

	return clans, nil
}

// GetClanDetails returns all details for a given clan by its game id and public id
func GetClanDetails(db DB, gameID, publicID string, maxClansPerPlayer int) (util.JSON, error) {
	query := `
	SELECT
		c.game_id GameID,
		c.public_id ClanPublicID, c.name ClanName, c.metadata ClanMetadata,
		m.membership_level MembershipLevel, m.approved MembershipApproved, m.denied MembershipDenied,
		m.Banned MembershipBanned,
		m.created_at MembershipCreatedAt, m.updated_at MembershipUpdatedAt,
		o.public_id OwnerPublicID, o.name OwnerName, o.metadata OwnerMetadata,
		p.public_id PlayerPublicID, p.name PlayerName, p.metadata DBPlayerMetadata,
		r.public_id RequestorPublicID, r.name RequestorName,
		Coalesce(apm.membership_count, 0) MembershipCount,
		Coalesce(com.ownership_count, 0) OwnershipCount
	FROM clans c
		INNER JOIN players o ON c.owner_id=o.id
		LEFT OUTER JOIN memberships m ON m.clan_id=c.id AND m.deleted_at=0
		LEFT OUTER JOIN players r ON m.requestor_id=r.id
		LEFT OUTER JOIN players p ON m.player_id=p.id
		LEFT OUTER JOIN (
			SELECT
				memberships.clan_id, memberships.player_id, count(*) membership_count
			FROM memberships
			WHERE memberships.approved=true
			GROUP BY memberships.clan_id, memberships.player_id
		) apm ON apm.clan_id=c.id AND apm.player_id=p.id
		LEFT OUTER JOIN (
			SELECT
				clans.owner_id, count(*) ownership_count
			FROM clans
			GROUP BY clans.owner_id
		) com ON com.owner_id=p.id
	WHERE
		c.game_id=$1 AND c.public_id=$2
	`
	var details []clanDetailsDAO
	_, err := db.Select(&details, query, gameID, publicID)
	if err != nil {
		return nil, err
	}

	if len(details) == 0 {
		return nil, &ModelNotFoundError{"Clan", publicID}
	}

	result := make(util.JSON)
	result["name"] = details[0].ClanName
	result["metadata"] = details[0].ClanMetadata
	result["allowApplication"] = details[0].ClanAllowApplication
	result["autoJoin"] = details[0].ClanAutoJoin

	result["owner"] = util.JSON{
		"publicID": details[0].OwnerPublicID,
		"name":     details[0].OwnerName,
		"metadata": details[0].OwnerMetadata,
	}

	if details[0].PlayerPublicID.Valid {
		// First row player public id is not null, meaning we found players!
		player := details[0]

		result["roster"] = make([]util.JSON, 0)
		result["memberships"] = util.JSON{
			"pending": []util.JSON{},
			"banned":  []util.JSON{},
			"denied":  []util.JSON{},
		}
		memberships := result["memberships"].(util.JSON)

		for _, member := range details {
			approved := nullOrBool(player.MembershipApproved)
			denied := nullOrBool(player.MembershipDenied)
			banned := nullOrBool(player.MembershipBanned)
			pending := !approved && !denied && !banned

			memberData := member.Serialize()

			switch {
			case pending:
				if member.MembershipCount+member.OwnershipCount < maxClansPerPlayer {
					memberships["pending"] = append(memberships["pending"].([]util.JSON), memberData)
				}
			case banned:
				memberships["banned"] = append(memberships["banned"].([]util.JSON), memberData)
			case denied:
				memberships["denied"] = append(memberships["denied"].([]util.JSON), memberData)
			case approved:
				result["roster"] = append(result["roster"].([]util.JSON), memberData)
			}
		}
	} else {
		//Otherwise return empty array of object
		result["roster"] = []util.JSON{}
		result["memberships"] = util.JSON{
			"pending": []util.JSON{},
			"banned":  []util.JSON{},
			"denied":  []util.JSON{},
		}
	}

	return result, nil
}

// SearchClan returns a list of clans for a given term (by name or publicID)
func SearchClan(db DB, gameID, term string) ([]Clan, error) {
	if term == "" {
		return nil, &EmptySearchTermError{}
	}

	query := `
	SELECT * FROM clans WHERE game_id=$1 AND (
		(lower(name) like $2) OR
		(lower(public_id) like $2)
	)`

	termStmt := fmt.Sprintf("%%%s%%", strings.ToLower(term))

	var clans []Clan
	_, err := db.Select(&clans, query, gameID, termStmt)
	if err != nil {
		return nil, err
	}

	return clans, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package to keep track of API Versions that should be registered in api.Scheme.
package registered

import (
	"os"
	"strings"

	"github.com/golang/glog"
)

// List of registered API versions.
// The list is in the order of most preferred to the least.
var RegisteredVersions []string

func init() {
	validAPIVersions := map[string]bool{
		"v1":      true,
		"v1beta1": true,
		"v1beta2": true,
		"v1beta3": true,
	}

	// The default list of supported api versions, in order of most preferred to the least.
	defaultSupportedVersions := "v1beta3,v1beta1,v1beta2,v1"
	// Env var KUBE_API_VERSIONS is a comma separated list of API versions that should be registered in the scheme.
	// The versions should be in the order of most preferred to the least.
	supportedVersions := os.Getenv("KUBE_API_VERSIONS")
	if supportedVersions == "" {
		supportedVersions = defaultSupportedVersions
	}
	versions := strings.Split(supportedVersions, ",")
	for _, version := range versions {
		// Verify that the version is valid.
		valid, ok := validAPIVersions[version]
		if !ok || !valid {
			// Not a valid API version.
			glog.Fatalf("invalid api version: %s in KUBE_API_VERSIONS: %s. List of valid API versions: %v",
				version, os.Getenv("KUBE_API_VERSIONS"), validAPIVersions)
		}
		RegisteredVersions = append(RegisteredVersions, version)
	}
}

// Returns true if the given api version is one of the registered api versions.
func IsRegisteredAPIVersion(version string) bool {
	for _, apiVersion := range RegisteredVersions {
		if apiVersion == version {
			return true
		}
	}
	return false
}
<commit_msg>UPSTREAM: Alter default Kubernetes API versions<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package to keep track of API Versions that should be registered in api.Scheme.
package registered

import (
	"os"
	"strings"

	"github.com/golang/glog"
)

// List of registered API versions.
// The list is in the order of most preferred to the least.
var RegisteredVersions []string

func init() {
	validAPIVersions := map[string]bool{
		"v1":      true,
		"v1beta3": true,
	}

	// The default list of supported api versions, in order of most preferred to the least.
	defaultSupportedVersions := "v1,v1beta3"
	// Env var KUBE_API_VERSIONS is a comma separated list of API versions that should be registered in the scheme.
	// The versions should be in the order of most preferred to the least.
	supportedVersions := os.Getenv("KUBE_API_VERSIONS")
	if supportedVersions == "" {
		supportedVersions = defaultSupportedVersions
	}
	versions := strings.Split(supportedVersions, ",")
	for _, version := range versions {
		// Verify that the version is valid.
		valid, ok := validAPIVersions[version]
		if !ok || !valid {
			// Not a valid API version.
			glog.Fatalf("invalid api version: %s in KUBE_API_VERSIONS: %s. List of valid API versions: %v",
				version, os.Getenv("KUBE_API_VERSIONS"), validAPIVersions)
		}
		RegisteredVersions = append(RegisteredVersions, version)
	}
}

// Returns true if the given api version is one of the registered api versions.
func IsRegisteredAPIVersion(version string) bool {
	for _, apiVersion := range RegisteredVersions {
		if apiVersion == version {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package models

type Link struct {
	Self string `json:"string"`
}
<commit_msg>remove unused code<commit_after><|endoftext|>


<commit_before>package raft

import (
	"reflect"
	"testing"
)

func makeRVWithTerm(term TermNo) *RpcRequestVote {
	return &RpcRequestVote{term, 0, 0}
}

// 1. Reply false if term < currentTerm (#5.1)
// Note: this test assumes server in sync with the Figure 7 leader
func TestCM_RpcRV_TermLessThanCurrentTerm(t *testing.T) {
	f := func(setup func(t *testing.T) (mcm *managedConsensusModule, mrs *mockRpcSender)) {
		mcm, _ := setup(t)
		serverTerm := mcm.pcm.persistentState.GetCurrentTerm()

		requestVote := makeRVWithTerm(serverTerm - 1)

		reply := mcm.pcm.rpc("s2", requestVote)

		expectedRpc := &RpcRequestVoteReply{serverTerm, false}
		if !reflect.DeepEqual(reply, expectedRpc) {
			t.Fatal(reply)
		}
	}

	f(testSetupMCM_Follower_Figure7LeaderLine)
	f(testSetupMCM_Candidate_Figure7LeaderLine)
	f(testSetupMCM_Leader_Figure7LeaderLine)
}
<commit_msg>Update RequestVote step 1 test data<commit_after>package raft

import (
	"reflect"
	"testing"
)

// 1. Reply false if term < currentTerm (#5.1)
// Note: test based on Figure 7; server is leader line; peer is case (a)
func TestCM_RpcRV_TermLessThanCurrentTerm(t *testing.T) {
	f := func(setup func(t *testing.T) (mcm *managedConsensusModule, mrs *mockRpcSender)) {
		mcm, _ := setup(t)
		serverTerm := mcm.pcm.persistentState.GetCurrentTerm()

		requestVote := &RpcRequestVote{7, 9, 6}

		reply := mcm.pcm.rpc("s2", requestVote)

		expectedRpc := &RpcRequestVoteReply{serverTerm, false}
		if !reflect.DeepEqual(reply, expectedRpc) {
			t.Fatal(reply)
		}
	}

	f(testSetupMCM_Follower_Figure7LeaderLine)
	f(testSetupMCM_Candidate_Figure7LeaderLine)
	f(testSetupMCM_Leader_Figure7LeaderLine)
}
<|endoftext|>


<commit_before>package bsdiff

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"runtime"
	"testing"

	"bytes"

	"index/suffixarray"

	"github.com/alecthomas/assert"
	"github.com/itchio/wharf/state"
)

func Test_QsufsortSeq(t *testing.T) {
	testQsufsort(t, 0)
}

func Test_QsufsortPar2(t *testing.T) {
	testQsufsort(t, 2)
}

func Test_QsufsortPar4(t *testing.T) {
	testQsufsort(t, 4)
}

func Test_QsufsortPar8(t *testing.T) {
	testQsufsort(t, 8)
}

func Test_Qsufsort64Seq(t *testing.T) {
	testQsufsort64(t, 0)
}

func Test_Qsufsort64Par2(t *testing.T) {
	testQsufsort64(t, 2)
}

func Test_Qsufsort64Par4(t *testing.T) {
	testQsufsort64(t, 4)
}

func Test_Qsufsort64Par8(t *testing.T) {
	testQsufsort64(t, 8)
}

var dictwords []byte
var dictcalls []byte

var result32 []int32
var result64 []int64

func testQsufsort(t *testing.T, concurrency int) {
	input := paper

	ctx := &DiffContext{
		SuffixSortConcurrency: concurrency,
	}
	consumer := &state.Consumer{}

	I := qsufsort(input, ctx, consumer)

	for i := range I {
		if i == 0 {
			continue
		}

		prev := input[I[i-1]:]
		next := input[I[i]:]
		assert.EqualValues(t, -1, bytes.Compare(prev, next))
	}
}

func testQsufsort64(t *testing.T, concurrency int) {
	input := paper

	ctx := &DiffContext{
		SuffixSortConcurrency: concurrency,
	}
	consumer := &state.Consumer{}

	I := qsufsort64(input, ctx, consumer)

	for i := range I {
		if i == 0 {
			continue
		}

		prev := input[I[i-1]:]
		next := input[I[i]:]
		assert.EqualValues(t, -1, bytes.Compare(prev, next))
	}
}

func benchQsuf(input []byte, concurrency int, b *testing.B) {
	ctx := &DiffContext{SuffixSortConcurrency: concurrency}
	consumer := &state.Consumer{}

	var r []int32
	for n := 0; n < b.N; n++ {
		r = qsufsort(input, ctx, consumer)
	}
	result32 = r
}

func benchQsuf64(input []byte, concurrency int, b *testing.B) {
	ctx := &DiffContext{SuffixSortConcurrency: concurrency}
	consumer := &state.Consumer{}

	var r []int64
	for n := 0; n < b.N; n++ {
		r = qsufsort64(input, ctx, consumer)
	}
	result64 = r
}

var sa *suffixarray.Index

func benchSuffixarray(input []byte, b *testing.B) {
	for n := 0; n < b.N; n++ {
		sa = suffixarray.New(input)
	}
}

func Benchmark_Qsufsort(b *testing.B) {
	// note: 'paper' is not worth benchmarking because it's too short
	var datasets = []struct {
		name string
		data []byte
	}{
		{"dictwords", dictwords},
		{"dictcalls", dictcalls},
	}

	for _, dataset := range datasets {
		testName := fmt.Sprintf("suffixarray-%s", dataset.name)
		b.Run(testName, func(b *testing.B) {
			benchSuffixarray(dataset.data, b)
		})
	}

	for _, dataset := range datasets {
		for _, concurrency := range []int{0, 2, 3, 4, 5, 6, 7, 8} {
			testName := fmt.Sprintf("qsufsort32-%s-j%d", dataset.name, concurrency)
			b.Run(testName, func(b *testing.B) {
				benchQsuf(dataset.data, concurrency, b)
			})
		}
	}

	for _, dataset := range datasets {
		for _, concurrency := range []int{0, 2, 3, 4, 5, 6, 7, 8} {
			testName := fmt.Sprintf("qsufsort64-%s-j%d", dataset.name, concurrency)
			b.Run(testName, func(b *testing.B) {
				benchQsuf64(dataset.data, concurrency, b)
			})
		}
	}
}

func init() {
	_, filename, _, _ := runtime.Caller(0)

	var err error
	dictwords, err = ioutil.ReadFile(filepath.Join(filepath.Dir(filename), "dictwords"))
	if err != nil {
		panic(err)
	}

	dictcalls, err = ioutil.ReadFile(filepath.Join(filepath.Dir(filename), "dictcalls"))
	if err != nil {
		panic(err)
	}
}

var paper []byte = []byte(`
    Quicksort is a textbook divide-and-conquer algorithm.
    To sort an array, choose a partitioning element, permute
    the elements such that lesser elements are on one side and
    greater elements are on the other, and then recursively sort
    the two subarrays. But what happens to elements equal to
    the partitioning value? Hoare’s partitioning method is
    binary: it places lesser elements on the left and greater elements
    on the right, but equal elements may appear on
    either side.

    Algorithm designers have long recognized the desirability
    and difficulty of a ternary partitioning method.
    Sedgewick [22] observes on page 244: ‘‘Ideally, we would
    like to get all [equal keys] into position in the file, with all
    the keys with a smaller value to their left, and all the keys
    with a larger value to their right. Unfortunately, no
    efficient method for doing so has yet been devised....’’
    Dijkstra [6] popularized this as ‘‘The Problem of the Dutch
    National Flag’’: we are to order a sequence of red, white
    and blue pebbles to appear in their order on Holland’s
    ensign. This corresponds to Quicksort partitioning when
    lesser elements are colored red, equal elements are white,
    and greater elements are blue. Dijkstra’s ternary algorithm
    requires linear time (it looks at each element exactly once),
    but code to implement it has a significantly larger constant
    factor than Hoare’s binary partitioning code.

    Wegner [27] describes more efficient ternary partitioning
    schemes. Bentley and McIlroy [2] present a ternary
    partition based on this counterintuitive loop invariant:

    The main partitioning loop has two inner loops. The first
    inner loop moves up the index b: it scans over lesser elements,
    swaps equal elements to a, and halts on a greater
    element. The second inner loop moves down the index c
    correspondingly: it scans over greater elements, swaps
    equal elements to d, and halts on a lesser element. The
    main loop then swaps the elements pointed to by b and c,
    increments b and decrements c, and continues until b and
    c cross. (Wegner proposed the same invariant, but maintained
    it with more complex code.) Afterwards, the equal
    elements on the edges are swapped to the middle of the
    array, without any extraneous comparisons. This code partitions
    an n-element array using n − 1 comparisons

    Quicksort has been extensively analyzed by authors
    including Hoare [9], van Emden [26], Knuth [11], and
    Sedgewick [23]. Most detailed analyses involve the harmonic
    numbers Hn = Σ 1≤i≤n 1/ i.

    Theorem 1. [Hoare] A Quicksort that partitions
    around a single randomly selected element sorts n distinct
    items in 2nHn + O(n) ∼∼ 1. 386n lg n expected
    comparisons.

    A common variant of Quicksort partitions around the
    median of a random sample.

    Theorem 2. [van Emden] A Quicksort that partitions
    around the median of 2t + 1 randomly selected elements
    sorts n distinct items in 2nHn / (H2t + 2 − Ht + 1 )
    + O(n) expected comparisons.
    By increasing t, we can push the expected number of comparisons
    close to n lg n + O(n).
    `)
<commit_msg>Fail gracefully in case dictwords/dictcalls aren't there<commit_after>package bsdiff

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"runtime"
	"testing"

	"bytes"

	"index/suffixarray"

	"github.com/alecthomas/assert"
	"github.com/itchio/wharf/state"
)

func Test_QsufsortSeq(t *testing.T) {
	testQsufsort(t, 0)
}

func Test_QsufsortPar2(t *testing.T) {
	testQsufsort(t, 2)
}

func Test_QsufsortPar4(t *testing.T) {
	testQsufsort(t, 4)
}

func Test_QsufsortPar8(t *testing.T) {
	testQsufsort(t, 8)
}

func Test_Qsufsort64Seq(t *testing.T) {
	testQsufsort64(t, 0)
}

func Test_Qsufsort64Par2(t *testing.T) {
	testQsufsort64(t, 2)
}

func Test_Qsufsort64Par4(t *testing.T) {
	testQsufsort64(t, 4)
}

func Test_Qsufsort64Par8(t *testing.T) {
	testQsufsort64(t, 8)
}

var dictwords []byte
var dictcalls []byte

var result32 []int32
var result64 []int64

func testQsufsort(t *testing.T, concurrency int) {
	input := paper

	ctx := &DiffContext{
		SuffixSortConcurrency: concurrency,
	}
	consumer := &state.Consumer{}

	I := qsufsort(input, ctx, consumer)

	for i := range I {
		if i == 0 {
			continue
		}

		prev := input[I[i-1]:]
		next := input[I[i]:]
		assert.EqualValues(t, -1, bytes.Compare(prev, next))
	}
}

func testQsufsort64(t *testing.T, concurrency int) {
	input := paper

	ctx := &DiffContext{
		SuffixSortConcurrency: concurrency,
	}
	consumer := &state.Consumer{}

	I := qsufsort64(input, ctx, consumer)

	for i := range I {
		if i == 0 {
			continue
		}

		prev := input[I[i-1]:]
		next := input[I[i]:]
		assert.EqualValues(t, -1, bytes.Compare(prev, next))
	}
}

func benchQsuf(input []byte, concurrency int, b *testing.B) {
	ctx := &DiffContext{SuffixSortConcurrency: concurrency}
	consumer := &state.Consumer{}

	var r []int32
	for n := 0; n < b.N; n++ {
		r = qsufsort(input, ctx, consumer)
	}
	result32 = r
}

func benchQsuf64(input []byte, concurrency int, b *testing.B) {
	ctx := &DiffContext{SuffixSortConcurrency: concurrency}
	consumer := &state.Consumer{}

	var r []int64
	for n := 0; n < b.N; n++ {
		r = qsufsort64(input, ctx, consumer)
	}
	result64 = r
}

var sa *suffixarray.Index

func benchSuffixarray(input []byte, b *testing.B) {
	for n := 0; n < b.N; n++ {
		sa = suffixarray.New(input)
	}
}

func Benchmark_Qsufsort(b *testing.B) {
	// note: 'paper' is not worth benchmarking because it's too short
	var datasets = []struct {
		name string
		data []byte
	}{
		{"dictwords", dictwords},
		{"dictcalls", dictcalls},
	}

	for _, dataset := range datasets {
		testName := fmt.Sprintf("suffixarray-%s", dataset.name)
		b.Run(testName, func(b *testing.B) {
			benchSuffixarray(dataset.data, b)
		})
	}

	for _, dataset := range datasets {
		for _, concurrency := range []int{0, 2, 3, 4, 5, 6, 7, 8} {
			testName := fmt.Sprintf("qsufsort32-%s-j%d", dataset.name, concurrency)
			b.Run(testName, func(b *testing.B) {
				benchQsuf(dataset.data, concurrency, b)
			})
		}
	}

	for _, dataset := range datasets {
		for _, concurrency := range []int{0, 2, 3, 4, 5, 6, 7, 8} {
			testName := fmt.Sprintf("qsufsort64-%s-j%d", dataset.name, concurrency)
			b.Run(testName, func(b *testing.B) {
				benchQsuf64(dataset.data, concurrency, b)
			})
		}
	}
}

func init() {
	_, filename, _, _ := runtime.Caller(0)

	var err error
	dictwords, err = ioutil.ReadFile(filepath.Join(filepath.Dir(filename), "dictwords"))
	if err != nil {
		fmt.Printf("Could not load dictwords, benchmarks won't be functional (see README.md)\n")
	}

	dictcalls, err = ioutil.ReadFile(filepath.Join(filepath.Dir(filename), "dictcalls"))
	if err != nil {
		fmt.Printf("Could not load dictcalls, benchmarks won't be functional (see README.md)\n")
	}
}

var paper []byte = []byte(`
    Quicksort is a textbook divide-and-conquer algorithm.
    To sort an array, choose a partitioning element, permute
    the elements such that lesser elements are on one side and
    greater elements are on the other, and then recursively sort
    the two subarrays. But what happens to elements equal to
    the partitioning value? Hoare’s partitioning method is
    binary: it places lesser elements on the left and greater elements
    on the right, but equal elements may appear on
    either side.

    Algorithm designers have long recognized the desirability
    and difficulty of a ternary partitioning method.
    Sedgewick [22] observes on page 244: ‘‘Ideally, we would
    like to get all [equal keys] into position in the file, with all
    the keys with a smaller value to their left, and all the keys
    with a larger value to their right. Unfortunately, no
    efficient method for doing so has yet been devised....’’
    Dijkstra [6] popularized this as ‘‘The Problem of the Dutch
    National Flag’’: we are to order a sequence of red, white
    and blue pebbles to appear in their order on Holland’s
    ensign. This corresponds to Quicksort partitioning when
    lesser elements are colored red, equal elements are white,
    and greater elements are blue. Dijkstra’s ternary algorithm
    requires linear time (it looks at each element exactly once),
    but code to implement it has a significantly larger constant
    factor than Hoare’s binary partitioning code.

    Wegner [27] describes more efficient ternary partitioning
    schemes. Bentley and McIlroy [2] present a ternary
    partition based on this counterintuitive loop invariant:

    The main partitioning loop has two inner loops. The first
    inner loop moves up the index b: it scans over lesser elements,
    swaps equal elements to a, and halts on a greater
    element. The second inner loop moves down the index c
    correspondingly: it scans over greater elements, swaps
    equal elements to d, and halts on a lesser element. The
    main loop then swaps the elements pointed to by b and c,
    increments b and decrements c, and continues until b and
    c cross. (Wegner proposed the same invariant, but maintained
    it with more complex code.) Afterwards, the equal
    elements on the edges are swapped to the middle of the
    array, without any extraneous comparisons. This code partitions
    an n-element array using n − 1 comparisons

    Quicksort has been extensively analyzed by authors
    including Hoare [9], van Emden [26], Knuth [11], and
    Sedgewick [23]. Most detailed analyses involve the harmonic
    numbers Hn = Σ 1≤i≤n 1/ i.

    Theorem 1. [Hoare] A Quicksort that partitions
    around a single randomly selected element sorts n distinct
    items in 2nHn + O(n) ∼∼ 1. 386n lg n expected
    comparisons.

    A common variant of Quicksort partitions around the
    median of a random sample.

    Theorem 2. [van Emden] A Quicksort that partitions
    around the median of 2t + 1 randomly selected elements
    sorts n distinct items in 2nHn / (H2t + 2 − Ht + 1 )
    + O(n) expected comparisons.
    By increasing t, we can push the expected number of comparisons
    close to n lg n + O(n).
    `)
<|endoftext|>


<commit_before>package users

import (
	"net/http"

	"github.com/goincremental/negroni-sessions"
	. "stevenbooru.cf/globals"
	"stevenbooru.cf/models"
)

type Middleware struct {
}

func (m *Middleware) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	sess := sessions.GetSession(r)

	uid, ok := sess.Get("uid").(string)
	if !ok || uid == "" {
		next(rw, r)
		return
	}

	user := &models.User{}
	Db.Where("uuid = ?", uid).First(user)

	r.Header.Set("uid", uid)
	r.Header.Set("username", user.DisplayName)
	r.Header.Set("role", user.Role)

	next(rw, r)
}
<commit_msg>users middleware: use defer<commit_after>package users

import (
	"net/http"

	"github.com/goincremental/negroni-sessions"
	. "stevenbooru.cf/globals"
	"stevenbooru.cf/models"
)

type Middleware struct {
}

func (m *Middleware) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	defer next(rw, r)

	sess := sessions.GetSession(r)

	uid, ok := sess.Get("uid").(string)
	if !ok || uid == "" {
		return
	}

	user := &models.User{}
	Db.Where("uuid = ?", uid).First(user)

	r.Header.Set("uid", uid)
	r.Header.Set("username", user.DisplayName)
	r.Header.Set("role", user.Role)
}
<|endoftext|>


<commit_before>package acceptance_test

import (
	"cf-pusher/cf_cli_adapter"
	"fmt"
	"io/ioutil"
	"lib/testsupport"
	"math/rand"
	"net/http"
	"os"
	"strings"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("external connectivity", func() {
	var (
		appA      string
		orgName   string
		spaceName string
		appRoute  string
		cli       *cf_cli_adapter.Adapter
	)

	BeforeEach(func() {
		if testConfig.Internetless {
			Skip("skipping external connectivity tests")
		}

		cli = &cf_cli_adapter.Adapter{CfCliPath: "cf"}
		appA = fmt.Sprintf("appA-%d", rand.Int31())

		orgName = testConfig.Prefix + "external-connectivity-org"
		spaceName = testConfig.Prefix + "space"
		setupOrgAndSpace(orgName, spaceName)

		By("unbinding all running ASGs")
		for _, sg := range testConfig.DefaultSecurityGroups {
			Expect(cf.Cf("unbind-running-security-group", sg).Wait(Timeout_Short)).To(gexec.Exit(0))
		}

		By("creating test-generated ASGs")
		for asgName, asgValue := range testASGs {
			createASG(cli, asgName, asgValue)
		}

		By("pushing the test app")
		pushProxy(appA)
		appRoute = fmt.Sprintf("http://%s.%s/", appA, config.AppsDomain)
	})

	AfterEach(func() {
		By("adding back all the original running ASGs")
		for _, sg := range testConfig.DefaultSecurityGroups {
			Expect(cf.Cf("bind-running-security-group", sg).Wait(Timeout_Short)).To(gexec.Exit(0))
		}

		By("deleting the test org")
		Expect(cf.Cf("delete-org", orgName, "-f").Wait(Timeout_Push)).To(gexec.Exit(0))

		By("removing test-generated ASGs")
		for asgName, _ := range testASGs {
			removeASG(cli, asgName)
		}
	})

	checkRequest := func(route string, expectedStatusCode int, expectedResponseSubstring string) error {
		resp, err := http.Get(route)
		if err != nil {
			return err
		}
		defer resp.Body.Close()

		respBytes, err := ioutil.ReadAll(resp.Body)
		Expect(err).NotTo(HaveOccurred())
		respBody := string(respBytes)

		if resp.StatusCode != expectedStatusCode {
			return fmt.Errorf("test http get to %s: expected response code %d but got %d.  response body:\n%s", route, expectedStatusCode, resp.StatusCode, respBody)
		}
		if !strings.Contains(respBody, expectedResponseSubstring) {
			return fmt.Errorf("test http get to %s: expected response to contain %q but instead saw:\n%s", route, expectedResponseSubstring, respBody)
		}
		return nil
	}

	canProxy := func() error {
		return checkRequest(appRoute+"proxy/example.com", 200, "Example Domain")
	}
	isReachable := func() error {
		return checkRequest(appRoute, 200, `{"ListenAddresses":[`)
	}
	canPing := func() error {
		return checkRequest(appRoute+"ping/example.com", 200, "Ping succeeded")
	}
	cannotProxy := func() error {
		return checkRequest(appRoute+"proxy/example.com", 500, "example.com")
	}
	cannotPing := func() error {
		return checkRequest(appRoute+"ping/example.com", 500, "Ping failed to destination: example.com")
	}

	Describe("basic (legacy) network behavior for an app", func() {
		It("makes the app reachable from the router, and the app can reach the internet only if allowed", func(done Done) {
			By("checking that the app is reachable via the router")
			Eventually(isReachable, "10s", "1s").Should(Succeed())
			Consistently(isReachable, "2s", "0.5s").Should(Succeed())

			By("checking that the app cannot reach the internet using http and dns")
			Eventually(cannotProxy, "10s", "1s").Should(Succeed())
			Consistently(cannotProxy, "2s", "0.5s").Should(Succeed())

			By("checking that the app cannot ping the internet (first time)")
			Consistently(cannotPing, "2s", "0.5s").Should(Succeed())

			By("creating and binding a tcp and udp security group")
			Expect(cli.BindSecurityGroup("tcp-asg", orgName, spaceName)).To(Succeed())
			Expect(cli.BindSecurityGroup("udp-asg", orgName, spaceName)).To(Succeed())

			By("restarting the app")
			Expect(cf.Cf("restart", appA).Wait(Timeout_Push)).To(gexec.Exit(0))

			By("checking that the app can use dns and http to reach the internet")
			Eventually(canProxy, "10s", "1s").Should(Succeed())
			Consistently(canProxy, "2s", "0.5s").Should(Succeed())

			By("checking that the app cannot ping the internet (second time)")
			Consistently(cannotPing, "2s", "1s").Should(Succeed())

			By("removing the tcp security groups")
			Expect(cli.UnbindSecurityGroup("tcp-asg", orgName, spaceName)).To(Succeed())

			By("restarting the app")
			Expect(cf.Cf("restart", appA).Wait(Timeout_Push)).To(gexec.Exit(0))

			By("checking that the app cannot use http to reach the internet")
			Consistently(cannotProxy, "2s", "0.5s").Should(Succeed())

			close(done)
		}, 180 /* <-- overall spec timeout in seconds */)

		It("allows outbound ICMP only if allowed", func(done Done) {
			if testConfig.SkipICMPTests {
				Skip("Test config has 'skip_icmp_test: true', skipping ICMP connectivity tests")
			}

			By("checking that the app cannot ping the internet")
			Consistently(cannotPing, "2s", "0.5s").Should(Succeed())

			By("creating and binding an icmp security group")
			Expect(cli.BindSecurityGroup("icmp-asg", orgName, spaceName)).To(Succeed())

			By("restarting the app")
			Expect(cf.Cf("restart", appA).Wait(Timeout_Push)).To(gexec.Exit(0))

			By("checking that the app can ping the internet")
			Eventually(canPing, "10s", "1s").Should(Succeed())
			Consistently(canPing, "2s", "0.5s").Should(Succeed())

			close(done)
		}, 180 /* <-- overall spec timeout in seconds */)
	})
})

func createASG(cli *cf_cli_adapter.Adapter, name string, asgDefinition string) {
	asgFile, err := testsupport.CreateASGFile(asgDefinition)
	Expect(err).NotTo(HaveOccurred())
	Expect(cli.CreateSecurityGroup(name, asgFile)).To(Succeed())
	Expect(os.Remove(asgFile)).To(Succeed())
}

func removeASG(cli *cf_cli_adapter.Adapter, name string) {
	Expect(cli.DeleteSecurityGroup(name)).To(Succeed())
}

func setupOrgAndSpace(orgName, spaceName string) {
	Expect(cf.Cf("create-org", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))
	Expect(cf.Cf("target", "-o", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))

	Expect(cf.Cf("create-space", spaceName, "-o", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))
	Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait(Timeout_Push)).To(gexec.Exit(0))
}

var testASGs = map[string]string{
	"tcp-asg": `
	[
		{
			"destination": "0.0.0.0-255.255.255.255",
			"protocol": "tcp",
			"ports": "80,443"
		}
	]
	`,
	"udp-asg": `
	[
		{
			"destination": "0.0.0.0-255.255.255.255",
			"protocol": "udp",
			"ports": "53"
		}
	]
	`,
	"icmp-asg": `
	[
		{
			"destination": "0.0.0.0-255.255.255.255",
			"protocol": "icmp",
			"type": 8,
			"code": 0
		}
	]
	`,
}
<commit_msg>add udp-asg before icmp assertion<commit_after>package acceptance_test

import (
	"cf-pusher/cf_cli_adapter"
	"fmt"
	"io/ioutil"
	"lib/testsupport"
	"math/rand"
	"net/http"
	"os"
	"strings"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("external connectivity", func() {
	var (
		appA      string
		orgName   string
		spaceName string
		appRoute  string
		cli       *cf_cli_adapter.Adapter
	)

	BeforeEach(func() {
		if testConfig.Internetless {
			Skip("skipping external connectivity tests")
		}

		cli = &cf_cli_adapter.Adapter{CfCliPath: "cf"}
		appA = fmt.Sprintf("appA-%d", rand.Int31())

		orgName = testConfig.Prefix + "external-connectivity-org"
		spaceName = testConfig.Prefix + "space"
		setupOrgAndSpace(orgName, spaceName)

		By("unbinding all running ASGs")
		for _, sg := range testConfig.DefaultSecurityGroups {
			Expect(cf.Cf("unbind-running-security-group", sg).Wait(Timeout_Short)).To(gexec.Exit(0))
		}

		By("creating test-generated ASGs")
		for asgName, asgValue := range testASGs {
			createASG(cli, asgName, asgValue)
		}

		By("pushing the test app")
		pushProxy(appA)
		appRoute = fmt.Sprintf("http://%s.%s/", appA, config.AppsDomain)
	})

	AfterEach(func() {
		By("adding back all the original running ASGs")
		for _, sg := range testConfig.DefaultSecurityGroups {
			Expect(cf.Cf("bind-running-security-group", sg).Wait(Timeout_Short)).To(gexec.Exit(0))
		}

		By("deleting the test org")
		Expect(cf.Cf("delete-org", orgName, "-f").Wait(Timeout_Push)).To(gexec.Exit(0))

		By("removing test-generated ASGs")
		for asgName, _ := range testASGs {
			removeASG(cli, asgName)
		}
	})

	checkRequest := func(route string, expectedStatusCode int, expectedResponseSubstring string) error {
		resp, err := http.Get(route)
		if err != nil {
			return err
		}
		defer resp.Body.Close()

		respBytes, err := ioutil.ReadAll(resp.Body)
		Expect(err).NotTo(HaveOccurred())
		respBody := string(respBytes)

		if resp.StatusCode != expectedStatusCode {
			return fmt.Errorf("test http get to %s: expected response code %d but got %d.  response body:\n%s", route, expectedStatusCode, resp.StatusCode, respBody)
		}
		if !strings.Contains(respBody, expectedResponseSubstring) {
			return fmt.Errorf("test http get to %s: expected response to contain %q but instead saw:\n%s", route, expectedResponseSubstring, respBody)
		}
		return nil
	}

	canProxy := func() error {
		return checkRequest(appRoute+"proxy/example.com", 200, "Example Domain")
	}
	isReachable := func() error {
		return checkRequest(appRoute, 200, `{"ListenAddresses":[`)
	}
	canPing := func() error {
		return checkRequest(appRoute+"ping/example.com", 200, "Ping succeeded")
	}
	cannotProxy := func() error {
		return checkRequest(appRoute+"proxy/example.com", 500, "example.com")
	}
	cannotPing := func() error {
		return checkRequest(appRoute+"ping/example.com", 500, "Ping failed to destination: example.com")
	}

	Describe("basic (legacy) network behavior for an app", func() {
		It("makes the app reachable from the router, and the app can reach the internet only if allowed", func(done Done) {
			By("checking that the app is reachable via the router")
			Eventually(isReachable, "10s", "1s").Should(Succeed())
			Consistently(isReachable, "2s", "0.5s").Should(Succeed())

			By("checking that the app cannot reach the internet using http and dns")
			Eventually(cannotProxy, "10s", "1s").Should(Succeed())
			Consistently(cannotProxy, "2s", "0.5s").Should(Succeed())

			By("checking that the app cannot ping the internet (first time)")
			Consistently(cannotPing, "2s", "0.5s").Should(Succeed())

			By("creating and binding a tcp and udp security group")
			Expect(cli.BindSecurityGroup("tcp-asg", orgName, spaceName)).To(Succeed())
			Expect(cli.BindSecurityGroup("udp-asg", orgName, spaceName)).To(Succeed())

			By("restarting the app")
			Expect(cf.Cf("restart", appA).Wait(Timeout_Push)).To(gexec.Exit(0))

			By("checking that the app can use dns and http to reach the internet")
			Eventually(canProxy, "10s", "1s").Should(Succeed())
			Consistently(canProxy, "2s", "0.5s").Should(Succeed())

			By("checking that the app cannot ping the internet (second time)")
			Consistently(cannotPing, "2s", "1s").Should(Succeed())

			By("removing the tcp security groups")
			Expect(cli.UnbindSecurityGroup("tcp-asg", orgName, spaceName)).To(Succeed())

			By("restarting the app")
			Expect(cf.Cf("restart", appA).Wait(Timeout_Push)).To(gexec.Exit(0))

			By("checking that the app cannot use http to reach the internet")
			Consistently(cannotProxy, "2s", "0.5s").Should(Succeed())

			close(done)
		}, 180 /* <-- overall spec timeout in seconds */)

		It("allows outbound ICMP only if allowed", func(done Done) {
			if testConfig.SkipICMPTests {
				Skip("Test config has 'skip_icmp_test: true', skipping ICMP connectivity tests")
			}

			By("checking that the app cannot ping the internet")
			Consistently(cannotPing, "2s", "0.5s").Should(Succeed())

			By("creating and binding an icmp security group")
			Expect(cli.BindSecurityGroup("icmp-asg", orgName, spaceName)).To(Succeed())

			Expect(cli.BindSecurityGroup("udp-asg", orgName, spaceName)).To(Succeed())

			By("restarting the app")
			Expect(cf.Cf("restart", appA).Wait(Timeout_Push)).To(gexec.Exit(0))

			By("checking that the app can ping the internet")
			Eventually(canPing, "10s", "1s").Should(Succeed())
			Consistently(canPing, "2s", "0.5s").Should(Succeed())

			close(done)
		}, 180 /* <-- overall spec timeout in seconds */)
	})
})

func createASG(cli *cf_cli_adapter.Adapter, name string, asgDefinition string) {
	asgFile, err := testsupport.CreateASGFile(asgDefinition)
	Expect(err).NotTo(HaveOccurred())
	Expect(cli.CreateSecurityGroup(name, asgFile)).To(Succeed())
	Expect(os.Remove(asgFile)).To(Succeed())
}

func removeASG(cli *cf_cli_adapter.Adapter, name string) {
	Expect(cli.DeleteSecurityGroup(name)).To(Succeed())
}

func setupOrgAndSpace(orgName, spaceName string) {
	Expect(cf.Cf("create-org", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))
	Expect(cf.Cf("target", "-o", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))

	Expect(cf.Cf("create-space", spaceName, "-o", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))
	Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait(Timeout_Push)).To(gexec.Exit(0))
}

var testASGs = map[string]string{
	"tcp-asg": `
	[
		{
			"destination": "0.0.0.0-255.255.255.255",
			"protocol": "tcp",
			"ports": "80,443"
		}
	]
	`,
	"udp-asg": `
	[
		{
			"destination": "0.0.0.0-255.255.255.255",
			"protocol": "udp",
			"ports": "53"
		}
	]
	`,
	"icmp-asg": `
	[
		{
			"destination": "0.0.0.0-255.255.255.255",
			"protocol": "icmp",
			"type": 8,
			"code": 0
		}
	]
	`,
}
<|endoftext|>


<commit_before>package spaten

import (
	"io"

	"github.com/thomersch/grandine/lib/spatial"
)

type Codec struct{}

func (c *Codec) Encode(w io.Writer, fc *spatial.FeatureCollection) error {
	err := WriteFileHeader(w)
	if err != nil {
		return err
	}

	for _, ftBlk := range geomBlocks(100, fc.Features) {
		var meta map[string]interface{}
		if len(fc.SRID) != 0 {
			meta = map[string]interface{}{
				"@srid": fc.SRID,
			}
		}

		err = WriteBlock(w, ftBlk, meta)
		if err != nil {
			return err
		}
	}
	return nil
}

// ChunkedDecode is the preferred method for reading large datasets. It retrieves a file block
// at a time, making it possible to traverse the file in a streaming manner without allocating
// enough memory to fit the whole file.
func (c *Codec) ChunkedDecode(r io.Reader) (spatial.Chunks, error) {
	_, err := ReadFileHeader(r)
	if err != nil {
		return nil, err
	}
	return &Chunks{
		reader: r,
	}, nil
}

func (c *Codec) Decode(r io.Reader, fc *spatial.FeatureCollection) error {
	_, err := ReadFileHeader(r)
	if err != nil {
		return err
	}
	err = ReadBlocks(r, fc)
	return err
}

func (c *Codec) Extensions() []string {
	return []string{"spaten"}
}

// geomBlocks slices a slice of geometries into slices with a max size
func geomBlocks(size int, src []spatial.Feature) [][]spatial.Feature {
	if len(src) <= size {
		return [][]spatial.Feature{src}
	}

	var (
		i   int
		res [][]spatial.Feature
		end int
	)
	for end < len(src) {
		end = (i + 1) * size
		if end > len(src) {
			end = len(src)
		}
		res = append(res, src[i*size:end])
		i++
	}
	return res
}
<commit_msg>lib/spaten: chunked spaten encoder<commit_after>package spaten

import (
	"io"

	"github.com/thomersch/grandine/lib/spatial"
)

type Codec struct {
	headerWritten bool
}

func (c *Codec) Encode(w io.Writer, fc *spatial.FeatureCollection) error {
	err := WriteFileHeader(w)
	if err != nil {
		return err
	}

	for _, ftBlk := range geomBlocks(100, fc.Features) {
		var meta map[string]interface{}
		if len(fc.SRID) != 0 {
			meta = map[string]interface{}{
				"@srid": fc.SRID,
			}
		}

		err = WriteBlock(w, ftBlk, meta)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *Codec) EncodeChunk(w io.Writer, fc *spatial.FeatureCollection) error {
	if !c.headerWritten {
		err := WriteFileHeader(w)
		if err != nil {
			return err
		}
		c.headerWritten = true
	}
	return WriteBlock(w, fc.Features, nil)
}

func (c *Codec) Close() error {
	return nil
}

// ChunkedDecode is the preferred method for reading large datasets. It retrieves a file block
// at a time, making it possible to traverse the file in a streaming manner without allocating
// enough memory to fit the whole file.
func (c *Codec) ChunkedDecode(r io.Reader) (spatial.Chunks, error) {
	_, err := ReadFileHeader(r)
	if err != nil {
		return nil, err
	}
	return &Chunks{
		reader: r,
	}, nil
}

func (c *Codec) Decode(r io.Reader, fc *spatial.FeatureCollection) error {
	_, err := ReadFileHeader(r)
	if err != nil {
		return err
	}
	err = ReadBlocks(r, fc)
	return err
}

func (c *Codec) Extensions() []string {
	return []string{"spaten"}
}

// geomBlocks slices a slice of geometries into slices with a max size
func geomBlocks(size int, src []spatial.Feature) [][]spatial.Feature {
	if len(src) <= size {
		return [][]spatial.Feature{src}
	}

	var (
		i   int
		res [][]spatial.Feature
		end int
	)
	for end < len(src) {
		end = (i + 1) * size
		if end > len(src) {
			end = len(src)
		}
		res = append(res, src[i*size:end])
		i++
	}
	return res
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	verbose    bool
	keepGopath bool
	runTests   bool
)

var config = struct {
	Name      string
	Namespace string
	Main      string
	Tests     []string
}{
	Name:      "restic",                           // name of the program executable and directory
	Namespace: "",                                 // subdir of GOPATH, e.g. "github.com/foo/bar"
	Main:      "cmds/restic",                      // package name for the main package
	Tests:     []string{"restic/...", "cmds/..."}, // tests to run
}

// specialDir returns true if the file begins with a special character ('.' or '_').
func specialDir(name string) bool {
	if name == "." {
		return false
	}

	base := filepath.Base(name)
	if base == "vendor" || base[0] == '_' || base[0] == '.' {
		return true
	}

	return false
}

// excludePath returns true if the file should not be copied to the new GOPATH.
func excludePath(name string) bool {
	ext := path.Ext(name)
	if ext == ".go" || ext == ".s" {
		return false
	}

	parentDir := filepath.Base(filepath.Dir(name))
	if parentDir == "testdata" {
		return false
	}

	return true
}

// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
// to dst/prefix/, so calling
//
//   updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic")
//
// with "/home/u/restic" containing the file "foo.go" yields the following tree
// at "/tmp/gopath":
//
//   /tmp/gopath
//   └── src
//       └── github.com
//           └── restic
//               └── restic
//                   └── foo.go
func updateGopath(dst, src, prefix string) error {
	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
		if specialDir(name) {
			if fi.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if fi.IsDir() {
			return nil
		}

		if excludePath(name) {
			return nil
		}

		intermediatePath, err := filepath.Rel(src, name)
		if err != nil {
			return err
		}

		fileSrc := filepath.Join(src, intermediatePath)
		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)

		return copyFile(fileDst, fileSrc)
	})
}

func directoryExists(dirname string) bool {
	stat, err := os.Stat(dirname)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return stat.IsDir()
}

// copyFile creates dst from src, preserving file attributes and timestamps.
func copyFile(dst, src string) error {
	fi, err := os.Stat(src)
	if err != nil {
		return err
	}

	fsrc, err := os.Open(src)
	if err != nil {
		return err
	}

	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst))
		return err
	}

	fdst, err := os.Create(dst)
	if err != nil {
		return err
	}

	if _, err = io.Copy(fdst, fsrc); err != nil {
		return err
	}

	if err == nil {
		err = fsrc.Close()
	}

	if err == nil {
		err = fdst.Close()
	}

	if err == nil {
		err = os.Chmod(dst, fi.Mode())
	}

	if err == nil {
		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
	}

	return nil
}

// die prints the message with fmt.Fprintf() to stderr and exits with an error
// code.
func die(message string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

func showUsage(output io.Writer) {
	fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n")
	fmt.Fprintf(output, "\n")
	fmt.Fprintf(output, "OPTIONS:\n")
	fmt.Fprintf(output, "  -v     --verbose       output more messages\n")
	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
	fmt.Fprintf(output, "  -T     --test          run tests\n")
	fmt.Fprintf(output, "  -o     --output        set output file name\n")
	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
}

func verbosePrintf(message string, args ...interface{}) {
	if !verbose {
		return
	}

	fmt.Printf("build: "+message, args...)
}

// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if
// present).
func cleanEnv() (env []string) {
	for _, v := range os.Environ() {
		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
			continue
		}

		env = append(env, v)
	}

	return env
}

// build runs "go build args..." with GOPATH set to gopath.
func build(cwd, goos, goarch, gopath string, args ...string) error {
	args = append([]string{"build"}, args...)
	cmd := exec.Command("go", args...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos, "CGO_ENABLED=0")
	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// test runs "go test args..." with GOPATH set to gopath.
func test(cwd, gopath string, args ...string) error {
	args = append([]string{"test"}, args...)
	cmd := exec.Command("go", args...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// getVersion returns the version string from the file VERSION in the current
// directory.
func getVersionFromFile() string {
	buf, err := ioutil.ReadFile("VERSION")
	if err != nil {
		verbosePrintf("error reading file VERSION: %v\n", err)
		return ""
	}

	return strings.TrimSpace(string(buf))
}

// getVersion returns a version string which is a combination of the contents
// of the file VERSION in the current directory and the version from git (if
// available).
func getVersion() string {
	versionFile := getVersionFromFile()
	versionGit := getVersionFromGit()

	verbosePrintf("version from file 'VERSION' is %q, version from git %q\n",
		versionFile, versionGit)

	switch {
	case versionFile == "":
		return versionGit
	case versionGit == "":
		return versionFile
	}

	return fmt.Sprintf("%s (%s)", versionFile, versionGit)
}

// getVersionFromGit returns a version string that identifies the currently
// checked out git commit.
func getVersionFromGit() string {
	cmd := exec.Command("git", "describe",
		"--long", "--tags", "--dirty", "--always")
	out, err := cmd.Output()
	if err != nil {
		verbosePrintf("git describe returned error: %v\n", err)
		return ""
	}

	version := strings.TrimSpace(string(out))
	verbosePrintf("git version is %s\n", version)
	return version
}

// Constants represents a set of constants that are set in the final binary to
// the given value via compiler flags.
type Constants map[string]string

// LDFlags returns the string that can be passed to go build's `-ldflags`.
func (cs Constants) LDFlags() string {
	l := make([]string, 0, len(cs))

	for k, v := range cs {
		l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
	}

	return strings.Join(l, " ")
}

func main() {
	buildTags := []string{}

	skipNext := false
	params := os.Args[1:]

	targetGOOS := runtime.GOOS
	targetGOARCH := runtime.GOARCH

	var outputFilename string

	for i, arg := range params {
		if skipNext {
			skipNext = false
			continue
		}

		switch arg {
		case "-v", "--verbose":
			verbose = true
		case "-k", "--keep-gopath":
			keepGopath = true
		case "-t", "-tags", "--tags":
			if i+1 >= len(params) {
				die("-t given but no tag specified")
			}
			skipNext = true
			buildTags = strings.Split(params[i+1], " ")
		case "-o", "--output":
			skipNext = true
			outputFilename = params[i+1]
		case "-T", "--test":
			runTests = true
		case "--goos":
			skipNext = true
			targetGOOS = params[i+1]
		case "--goarch":
			skipNext = true
			targetGOARCH = params[i+1]
		case "-h":
			showUsage(os.Stdout)
			return
		default:
			fmt.Fprintf(os.Stderr, "Error: unknown option %q\n\n", arg)
			showUsage(os.Stderr)
			os.Exit(1)
		}
	}

	if len(buildTags) == 0 {
		verbosePrintf("adding build-tag release\n")
		buildTags = []string{"release"}
	}

	for i := range buildTags {
		buildTags[i] = strings.TrimSpace(buildTags[i])
	}

	verbosePrintf("build tags: %s\n", buildTags)

	root, err := os.Getwd()
	if err != nil {
		die("Getwd(): %v\n", err)
	}

	gopath, err := ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
	if err != nil {
		die("TempDir(): %v\n", err)
	}

	verbosePrintf("create GOPATH at %v\n", gopath)
	if err = updateGopath(gopath, filepath.Join(root, "src"), config.Namespace); err != nil {
		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
	}

	vendor := filepath.Join(root, "vendor", "src")
	if directoryExists(vendor) {
		if err = updateGopath(gopath, vendor, ""); err != nil {
			die("copying files from %v to %v failed: %v\n", root, gopath, err)
		}
	}

	defer func() {
		if !keepGopath {
			verbosePrintf("remove %v\n", gopath)
			if err = os.RemoveAll(gopath); err != nil {
				die("remove GOPATH at %s failed: %v\n", err)
			}
		} else {
			verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
		}
	}()

	if outputFilename == "" {
		outputFilename = config.Name
		if targetGOOS == "windows" {
			outputFilename += ".exe"
		}
	}

	cwd, err := os.Getwd()
	if err != nil {
		die("Getwd() returned %v\n", err)
	}
	output := filepath.Join(cwd, outputFilename)

	version := getVersion()
	constants := Constants{}
	if version != "" {
		constants["main.version"] = version
	}
	ldflags := "-s -w " + constants.LDFlags()
	verbosePrintf("ldflags: %s\n", ldflags)

	args := []string{
		"-tags", strings.Join(buildTags, " "),
		"-ldflags", ldflags,
		"-o", output, config.Main,
	}

	err = build(filepath.Join(gopath, "src"), targetGOOS, targetGOARCH, gopath, args...)
	if err != nil {
		die("build failed: %v\n", err)
	}

	if runTests {
		verbosePrintf("running tests\n")

		err = test(cwd, gopath, config.Tests...)
		if err != nil {
			die("running tests failed: %v\n", err)
		}
	}
}
<commit_msg>Check version in build.go<commit_after>// +build ignore

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	verbose    bool
	keepGopath bool
	runTests   bool
)

var config = struct {
	Name      string
	Namespace string
	Main      string
	Tests     []string
}{
	Name:      "restic",                           // name of the program executable and directory
	Namespace: "",                                 // subdir of GOPATH, e.g. "github.com/foo/bar"
	Main:      "cmds/restic",                      // package name for the main package
	Tests:     []string{"restic/...", "cmds/..."}, // tests to run
}

// specialDir returns true if the file begins with a special character ('.' or '_').
func specialDir(name string) bool {
	if name == "." {
		return false
	}

	base := filepath.Base(name)
	if base == "vendor" || base[0] == '_' || base[0] == '.' {
		return true
	}

	return false
}

// excludePath returns true if the file should not be copied to the new GOPATH.
func excludePath(name string) bool {
	ext := path.Ext(name)
	if ext == ".go" || ext == ".s" {
		return false
	}

	parentDir := filepath.Base(filepath.Dir(name))
	if parentDir == "testdata" {
		return false
	}

	return true
}

// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
// to dst/prefix/, so calling
//
//   updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic")
//
// with "/home/u/restic" containing the file "foo.go" yields the following tree
// at "/tmp/gopath":
//
//   /tmp/gopath
//   └── src
//       └── github.com
//           └── restic
//               └── restic
//                   └── foo.go
func updateGopath(dst, src, prefix string) error {
	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
		if specialDir(name) {
			if fi.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if fi.IsDir() {
			return nil
		}

		if excludePath(name) {
			return nil
		}

		intermediatePath, err := filepath.Rel(src, name)
		if err != nil {
			return err
		}

		fileSrc := filepath.Join(src, intermediatePath)
		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)

		return copyFile(fileDst, fileSrc)
	})
}

func directoryExists(dirname string) bool {
	stat, err := os.Stat(dirname)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return stat.IsDir()
}

// copyFile creates dst from src, preserving file attributes and timestamps.
func copyFile(dst, src string) error {
	fi, err := os.Stat(src)
	if err != nil {
		return err
	}

	fsrc, err := os.Open(src)
	if err != nil {
		return err
	}

	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst))
		return err
	}

	fdst, err := os.Create(dst)
	if err != nil {
		return err
	}

	if _, err = io.Copy(fdst, fsrc); err != nil {
		return err
	}

	if err == nil {
		err = fsrc.Close()
	}

	if err == nil {
		err = fdst.Close()
	}

	if err == nil {
		err = os.Chmod(dst, fi.Mode())
	}

	if err == nil {
		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
	}

	return nil
}

// die prints the message with fmt.Fprintf() to stderr and exits with an error
// code.
func die(message string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

func showUsage(output io.Writer) {
	fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n")
	fmt.Fprintf(output, "\n")
	fmt.Fprintf(output, "OPTIONS:\n")
	fmt.Fprintf(output, "  -v     --verbose       output more messages\n")
	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
	fmt.Fprintf(output, "  -T     --test          run tests\n")
	fmt.Fprintf(output, "  -o     --output        set output file name\n")
	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
}

func verbosePrintf(message string, args ...interface{}) {
	if !verbose {
		return
	}

	fmt.Printf("build: "+message, args...)
}

// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if
// present).
func cleanEnv() (env []string) {
	for _, v := range os.Environ() {
		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
			continue
		}

		env = append(env, v)
	}

	return env
}

// build runs "go build args..." with GOPATH set to gopath.
func build(cwd, goos, goarch, gopath string, args ...string) error {
	args = append([]string{"build"}, args...)
	cmd := exec.Command("go", args...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos, "CGO_ENABLED=0")
	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// test runs "go test args..." with GOPATH set to gopath.
func test(cwd, gopath string, args ...string) error {
	args = append([]string{"test"}, args...)
	cmd := exec.Command("go", args...)
	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
	cmd.Dir = cwd
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	verbosePrintf("go %s\n", args)

	return cmd.Run()
}

// getVersion returns the version string from the file VERSION in the current
// directory.
func getVersionFromFile() string {
	buf, err := ioutil.ReadFile("VERSION")
	if err != nil {
		verbosePrintf("error reading file VERSION: %v\n", err)
		return ""
	}

	return strings.TrimSpace(string(buf))
}

// getVersion returns a version string which is a combination of the contents
// of the file VERSION in the current directory and the version from git (if
// available).
func getVersion() string {
	versionFile := getVersionFromFile()
	versionGit := getVersionFromGit()

	verbosePrintf("version from file 'VERSION' is %q, version from git %q\n",
		versionFile, versionGit)

	switch {
	case versionFile == "":
		return versionGit
	case versionGit == "":
		return versionFile
	}

	return fmt.Sprintf("%s (%s)", versionFile, versionGit)
}

// getVersionFromGit returns a version string that identifies the currently
// checked out git commit.
func getVersionFromGit() string {
	cmd := exec.Command("git", "describe",
		"--long", "--tags", "--dirty", "--always")
	out, err := cmd.Output()
	if err != nil {
		verbosePrintf("git describe returned error: %v\n", err)
		return ""
	}

	version := strings.TrimSpace(string(out))
	verbosePrintf("git version is %s\n", version)
	return version
}

// Constants represents a set of constants that are set in the final binary to
// the given value via compiler flags.
type Constants map[string]string

// LDFlags returns the string that can be passed to go build's `-ldflags`.
func (cs Constants) LDFlags() string {
	l := make([]string, 0, len(cs))

	for k, v := range cs {
		l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
	}

	return strings.Join(l, " ")
}

func main() {
	ver := runtime.Version()
	if strings.HasPrefix(ver, "go1") && ver < "go1.7" {
		fmt.Fprintf(os.Stderr, "Go version %s detected, restic requires at least Go 1.7\n", ver)
		os.Exit(1)
	}

	buildTags := []string{}

	skipNext := false
	params := os.Args[1:]

	targetGOOS := runtime.GOOS
	targetGOARCH := runtime.GOARCH

	var outputFilename string

	for i, arg := range params {
		if skipNext {
			skipNext = false
			continue
		}

		switch arg {
		case "-v", "--verbose":
			verbose = true
		case "-k", "--keep-gopath":
			keepGopath = true
		case "-t", "-tags", "--tags":
			if i+1 >= len(params) {
				die("-t given but no tag specified")
			}
			skipNext = true
			buildTags = strings.Split(params[i+1], " ")
		case "-o", "--output":
			skipNext = true
			outputFilename = params[i+1]
		case "-T", "--test":
			runTests = true
		case "--goos":
			skipNext = true
			targetGOOS = params[i+1]
		case "--goarch":
			skipNext = true
			targetGOARCH = params[i+1]
		case "-h":
			showUsage(os.Stdout)
			return
		default:
			fmt.Fprintf(os.Stderr, "Error: unknown option %q\n\n", arg)
			showUsage(os.Stderr)
			os.Exit(1)
		}
	}

	if len(buildTags) == 0 {
		verbosePrintf("adding build-tag release\n")
		buildTags = []string{"release"}
	}

	for i := range buildTags {
		buildTags[i] = strings.TrimSpace(buildTags[i])
	}

	verbosePrintf("build tags: %s\n", buildTags)

	root, err := os.Getwd()
	if err != nil {
		die("Getwd(): %v\n", err)
	}

	gopath, err := ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
	if err != nil {
		die("TempDir(): %v\n", err)
	}

	verbosePrintf("create GOPATH at %v\n", gopath)
	if err = updateGopath(gopath, filepath.Join(root, "src"), config.Namespace); err != nil {
		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
	}

	vendor := filepath.Join(root, "vendor", "src")
	if directoryExists(vendor) {
		if err = updateGopath(gopath, vendor, ""); err != nil {
			die("copying files from %v to %v failed: %v\n", root, gopath, err)
		}
	}

	defer func() {
		if !keepGopath {
			verbosePrintf("remove %v\n", gopath)
			if err = os.RemoveAll(gopath); err != nil {
				die("remove GOPATH at %s failed: %v\n", err)
			}
		} else {
			verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
		}
	}()

	if outputFilename == "" {
		outputFilename = config.Name
		if targetGOOS == "windows" {
			outputFilename += ".exe"
		}
	}

	cwd, err := os.Getwd()
	if err != nil {
		die("Getwd() returned %v\n", err)
	}
	output := filepath.Join(cwd, outputFilename)

	version := getVersion()
	constants := Constants{}
	if version != "" {
		constants["main.version"] = version
	}
	ldflags := "-s -w " + constants.LDFlags()
	verbosePrintf("ldflags: %s\n", ldflags)

	args := []string{
		"-tags", strings.Join(buildTags, " "),
		"-ldflags", ldflags,
		"-o", output, config.Main,
	}

	err = build(filepath.Join(gopath, "src"), targetGOOS, targetGOARCH, gopath, args...)
	if err != nil {
		die("build failed: %v\n", err)
	}

	if runTests {
		verbosePrintf("running tests\n")

		err = test(cwd, gopath, config.Tests...)
		if err != nil {
			die("running tests failed: %v\n", err)
		}
	}
}
<|endoftext|>


<commit_before>package metainspector

import (
	"code.google.com/p/go.net/html"
	"fmt"
	"net/url"
	"testing"
)

func TestFindCharset(t *testing.T) {
	content := "text/html;charset=utf-8"
	if charset := ExportFindCharset(content); charset != "utf-8" {
		t.Errorf(msgFail, "Charset", "utf-8", charset)
	}

	content = "text/html"
	if charset := ExportFindCharset(content); charset != "" {
		t.Errorf(msgFail, "Charset", "", charset)
	}
}

func TestMapifyStr(t *testing.T) {
	content := "foo=3,bar=sh,roo=pi"
	m := ExportMapifyStr(content)
	if m["foo"] != "3" {
		t.Errorf(msgFail, "m[foo]", "3", m["foo"])
	} else if m["bar"] != "sh" {
		t.Errorf(msgFail, "m[bar]", "sh", m["bar"])
	} else if m["roo"] != "pi" {
		t.Errorf(msgFail, "m[roo]", "pi", m["roo"])
	}
}

func TestFindAttribute(t *testing.T) {
	n := &html.Node{
		Type:      1,
		DataAtom:  1,
		Data:      "meta",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "name",
				Val:       "author"},
			html.Attribute{
				Namespace: "",
				Key:       "content",
				Val:       "cloudControl GmbH"}}}
	key := "content"

	if at := ExportFindAttribute(n, key); at != "cloudControl GmbH" {
		t.Errorf(msgFail, "Attribute", "cloudControl GmbH", at)
	}
}

func TestAddElement(t *testing.T) {
	elems := make([]string, 0)

	n1 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{Namespace: "",
				Key: "href",
				Val: "/pricing"}}}
	n2 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "href",
				Val:       "http://status.cloudcontrol.com"}}}
	n3 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "href",
				Val:       "//console.cloudcontrol.com"}}}
	n4 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "href",
				Val:       "#part"}}}

	u, err := url.Parse("http://www.cloudcontrol.com")
	if err != nil {
		t.Errorf("%v", err)
	}

	attr := "href"

	if e1 := fmt.Sprintf("%v", ExportAddElement(elems, u, n1, attr)); e1 != "[http://www.cloudcontrol.com/pricing]" {
		t.Errorf(msgFail, "Internal Link", "[http://www.cloudcontrol.com/pricing]", e1)
	}

	if e2 := fmt.Sprintf("%v", ExportAddElement(elems, u, n2, attr)); e2 != "[http://status.cloudcontrol.com]" {
		t.Errorf(msgFail, "External Link", "[http://status.cloudcontrol.com]", e2)
	}

	if e3 := fmt.Sprintf("%v", ExportAddElement(elems, u, n3, attr)); e3 != "[http://console.cloudcontrol.com]" {
		t.Errorf(msgFail, "External Link 2", "[http://console.cloudcontrol.com]", e3)
	}

	if e4 := fmt.Sprintf("%v", ExportAddElement(elems, u, n4, attr)); e4 != "[http://www.cloudcontrol.com#part]" {
		t.Errorf(msgFail, "Internal Link #", "[http://www.cloudcontrol.com#part]", e4)
	}
}

func TestTimeoutDialer(t *testing.T) {
	secs := 20
	f1 := ExportTimeoutDialer(secs)
	c, err := f1("tcp", "google.com:80")
	if err != nil {
		t.Errorf(msgFail, "Timeout Dialer", "TCPConn", c)
	}
}

func TestNewScraper(t *testing.T) {
	u, err := url.Parse("http://www.cloudcontrol.com")
	if err != nil {
		t.Errorf("%v", err)
	}

	_, err = ExportNewScraper(u, 20)
	if err != nil {
		t.Errorf(msgFail, "scraper", "Scraper", err)
	}
}

func TestHasProtocolAsPrefix(t *testing.T) {
	val1 := "http://example.com/image.jpg"
	b1 := ExportHasProtocolAsPrefix(val1)
	if !b1 {
		t.Errorf(msgFail, "HasProtocolAsPrefix", true, false)
	}

	val2 := "image.jpg"
	b2 := ExportHasProtocolAsPrefix(val2)
	if b2 {
		t.Errorf(msgFail, "HasProtocolAsPrefix", false, true)
	}

	val3 := "https://www.example.com/image.jpg"
	b3 := ExportHasProtocolAsPrefix(val3)
	if b3 {
		t.Errorf(msgFail, "HasProtocolAsPrefix", true, false)
	}

}
<commit_msg>Fix test<commit_after>package metainspector

import (
	"code.google.com/p/go.net/html"
	"fmt"
	"net/url"
	"testing"
)

func TestFindCharset(t *testing.T) {
	content := "text/html;charset=utf-8"
	if charset := ExportFindCharset(content); charset != "utf-8" {
		t.Errorf(msgFail, "Charset", "utf-8", charset)
	}

	content = "text/html"
	if charset := ExportFindCharset(content); charset != "" {
		t.Errorf(msgFail, "Charset", "", charset)
	}
}

func TestMapifyStr(t *testing.T) {
	content := "foo=3,bar=sh,roo=pi"
	m := ExportMapifyStr(content)
	if m["foo"] != "3" {
		t.Errorf(msgFail, "m[foo]", "3", m["foo"])
	} else if m["bar"] != "sh" {
		t.Errorf(msgFail, "m[bar]", "sh", m["bar"])
	} else if m["roo"] != "pi" {
		t.Errorf(msgFail, "m[roo]", "pi", m["roo"])
	}
}

func TestFindAttribute(t *testing.T) {
	n := &html.Node{
		Type:      1,
		DataAtom:  1,
		Data:      "meta",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "name",
				Val:       "author"},
			html.Attribute{
				Namespace: "",
				Key:       "content",
				Val:       "cloudControl GmbH"}}}
	key := "content"

	if at := ExportFindAttribute(n, key); at != "cloudControl GmbH" {
		t.Errorf(msgFail, "Attribute", "cloudControl GmbH", at)
	}
}

func TestAddElement(t *testing.T) {
	elems := make([]string, 0)

	n1 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{Namespace: "",
				Key: "href",
				Val: "/pricing"}}}
	n2 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "href",
				Val:       "http://status.cloudcontrol.com"}}}
	n3 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "href",
				Val:       "//console.cloudcontrol.com"}}}
	n4 := &html.Node{
		Type:      3,
		DataAtom:  1,
		Data:      "a",
		Namespace: "",
		Attr: []html.Attribute{
			html.Attribute{
				Namespace: "",
				Key:       "href",
				Val:       "#part"}}}

	u, err := url.Parse("http://www.cloudcontrol.com")
	if err != nil {
		t.Errorf("%v", err)
	}

	attr := "href"

	if e1 := fmt.Sprintf("%v", ExportAddElement(elems, u, n1, attr)); e1 != "[http://www.cloudcontrol.com/pricing]" {
		t.Errorf(msgFail, "Internal Link", "[http://www.cloudcontrol.com/pricing]", e1)
	}

	if e2 := fmt.Sprintf("%v", ExportAddElement(elems, u, n2, attr)); e2 != "[http://status.cloudcontrol.com]" {
		t.Errorf(msgFail, "External Link", "[http://status.cloudcontrol.com]", e2)
	}

	if e3 := fmt.Sprintf("%v", ExportAddElement(elems, u, n3, attr)); e3 != "[http://console.cloudcontrol.com]" {
		t.Errorf(msgFail, "External Link 2", "[http://console.cloudcontrol.com]", e3)
	}

	if e4 := fmt.Sprintf("%v", ExportAddElement(elems, u, n4, attr)); e4 != "[http://www.cloudcontrol.com#part]" {
		t.Errorf(msgFail, "Internal Link #", "[http://www.cloudcontrol.com#part]", e4)
	}
}

func TestTimeoutDialer(t *testing.T) {
	secs := 20
	f1 := ExportTimeoutDialer(secs)
	c, err := f1("tcp", "google.com:80")
	if err != nil {
		t.Errorf(msgFail, "Timeout Dialer", "TCPConn", c)
	}
}

func TestNewScraper(t *testing.T) {
	u, err := url.Parse("http://www.cloudcontrol.com")
	if err != nil {
		t.Errorf("%v", err)
	}

	_, err = ExportNewScraper(u, 20)
	if err != nil {
		t.Errorf(msgFail, "scraper", "Scraper", err)
	}
}

func TestHasProtocolAsPrefix(t *testing.T) {
	val1 := "http://example.com/image.jpg"
	b1 := ExportHasProtocolAsPrefix(val1)
	if !b1 {
		t.Errorf(msgFail, "HasProtocolAsPrefix", true, false)
	}

	val2 := "image.jpg"
	b2 := ExportHasProtocolAsPrefix(val2)
	if b2 {
		t.Errorf(msgFail, "HasProtocolAsPrefix", false, true)
	}

	val3 := "https://www.example.com/image.jpg"
	b3 := ExportHasProtocolAsPrefix(val3)
	if !b3 {
		t.Errorf(msgFail, "HasProtocolAsPrefix", true, false)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/stripe/sequins/blocks"
	"github.com/stripe/sequins/sequencefile"
)

// build prepares the version, blocking until all local partitions are ready,
// then returns it. If onlyFromManifest is true, it will only load data on local
// disk from a manifest, and fail otherwise.
func (vs *version) build(files []string) error {
	if vs.blockStore != nil {
		return nil
	}

	if len(files) == 0 {
		log.Println("Version", vs.name, "of", vs.db, "has no data. Loading it anyway.")
		return nil
	}

	var local map[int]bool
	if vs.sequins.peers != nil {
		local = vs.partitions.pickLocalPartitions()
		if len(local) == 0 {
			log.Println("All valid partitions for", vs.db, "version", vs.name,
				"are already spoken for. Consider increasing the replication level.")
			return nil
		}
	}

	log.Println("Loading", vs.db, "version", vs.name, "from",
		vs.sequins.backend.DisplayPath(vs.name), "into local directory", vs.path)

	blockStore, err := vs.createStore(files, local)
	if err != nil {
		return err
	}

	vs.blockStore = blockStore
	if vs.partitions != nil {
		vs.partitions.updateLocalPartitions(local)
	}

	return nil
}

// TODO: parallelize files

func (vs *version) createStore(files []string, partitions map[int]bool) (*blocks.BlockStore, error) {
	if _, err := os.Stat(vs.path); err == nil {
		log.Println("Clearing local directory", vs.path)
	}

	os.RemoveAll(vs.path)
	err := os.MkdirAll(vs.path, 0755|os.ModeDir)
	if err != nil {
		return nil, fmt.Errorf("error creating local storage directory: %s", err)
	}

	blockStore := blocks.New(vs.path, len(files), partitions)
	for _, file := range files {
		err := vs.addFile(blockStore, file)
		if err != nil {
			return nil, err
		}
	}

	blockStore.Save()
	return blockStore, nil
}

func (vs *version) addFile(bs *blocks.BlockStore, file string) error {
	disp := vs.sequins.backend.DisplayPath(vs.db, vs.name, file)
	log.Println("Reading records from", disp)

	stream, err := vs.sequins.backend.Open(vs.db, vs.name, file)
	if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	sf := sequencefile.New(stream)
	err = sf.ReadHeader()
	if err != nil {
		return fmt.Errorf("reading header from %s: %s", disp, err)
	}

	err = bs.AddFile(sf, vs.sequins.config.ThrottleLoads.Duration)
	if err == blocks.ErrWrongPartition {
		log.Println("Skipping", disp, "because it contains no relevant partitions")
	} else if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	return nil
}
<commit_msg>After building a dataset, list the files again to see if they changed<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
	"os"

	"github.com/stripe/sequins/blocks"
	"github.com/stripe/sequins/sequencefile"
)

var errFilesChanged = errors.New("the list of remote files changed while building")

// build prepares the version, blocking until all local partitions are ready,
// then returns it. If onlyFromManifest is true, it will only load data on local
// disk from a manifest, and fail otherwise.
func (vs *version) build(files []string) error {
	if vs.blockStore != nil {
		return nil
	}

	if len(files) == 0 {
		log.Println("Version", vs.name, "of", vs.db, "has no data. Loading it anyway.")
		return nil
	}

	var local map[int]bool
	if vs.sequins.peers != nil {
		local = vs.partitions.pickLocalPartitions()
		if len(local) == 0 {
			log.Println("All valid partitions for", vs.db, "version", vs.name,
				"are already spoken for. Consider increasing the replication level.")
			return nil
		}
	}

	log.Println("Loading", vs.db, "version", vs.name, "from",
		vs.sequins.backend.DisplayPath(vs.name), "into local directory", vs.path)

	blockStore, err := vs.createStore(files, local)
	if err != nil {
		return err
	}

	// Verify that the list of files stayed the same.
	newFiles, err := vs.sequins.backend.ListFiles(vs.db, vs.name)
	if err != nil {
		return err
	} else {
		if vs.compareFileSets(files, newFiles) {
			return errFilesChanged
		}
	}

	vs.blockStore = blockStore
	if vs.partitions != nil {
		vs.partitions.updateLocalPartitions(local)
	}

	return nil
}

func (vs *version) compareFileSets(oldFiles, newFiles []string) bool {
	setOld := make(map[string]bool, len(oldFiles))
	setNew := make(map[string]bool, len(newFiles))
	different := false

	if len(oldFiles) != len(newFiles) {
		log.Printf("Number of files under %s changed (%d vs %d)",
			vs.sequins.backend.DisplayPath(vs.db, vs.name), len(oldFiles), len(newFiles))
		different = true
	}

	for _, f := range oldFiles {
		setOld[f] = true
	}

	for _, f := range newFiles {
		setNew[f] = true
		if !setOld[f] {
			log.Println("New file:", vs.sequins.backend.DisplayPath(vs.db, vs.name, f))
			different = true
		}
	}

	for _, f := range oldFiles {
		if !setNew[f] {
			log.Println("Missing file:", vs.sequins.backend.DisplayPath(vs.db, vs.name, f))
			different = true
		}
	}

	return different
}

// TODO: parallelize files

func (vs *version) createStore(files []string, partitions map[int]bool) (*blocks.BlockStore, error) {
	if _, err := os.Stat(vs.path); err == nil {
		log.Println("Clearing local directory", vs.path)
	}

	os.RemoveAll(vs.path)
	err := os.MkdirAll(vs.path, 0755|os.ModeDir)
	if err != nil {
		return nil, fmt.Errorf("error creating local storage directory: %s", err)
	}

	blockStore := blocks.New(vs.path, len(files), partitions)
	for _, file := range files {
		err := vs.addFile(blockStore, file)
		if err != nil {
			return nil, err
		}
	}

	blockStore.Save()
	return blockStore, nil
}

func (vs *version) addFile(bs *blocks.BlockStore, file string) error {
	disp := vs.sequins.backend.DisplayPath(vs.db, vs.name, file)
	log.Println("Reading records from", disp)

	stream, err := vs.sequins.backend.Open(vs.db, vs.name, file)
	if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	sf := sequencefile.New(stream)
	err = sf.ReadHeader()
	if err != nil {
		return fmt.Errorf("reading header from %s: %s", disp, err)
	}

	err = bs.AddFile(sf, vs.sequins.config.ThrottleLoads.Duration)
	if err == blocks.ErrWrongPartition {
		log.Println("Skipping", disp, "because it contains no relevant partitions")
	} else if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	return nil
}
<|endoftext|>