<commit_before>package hammer

import (
	"context"
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/antihax/goesi"
	"golang.org/x/oauth2"
)

func init() {
	registerConsumer("characterContactSync", characterContactSyncConsumer)
}

func characterContactSyncConsumer(s *Hammer, parameter interface{}) {
	// dereference the parameters
	parameters := parameter.([]interface{})
	characterID := int32(parameters[0].(int))
	source := int32(parameters[1].(int))
	destinations := parameters[2].(string)

	char, _, err := s.esi.ESI.CharacterApi.GetCharactersCharacterId(nil, int32(source), nil)
	if err != nil {
		log.Println(err)
		return
	}

	corp, _, err := s.esi.ESI.CorporationApi.GetCorporationsCorporationId(nil, char.CorporationId, nil)
	if err != nil {
		log.Println(err)
		return
	}

	// Find the Entity ID to search for wars.
	var searchID int32
	if corp.AllianceId > 0 {
		searchID = corp.AllianceId
	} else {
		searchID = char.CorporationId
	}

	// Map of tokens
	type characterToken struct {
		token *oauth2.TokenSource
		cid   int32
	}
	tokens := make(map[int64]characterToken)

	// Get the tokens for our destinations
	for _, cidS := range strings.Split(destinations, ",") {
		cid, _ := strconv.ParseInt(cidS, 10, 64)
		a, err := s.tokenStore.GetTokenSource(int32(characterID), int32(cid))
		if err != nil {
			log.Println(err, characterID, cidS)
			return
		}
		// Save the token.
		tokens[cid] = characterToken{token: &a, cid: int32(cid)}
	}

	// Active Wars
	activeWars, err := s.GetActiveWarsByID((int64)(searchID))
	if err != nil {
		log.Println(err)
		return
	}

	// Pending Wars
	pendingWars, err := s.GetPendingWarsByID((int64)(searchID))
	if err != nil {
		log.Println(err)
		return
	}

	// Faction Wars
	var factionWars []FactionWarEntities
	if corp.FactionId > 0 {
		factionWars, err = s.GetFactionWarEntitiesForID(corp.FactionId)
		if err != nil {
			log.Println(err)
			return
		}
	}

	// Loop through all the destinations
	for _, token := range tokens {
		// authentication token context for destination char
		auth := context.WithValue(context.Background(), goesi.ContextOAuth2, *token.token)

		contacts, _, err := s.esi.ESI.ContactsApi.GetCharactersCharacterIdContacts(auth, (int32)(token.cid), nil)
		if err != nil {
			log.Println(err)
			return
		}

		var erase []int32
		var active []int32
		var pending []int32
		var pendingMove []int32
		var activeMove []int32
		var untouchableContacts int

		// Figure out how many contacts they have outside of ours
		for _, contact := range contacts {
			if contact.Standing > -0.4 {
				untouchableContacts++
			}
		}

		// Faction wars can get over the 1024 contact limit so we need to trim
		// real wars will take precedence.
		trim := len(activeWars) + len(pendingWars)

		activeCheck := make(map[int32]bool)
		pendingCheck := make(map[int32]bool)

		// Build a map of active wars
		for _, war := range activeWars {
			activeCheck[(int32)(war.ID)] = true
		}

		// Add faction wars to the active list
		maxFactionWarLength := min(980-trim-untouchableContacts, len(factionWars))
		for _, war := range factionWars[:maxFactionWarLength] {
			activeCheck[(int32)(war.ID)] = true
		}

		// Build a map of pending wars
		for _, war := range pendingWars {
			pendingCheck[(int32)(war.ID)] = true
		}

		// Loop through all current contacts and figure out needed moves
		for _, contact := range contacts {
			// skip anything > -0.4
			if contact.Standing > -0.4 {
				continue
			}

			pend := pendingCheck[contact.ContactId]
			act := activeCheck[contact.ContactId]

			// Is this existing contact in the active list
			if !act {
				// Is this existing contact in the pending list
				if !pend { // Not in either list. delete it.
					erase = append(erase, (int32)(contact.ContactId))
				} else if pend && contact.Standing > -5.0 { // in pending list but wrong standing
					// Take it out of the active list and put into pending move.
					delete(pendingCheck, contact.ContactId)
					pendingMove = append(pendingMove, (int32)(contact.ContactId))
				} else if pend && contact.Standing == -5.0 { // Contact correct, do nothing.
					delete(pendingCheck, contact.ContactId)
				}
			} else if act && contact.Standing != -10.0 { // in active list, but wrong standing
				delete(activeCheck, contact.ContactId)
				activeMove = append(activeMove, (int32)(contact.ContactId))
			} else if act && contact.Standing == -10.0 { // Contact correct, do nothing.
				delete(activeCheck, contact.ContactId)
			}
		}

		// Build a list of active wars to add
		for con := range activeCheck {
			active = append(active, con)
		}

		// Build a list of pending wars to add
		for con := range pendingCheck {
			pending = append(pending, con)
		}

		// Erase contacts which have no wars.
		if len(erase) > 0 {
			for start := 0; start < len(erase); start = start + 20 {
				end := min(start+20, len(erase))
				if _, err := s.esi.ESI.ContactsApi.DeleteCharactersCharacterIdContacts(auth, token.cid, erase[start:end], nil); err != nil {
					//if err := s.deleteContactsCREST(auth, token.cid, erase[start:end]); err != nil {
					log.Println(err)
					return
				}
			}
		}

		// Add contacts for active wars
		if len(active) > 0 {
			for start := 0; start < len(active); start = start + 100 {
				end := min(start+100, len(active))
				if _, _, err := s.esi.ESI.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), active[start:end], -10, nil); err != nil {
					log.Println(err)
					return
				}
			}
		}

		// Add contacts for pending wars
		if len(pending) > 0 {
			for start := 0; start < len(pending); start = start + 100 {
				end := min(start+100, len(pending))
				if _, _, err := s.esi.ESI.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), pending[start:end], -5, nil); err != nil {
					log.Println(err)
					return
				}
			}
		}

		// Move contacts to active wars
		if len(activeMove) > 0 {
			for start := 0; start < len(activeMove); start = start + 100 {
				end := min(start+100, len(activeMove))
				if _, err := s.esi.ESI.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), activeMove[start:end], -10, nil); err != nil {
					log.Println(err)
					return
				}
			}
		}

		// Move contacts to pending wars
		if len(pendingMove) > 0 {
			for start := 0; start < len(pendingMove); start = start + 100 {
				end := min(start+100, len(pendingMove))
				if _, err := s.esi.ESI.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), pendingMove[start:end], -5, nil); err != nil {
					log.Println(err)
					return
				}
			}
		}
	}

}

func (s *Hammer) deleteContactsCREST(auth context.Context, characterID int32, contacts []int32) error {
	names, _, err := s.esi.ESI.UniverseApi.PostUniverseNames(context.Background(), contacts, nil)
	if err != nil {
		return err
	}

	tokenSource, ok := auth.Value(goesi.ContextOAuth2).(oauth2.TokenSource)
	if ok {
		for _, erase := range names {
			ref := fmt.Sprintf("https://crest-tq.eveonline.com/%ss/%d/", erase.Category, erase.Id)
			err := s.esi.EVEAPI.ContactDeleteV1(tokenSource, int64(characterID), int64(erase.Id), ref)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}

// Entity denormalizes corporations, alliance, and characters
type Entity struct {
	ID   int64
	Type string
}

// GetActiveWarsByID gets active wars for an entityID
// [BENCHMARK] 0.000 sec / 0.000 sec
func (s *Hammer) GetActiveWarsByID(id int64) ([]Entity, error) {
	w := []Entity{}
	if err := s.db.Select(&w, `
			SELECT K.id, type FROM
			(SELECT defenderID AS id FROM evedata.wars WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND aggressorID = ?
			UNION
			SELECT aggressorID AS id FROM evedata.wars WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND defenderID = ?
			UNION
			SELECT aggressorID  AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND allyID = ?
			UNION
			SELECT allyID AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND aggressorID = ?) AS K
			INNER JOIN evedata.entities C ON C.id = K.id
		`, id, id, id, id); err != nil {
		return nil, err
	}
	return w, nil
}

// GetPendingWarsByID gets pending wars for an entityID
// [BENCHMARK] 0.000 sec / 0.000 sec
func (s *Hammer) GetPendingWarsByID(id int64) ([]Entity, error) {
	w := []Entity{}
	if err := s.db.Select(&w, `
			SELECT K.id, type FROM
			(SELECT defenderID AS id FROM evedata.wars WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND aggressorID = ?
			UNION
			SELECT aggressorID AS id FROM evedata.wars WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND defenderID = ?
			UNION
			SELECT aggressorID  AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND allyID = ?
			UNION
			SELECT allyID AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND aggressorID = ?) AS K
			INNER JOIN evedata.entities C ON C.id = K.id
		`, id, id, id, id); err != nil {
		return nil, err
	}
	return w, nil
}

// FactionsByName resolves faction name to ID
var FactionsByName = map[string]int32{"Caldari": 500001, "Minmatar": 500002, "Amarr": 500003, "Gallente": 500004}

// FactionsByID resolves faction ID to Name
var FactionsByID = map[int32]string{500001: "Caldari", 500002: "Minmatar", 500003: "Amarr", 500004: "Gallente"}

// FactionsAtWar resolves two enemy parties for each factionID
var FactionsAtWar = map[int32][]int32{
	500001: {500002, 500004}, // Caldari  : Minmatar, Gallente
	500003: {500002, 500004}, // Amarr    : Minmatar, Gallente
	500002: {500001, 500003}, // Minmatar : Caldari, Amarr
	500004: {500001, 500003}, // Gallente : Caldari, Amarr
}

type FactionWarEntities struct {
	ID   int64  `db:"id" json:"id"`
	Name string `db:"name" json:"name"`
	Type string `db:"type" json:"type"`
}

// [BENCHMARK] 0.031 sec / 0.000 sec
func (s *Hammer) GetFactionWarEntitiesForID(factionID int32) ([]FactionWarEntities, error) {
	if FactionsByID[factionID] == "" {
		return nil, errors.New("Unknown FactionID")
	}

	// Due to CCP limitation, make sure count is under 1024, cut stuff off until it is.

	wars := FactionsAtWar[factionID]
	w := []FactionWarEntities{}
	if err := s.db.Select(&w, `
		SELECT
			DISTINCT IF(C.allianceID > 0, C.allianceID, corporationID) AS id,
			IF(C.allianceID > 0, A.name, C.name) AS name,
			IF(C.allianceID > 0, "alliance", "corporation") AS type
			FROM evedata.corporations C
			LEFT OUTER JOIN evedata.alliances A ON C.allianceID = A.allianceID
			INNER JOIN evedata.entityKillStats K ON K.id = IF(C.allianceID > 0, C.allianceID, C.corporationID)
			WHERE factionID IN (?, ?) AND C.memberCount > 0
			ORDER BY K.kills + K.losses + C.memberCount DESC, name ASC;
			`, wars[0], wars[1]); err != nil {
		return nil, err
	}

	return w, nil
}
<commit_msg>bit more on this error<commit_after>package hammer

import (
	"context"
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/antihax/goesi"
	"golang.org/x/oauth2"
)

func init() {
	registerConsumer("characterContactSync", characterContactSyncConsumer)
}

func characterContactSyncConsumer(s *Hammer, parameter interface{}) {
	// dereference the parameters
	parameters := parameter.([]interface{})
	characterID := int32(parameters[0].(int))
	source := int32(parameters[1].(int))
	destinations := parameters[2].(string)

	char, _, err := s.esi.ESI.CharacterApi.GetCharactersCharacterId(nil, int32(source), nil)
	if err != nil {
		log.Println(err)
		return
	}

	corp, _, err := s.esi.ESI.CorporationApi.GetCorporationsCorporationId(nil, char.CorporationId, nil)
	if err != nil {
		log.Println(err)
		return
	}

	// Find the Entity ID to search for wars.
	var searchID int32
	if corp.AllianceId > 0 {
		searchID = corp.AllianceId
	} else {
		searchID = char.CorporationId
	}

	// Map of tokens
	type characterToken struct {
		token *oauth2.TokenSource
		cid   int32
	}
	tokens := make(map[int64]characterToken)

	// Get the tokens for our destinations
	for _, cidS := range strings.Split(destinations, ",") {
		cid, _ := strconv.ParseInt(cidS, 10, 64)
		a, err := s.tokenStore.GetTokenSource(int32(characterID), int32(cid))
		if err != nil {
			log.Println(err, characterID, cidS)
			return
		}
		// Save the token.
		tokens[cid] = characterToken{token: &a, cid: int32(cid)}
	}

	// Active Wars
	activeWars, err := s.GetActiveWarsByID((int64)(searchID))
	if err != nil {
		log.Println(err)
		return
	}

	// Pending Wars
	pendingWars, err := s.GetPendingWarsByID((int64)(searchID))
	if err != nil {
		log.Println(err)
		return
	}

	// Faction Wars
	var factionWars []FactionWarEntities
	if corp.FactionId > 0 {
		factionWars, err = s.GetFactionWarEntitiesForID(corp.FactionId)
		if err != nil {
			log.Println(err)
			return
		}
	}

	// Loop through all the destinations
	for _, token := range tokens {
		// authentication token context for destination char
		auth := context.WithValue(context.Background(), goesi.ContextOAuth2, *token.token)

		contacts, _, err := s.esi.ESI.ContactsApi.GetCharactersCharacterIdContacts(auth, (int32)(token.cid), nil)
		if err != nil {
			log.Println(err)
			return
		}

		var erase []int32
		var active []int32
		var pending []int32
		var pendingMove []int32
		var activeMove []int32
		var untouchableContacts int

		// Figure out how many contacts they have outside of ours
		for _, contact := range contacts {
			if contact.Standing > -0.4 {
				untouchableContacts++
			}
		}

		// Faction wars can get over the 1024 contact limit so we need to trim
		// real wars will take precedence.
		trim := len(activeWars) + len(pendingWars)

		activeCheck := make(map[int32]bool)
		pendingCheck := make(map[int32]bool)

		// Build a map of active wars
		for _, war := range activeWars {
			activeCheck[(int32)(war.ID)] = true
		}

		// Add faction wars to the active list
		maxFactionWarLength := min(980-trim-untouchableContacts, len(factionWars))
		for _, war := range factionWars[:maxFactionWarLength] {
			activeCheck[(int32)(war.ID)] = true
		}

		// Build a map of pending wars
		for _, war := range pendingWars {
			pendingCheck[(int32)(war.ID)] = true
		}

		// Loop through all current contacts and figure out needed moves
		for _, contact := range contacts {
			// skip anything > -0.4
			if contact.Standing > -0.4 {
				continue
			}

			pend := pendingCheck[contact.ContactId]
			act := activeCheck[contact.ContactId]

			// Is this existing contact in the active list
			if !act {
				// Is this existing contact in the pending list
				if !pend { // Not in either list. delete it.
					erase = append(erase, (int32)(contact.ContactId))
				} else if pend && contact.Standing > -5.0 { // in pending list but wrong standing
					// Take it out of the active list and put into pending move.
					delete(pendingCheck, contact.ContactId)
					pendingMove = append(pendingMove, (int32)(contact.ContactId))
				} else if pend && contact.Standing == -5.0 { // Contact correct, do nothing.
					delete(pendingCheck, contact.ContactId)
				}
			} else if act && contact.Standing != -10.0 { // in active list, but wrong standing
				delete(activeCheck, contact.ContactId)
				activeMove = append(activeMove, (int32)(contact.ContactId))
			} else if act && contact.Standing == -10.0 { // Contact correct, do nothing.
				delete(activeCheck, contact.ContactId)
			}
		}

		// Build a list of active wars to add
		for con := range activeCheck {
			active = append(active, con)
		}

		// Build a list of pending wars to add
		for con := range pendingCheck {
			pending = append(pending, con)
		}

		// Erase contacts which have no wars.
		if len(erase) > 0 {
			for start := 0; start < len(erase); start = start + 20 {
				end := min(start+20, len(erase))
				if _, err := s.esi.ESI.ContactsApi.DeleteCharactersCharacterIdContacts(auth, token.cid, erase[start:end], nil); err != nil {
					log.Println(err)
					return
				}
			}
		}

		// Add contacts for active wars
		if len(active) > 0 {
			for start := 0; start < len(active); start = start + 100 {
				end := min(start+100, len(active))
				if _, _, err := s.esi.ESI.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), active[start:end], -10, nil); err != nil {
					log.Println(err, active[start:end])
					return
				}
			}
		}

		// Add contacts for pending wars
		if len(pending) > 0 {
			for start := 0; start < len(pending); start = start + 100 {
				end := min(start+100, len(pending))
				if _, _, err := s.esi.ESI.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), pending[start:end], -5, nil); err != nil {
					log.Println(err)
					return
				}
			}
		}

		// Move contacts to active wars
		if len(activeMove) > 0 {
			for start := 0; start < len(activeMove); start = start + 100 {
				end := min(start+100, len(activeMove))
				if _, err := s.esi.ESI.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), activeMove[start:end], -10, nil); err != nil {
					log.Println(err)
					return
				}
			}
		}

		// Move contacts to pending wars
		if len(pendingMove) > 0 {
			for start := 0; start < len(pendingMove); start = start + 100 {
				end := min(start+100, len(pendingMove))
				if _, err := s.esi.ESI.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), pendingMove[start:end], -5, nil); err != nil {
					log.Println(err)
					return
				}
			}
		}
	}

}

func (s *Hammer) deleteContactsCREST(auth context.Context, characterID int32, contacts []int32) error {
	names, _, err := s.esi.ESI.UniverseApi.PostUniverseNames(context.Background(), contacts, nil)
	if err != nil {
		return err
	}

	tokenSource, ok := auth.Value(goesi.ContextOAuth2).(oauth2.TokenSource)
	if ok {
		for _, erase := range names {
			ref := fmt.Sprintf("https://crest-tq.eveonline.com/%ss/%d/", erase.Category, erase.Id)
			err := s.esi.EVEAPI.ContactDeleteV1(tokenSource, int64(characterID), int64(erase.Id), ref)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}

// Entity denormalizes corporations, alliance, and characters
type Entity struct {
	ID   int64
	Type string
}

// GetActiveWarsByID gets active wars for an entityID
// [BENCHMARK] 0.000 sec / 0.000 sec
func (s *Hammer) GetActiveWarsByID(id int64) ([]Entity, error) {
	w := []Entity{}
	if err := s.db.Select(&w, `
			SELECT K.id, type FROM
			(SELECT defenderID AS id FROM evedata.wars WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND aggressorID = ?
			UNION
			SELECT aggressorID AS id FROM evedata.wars WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND defenderID = ?
			UNION
			SELECT aggressorID  AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND allyID = ?
			UNION
			SELECT allyID AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE (timeFinished = "0001-01-01 00:00:00" OR timeFinished IS NULL OR timeFinished >= UTC_TIMESTAMP()) AND timeStarted <= UTC_TIMESTAMP() AND aggressorID = ?) AS K
			INNER JOIN evedata.entities C ON C.id = K.id
		`, id, id, id, id); err != nil {
		return nil, err
	}
	return w, nil
}

// GetPendingWarsByID gets pending wars for an entityID
// [BENCHMARK] 0.000 sec / 0.000 sec
func (s *Hammer) GetPendingWarsByID(id int64) ([]Entity, error) {
	w := []Entity{}
	if err := s.db.Select(&w, `
			SELECT K.id, type FROM
			(SELECT defenderID AS id FROM evedata.wars WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND aggressorID = ?
			UNION
			SELECT aggressorID AS id FROM evedata.wars WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND defenderID = ?
			UNION
			SELECT aggressorID  AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND allyID = ?
			UNION
			SELECT allyID AS id FROM evedata.wars W INNER JOIN evedata.warAllies A on A.id = W.id WHERE timeStarted > timeDeclared AND timeStarted > UTC_TIMESTAMP() AND aggressorID = ?) AS K
			INNER JOIN evedata.entities C ON C.id = K.id
		`, id, id, id, id); err != nil {
		return nil, err
	}
	return w, nil
}

// FactionsByName resolves faction name to ID
var FactionsByName = map[string]int32{"Caldari": 500001, "Minmatar": 500002, "Amarr": 500003, "Gallente": 500004}

// FactionsByID resolves faction ID to Name
var FactionsByID = map[int32]string{500001: "Caldari", 500002: "Minmatar", 500003: "Amarr", 500004: "Gallente"}

// FactionsAtWar resolves two enemy parties for each factionID
var FactionsAtWar = map[int32][]int32{
	500001: {500002, 500004}, // Caldari  : Minmatar, Gallente
	500003: {500002, 500004}, // Amarr    : Minmatar, Gallente
	500002: {500001, 500003}, // Minmatar : Caldari, Amarr
	500004: {500001, 500003}, // Gallente : Caldari, Amarr
}

type FactionWarEntities struct {
	ID   int64  `db:"id" json:"id"`
	Name string `db:"name" json:"name"`
	Type string `db:"type" json:"type"`
}

// [BENCHMARK] 0.031 sec / 0.000 sec
func (s *Hammer) GetFactionWarEntitiesForID(factionID int32) ([]FactionWarEntities, error) {
	if FactionsByID[factionID] == "" {
		return nil, errors.New("Unknown FactionID")
	}

	// Due to CCP limitation, make sure count is under 1024, cut stuff off until it is.

	wars := FactionsAtWar[factionID]
	w := []FactionWarEntities{}
	if err := s.db.Select(&w, `
		SELECT
			DISTINCT IF(C.allianceID > 0, C.allianceID, corporationID) AS id,
			IF(C.allianceID > 0, A.name, C.name) AS name,
			IF(C.allianceID > 0, "alliance", "corporation") AS type
			FROM evedata.corporations C
			LEFT OUTER JOIN evedata.alliances A ON C.allianceID = A.allianceID
			INNER JOIN evedata.entityKillStats K ON K.id = IF(C.allianceID > 0, C.allianceID, C.corporationID)
			WHERE factionID IN (?, ?) AND C.memberCount > 0
			ORDER BY K.kills + K.losses + C.memberCount DESC, name ASC;
			`, wars[0], wars[1]); err != nil {
		return nil, err
	}

	return w, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"sync"
	"time"
)

var verbose = flag.Bool("v", false, "verbosity")
var noop = flag.Bool("n", false, "don't actually compact")

func init() {
	log.SetFlags(log.Lmicroseconds)
}

func maybeFatal(err error, fmt string, args ...interface{}) {
	if err != nil {
		log.Fatalf(fmt, args...)
	}
}

func listDatabases(u url.URL) []string {
	u.Path = "/_all_dbs"
	res, err := http.Get(u.String())
	maybeFatal(err, "Error listing databases: %v", err)
	defer res.Body.Close()

	rv := []string{}
	d := json.NewDecoder(res.Body)
	err = d.Decode(&rv)
	maybeFatal(err, "Error decoding database list: %v", err)
	return rv
}

func vlog(s string, a ...interface{}) {
	if *verbose {
		log.Printf(s, a...)
	}
}

func compact(wg *sync.WaitGroup, u url.URL, ch <-chan string) {
	defer wg.Done()

	for db := range ch {
		start := time.Now()
		vlog("Compacting %v", db)
		if !*noop {
			u.Path = fmt.Sprintf("/%v/_compact", db)
			req, err := http.NewRequest("POST", u.String(), nil)
			maybeFatal(err, "Error creating request: %v", err)
			res, err := http.DefaultClient.Do(req)
			if err != nil {
				vlog("Error compacting %v: %v", db, err)
			}
			res.Body.Close()
			if res.StatusCode != 200 {
				vlog("Error compacting %v: %v", db,
					res.Status)
			}
		}
		vlog("Finished compacting %v in %v", db, time.Since(start))
	}
}

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		log.Fatalf("Seriesly URL required")
	}

	u, err := url.Parse(flag.Arg(0))
	maybeFatal(err, "Parsing %v: %v", flag.Arg(0), err)

	wg := &sync.WaitGroup{}
	ch := make(chan string)

	for i := 0; i < 2; i++ {
		wg.Add(1)
		go compact(wg, *u, ch)
	}

	for _, db := range listDatabases(*u) {
		ch <- db
	}
	close(ch)

	wg.Wait()
}
<commit_msg>Configurable number of concurrent compaction workers.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"sync"
	"time"
)

var verbose = flag.Bool("v", false, "verbosity")
var noop = flag.Bool("n", false, "don't actually compact")
var concurrency = flag.Int("j", 2,
	"number of concurrent compactions")

func init() {
	log.SetFlags(log.Lmicroseconds)
}

func maybeFatal(err error, fmt string, args ...interface{}) {
	if err != nil {
		log.Fatalf(fmt, args...)
	}
}

func listDatabases(u url.URL) []string {
	u.Path = "/_all_dbs"
	res, err := http.Get(u.String())
	maybeFatal(err, "Error listing databases: %v", err)
	defer res.Body.Close()

	rv := []string{}
	d := json.NewDecoder(res.Body)
	err = d.Decode(&rv)
	maybeFatal(err, "Error decoding database list: %v", err)
	return rv
}

func vlog(s string, a ...interface{}) {
	if *verbose {
		log.Printf(s, a...)
	}
}

func compact(wg *sync.WaitGroup, u url.URL, ch <-chan string) {
	defer wg.Done()

	for db := range ch {
		start := time.Now()
		vlog("Compacting %v", db)
		if !*noop {
			u.Path = fmt.Sprintf("/%v/_compact", db)
			req, err := http.NewRequest("POST", u.String(), nil)
			maybeFatal(err, "Error creating request: %v", err)
			res, err := http.DefaultClient.Do(req)
			if err != nil {
				vlog("Error compacting %v: %v", db, err)
			}
			res.Body.Close()
			if res.StatusCode != 200 {
				vlog("Error compacting %v: %v", db,
					res.Status)
			}
		}
		vlog("Finished compacting %v in %v", db, time.Since(start))
	}
}

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		log.Fatalf("Seriesly URL required")
	}

	u, err := url.Parse(flag.Arg(0))
	maybeFatal(err, "Parsing %v: %v", flag.Arg(0), err)

	wg := &sync.WaitGroup{}
	ch := make(chan string)

	for i := 0; i < *concurrency; i++ {
		wg.Add(1)
		go compact(wg, *u, ch)
	}

	for _, db := range listDatabases(*u) {
		ch <- db
	}
	close(ch)

	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package templates contains functions for template processing of Resource objects.
package templates

import (
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/resources"
	"github.com/gohugoio/hugo/resources/internal"
	"github.com/gohugoio/hugo/resources/resource"
	"github.com/gohugoio/hugo/tpl"
	"github.com/pkg/errors"
)

// Client contains methods to perform template processing of Resource objects.
type Client struct {
	rs *resources.Spec
	t  tpl.TemplatesProvider
}

// New creates a new Client with the given specification.
func New(rs *resources.Spec, t tpl.TemplatesProvider) *Client {
	if rs == nil {
		panic("must provice a resource Spec")
	}
	if t == nil {
		panic("must provide a template provider")
	}
	return &Client{rs: rs, t: t}
}

type executeAsTemplateTransform struct {
	rs         *resources.Spec
	t          tpl.TemplatesProvider
	targetPath string
	data       interface{}
}

func (t *executeAsTemplateTransform) Key() internal.ResourceTransformationKey {
	return internal.NewResourceTransformationKey("execute-as-template", t.targetPath)
}

func (t *executeAsTemplateTransform) Transform(ctx *resources.ResourceTransformationCtx) error {
	tplStr := helpers.ReaderToString(ctx.From)
	templ, err := t.t.TextTmpl().Parse(ctx.InPath, tplStr)
	if err != nil {
		return errors.Wrapf(err, "failed to parse Resource %q as Template:", ctx.InPath)
	}

	ctx.OutPath = t.targetPath

	return t.t.Tmpl().Execute(templ, ctx.To, t.data)
}

func (c *Client) ExecuteAsTemplate(res resources.ResourceTransformer, targetPath string, data interface{}) (resource.Resource, error) {
	return res.Transform(&executeAsTemplateTransform{
		rs:         c.rs,
		targetPath: helpers.ToSlashTrimLeading(targetPath),
		t:          c.t,
		data:       data,
	})
}
<commit_msg>resources: Add data context to the key in ExecuteAsTemplate<commit_after>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package templates contains functions for template processing of Resource objects.
package templates

import (
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/resources"
	"github.com/gohugoio/hugo/resources/internal"
	"github.com/gohugoio/hugo/resources/resource"
	"github.com/gohugoio/hugo/tpl"
	"github.com/pkg/errors"
)

// Client contains methods to perform template processing of Resource objects.
type Client struct {
	rs *resources.Spec
	t  tpl.TemplatesProvider
}

// New creates a new Client with the given specification.
func New(rs *resources.Spec, t tpl.TemplatesProvider) *Client {
	if rs == nil {
		panic("must provice a resource Spec")
	}
	if t == nil {
		panic("must provide a template provider")
	}
	return &Client{rs: rs, t: t}
}

type executeAsTemplateTransform struct {
	rs         *resources.Spec
	t          tpl.TemplatesProvider
	targetPath string
	data       interface{}
}

func (t *executeAsTemplateTransform) Key() internal.ResourceTransformationKey {
	return internal.NewResourceTransformationKey("execute-as-template", t.targetPath, t.data)
}

func (t *executeAsTemplateTransform) Transform(ctx *resources.ResourceTransformationCtx) error {
	tplStr := helpers.ReaderToString(ctx.From)
	templ, err := t.t.TextTmpl().Parse(ctx.InPath, tplStr)
	if err != nil {
		return errors.Wrapf(err, "failed to parse Resource %q as Template:", ctx.InPath)
	}

	ctx.OutPath = t.targetPath

	return t.t.Tmpl().Execute(templ, ctx.To, t.data)
}

func (c *Client) ExecuteAsTemplate(res resources.ResourceTransformer, targetPath string, data interface{}) (resource.Resource, error) {
	return res.Transform(&executeAsTemplateTransform{
		rs:         c.rs,
		targetPath: helpers.ToSlashTrimLeading(targetPath),
		t:          c.t,
		data:       data,
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationtarget

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/migration"
	"github.com/juju/juju/state"
)

var logger loggo.Logger

func init() {
	common.RegisterStandardFacade("MigrationTarget", 1, NewAPI)
}

// API implements the API required for the model migration
// master worker when communicating with the target controller.
type API struct {
	state      *state.State
	authorizer common.Authorizer
	resources  *common.Resources
}

// NewAPI returns a new API.
func NewAPI(
	st *state.State,
	resources *common.Resources,
	authorizer common.Authorizer,
) (*API, error) {
	if err := checkAuth(authorizer, st); err != nil {
		return nil, errors.Trace(err)
	}
	return &API{
		state:      st,
		authorizer: authorizer,
		resources:  resources,
	}, nil
}

func checkAuth(authorizer common.Authorizer, st *state.State) error {
	if !authorizer.AuthClient() {
		return errors.Trace(common.ErrPerm)
	}

	// Type assertion is fine because AuthClient is true.
	apiUser := authorizer.GetAuthTag().(names.UserTag)
	if isAdmin, err := st.IsControllerAdministrator(apiUser); err != nil {
		return errors.Trace(err)
	} else if !isAdmin {
		// The entire facade is only accessible to controller administrators.
		return errors.Trace(common.ErrPerm)
	}
	return nil
}

// Import takes a serialized Juju model, deserializes it, and
// recreates it in the receiving controller.
func (api *API) Import(serialized params.SerializedModel) error {
	_, st, err := migration.ImportModel(api.state, serialized.Bytes)
	if err != nil {
		return err
	}
	defer st.Close()
	// TODO(mjs) - post import checks
	return err
}
<commit_msg>Removed unused logger<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationtarget

import (
	"github.com/juju/errors"
	"github.com/juju/names"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/migration"
	"github.com/juju/juju/state"
)

func init() {
	common.RegisterStandardFacade("MigrationTarget", 1, NewAPI)
}

// API implements the API required for the model migration
// master worker when communicating with the target controller.
type API struct {
	state      *state.State
	authorizer common.Authorizer
	resources  *common.Resources
}

// NewAPI returns a new API.
func NewAPI(
	st *state.State,
	resources *common.Resources,
	authorizer common.Authorizer,
) (*API, error) {
	if err := checkAuth(authorizer, st); err != nil {
		return nil, errors.Trace(err)
	}
	return &API{
		state:      st,
		authorizer: authorizer,
		resources:  resources,
	}, nil
}

func checkAuth(authorizer common.Authorizer, st *state.State) error {
	if !authorizer.AuthClient() {
		return errors.Trace(common.ErrPerm)
	}

	// Type assertion is fine because AuthClient is true.
	apiUser := authorizer.GetAuthTag().(names.UserTag)
	if isAdmin, err := st.IsControllerAdministrator(apiUser); err != nil {
		return errors.Trace(err)
	} else if !isAdmin {
		// The entire facade is only accessible to controller administrators.
		return errors.Trace(common.ErrPerm)
	}
	return nil
}

// Import takes a serialized Juju model, deserializes it, and
// recreates it in the receiving controller.
func (api *API) Import(serialized params.SerializedModel) error {
	_, st, err := migration.ImportModel(api.state, serialized.Bytes)
	if err != nil {
		return err
	}
	defer st.Close()
	// TODO(mjs) - post import checks
	return err
}
<|endoftext|>


<commit_before>package serviceenv

// Configuration is the generic configuration structure used to access configuration fields.
type Configuration struct {
	*GlobalConfiguration
	*PachdSpecificConfiguration
	*WorkerSpecificConfiguration
}

// GlobalConfiguration contains the global configuration.
type GlobalConfiguration struct {
	FeatureFlags
	EtcdHost                       string `env:"ETCD_SERVICE_HOST,required"`
	EtcdPort                       string `env:"ETCD_SERVICE_PORT,required"`
	PPSWorkerPort                  uint16 `env:"PPS_WORKER_GRPC_PORT,default=1080"`
	Port                           uint16 `env:"PORT,default=1650"`
	PrometheusPort                 uint16 `env:"PROMETHEUS_PORT,default=1656"`
	PeerPort                       uint16 `env:"PEER_PORT,default=1653"`
	S3GatewayPort                  uint16 `env:"S3GATEWAY_PORT,default=1600"`
	PPSEtcdPrefix                  string `env:"PPS_ETCD_PREFIX,default=pachyderm_pps"`
	Namespace                      string `env:"PACH_NAMESPACE,default=default"`
	StorageRoot                    string `env:"PACH_ROOT,default=/pach"`
	GCPercent                      int    `env:"GC_PERCENT,default=50"`
	LokiHost                       string `env:"LOKI_SERVICE_HOST"`
	LokiPort                       string `env:"LOKI_SERVICE_PORT"`
	OidcPort                       uint16 `env:"OIDC_PORT,default=1657"`
	PGBouncerHost                  string `env:"PG_BOUNCER_HOST"`
	PGBouncerPort                  int    `env:"PG_BOUNCER_PORT"`
	PostgresSSL                    string `env:"POSTGRES_SSL,default=disable"`
	PostgresDBName                 string `env:"POSTGRES_DATABASE,required"`
	PostgresUser                   string `env:"POSTGRES_USER,required"`
	PostgresPassword               string `env:"POSTGRES_PASSWORD"`
	PostgresMaxOpenConns           int    `env:"POSTGRES_MAX_OPEN_CONNS,default=10"`
	PostgresMaxIdleConns           int    `env:"POSTGRES_MAX_IDLE_CONNS,default=2"`
	PGBouncerMaxOpenConns          int    `env:"PG_BOUNCER_MAX_OPEN_CONNS,default=100"`
	PGBouncerMaxIdleConns          int    `env:"PG_BOUNCER_MAX_IDLE_CONNS,default=20"`
	PostgresConnMaxLifetimeSeconds int    `env:"POSTGRES_CONN_MAX_LIFETIME_SECONDS,default=0"`
	PostgresConnMaxIdleSeconds     int    `env:"POSTGRES_CONN_MAX_IDLE_SECONDS,default=0"`
	PachdServiceHost               string `env:"PACHD_SERVICE_HOST"`
	PachdServicePort               string `env:"PACHD_SERVICE_PORT"`

	EtcdPrefix           string `env:"ETCD_PREFIX,default="`
	DeploymentID         string `env:"CLUSTER_DEPLOYMENT_ID,default="`
	LogLevel             string `env:"LOG_LEVEL,default=info"`
	EnterpriseEtcdPrefix string `env:"PACHYDERM_ENTERPRISE_ETCD_PREFIX,default=pachyderm_enterprise"`
	Metrics              bool   `env:"METRICS,default=true"`
	MetricsEndpoint      string `env:"METRICS_ENDPOINT,default="`

	// SessionDurationMinutes it how long auth tokens are valid for, defaults to 30 days (30 * 24 * 60)
	SessionDurationMinutes int `env:"SESSION_DURATION_MINUTES,default=43200"`

	IdentityServerDatabase string `env:"IDENTITY_SERVER_DATABASE,default=dex"`

	// PPSSpecCommitID and PPSPipelineName are only set for workers and sidecar
	// pachd instances. Because both pachd and worker need to know the spec commit
	// (the worker so that it can avoid jobs for other versions of the same pipelines
	// and the sidecar so that it can serve the S3 gateway) it's stored in the
	// GlobalConfiguration, but it isn't set in a cluster's main pachd containers.
	PPSSpecCommitID string `env:"PPS_SPEC_COMMIT"`
	// The name of the pipeline that this worker belongs to
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`

	// If set to the name of a GCP project, enable GCP-specific continuous profiling and send
	// profiles to that project: https://cloud.google.com/profiler/docs.  Requires that pachd
	// has google application credentials (through environment variables or workload identity),
	// and that the service account associated with the credentials has 'cloudprofiler.agent' on
	// the target project.  If set on a pachd pod, propagates to workers and sidecars (which
	// also need permission).
	GoogleCloudProfilerProject string `env:"GOOGLE_CLOUD_PROFILER_PROJECT"`

	PostgresHost string `env:"POSTGRES_HOST"`
	PostgresPort int    `env:"POSTGRES_PORT"`
}

// PachdFullConfiguration contains the full pachd configuration.
type PachdFullConfiguration struct {
	GlobalConfiguration
	PachdSpecificConfiguration
}

// PachdSpecificConfiguration contains the pachd specific configuration.
type PachdSpecificConfiguration struct {
	StorageConfiguration
	StorageBackend             string `env:"STORAGE_BACKEND,required"`
	StorageHostPath            string `env:"STORAGE_HOST_PATH,default="`
	PFSEtcdPrefix              string `env:"PFS_ETCD_PREFIX,default=pachyderm_pfs"`
	KubeAddress                string `env:"KUBERNETES_PORT_443_TCP_ADDR,required"`
	Init                       bool   `env:"INIT,default=false"`
	WorkerImage                string `env:"WORKER_IMAGE,default="`
	WorkerSidecarImage         string `env:"WORKER_SIDECAR_IMAGE,default="`
	WorkerImagePullPolicy      string `env:"WORKER_IMAGE_PULL_POLICY,default="`
	ImagePullSecret            string `env:"IMAGE_PULL_SECRET,default="`
	MemoryRequest              string `env:"PACHD_MEMORY_REQUEST,default=1T"`
	WorkerUsesRoot             bool   `env:"WORKER_USES_ROOT,default=false"`
	RequireCriticalServersOnly bool   `env:"REQUIRE_CRITICAL_SERVERS_ONLY,default=false"`
	// TODO: Merge this with the worker specific pod name (PPS_POD_NAME) into a global configuration pod name.
	PachdPodName string `env:"PACHD_POD_NAME,required"`
}

// StorageConfiguration contains the storage configuration.
type StorageConfiguration struct {
	StorageMemoryThreshold         int64  `env:"STORAGE_MEMORY_THRESHOLD"`
	StorageShardThreshold          int64  `env:"STORAGE_SHARD_THRESHOLD"`
	StorageLevelFactor             int64  `env:"STORAGE_LEVEL_FACTOR"`
	StorageUploadConcurrencyLimit  int    `env:"STORAGE_UPLOAD_CONCURRENCY_LIMIT,default=100"`
	StoragePutFileConcurrencyLimit int    `env:"STORAGE_PUT_FILE_CONCURRENCY_LIMIT,default=100"`
	StorageGCPolling               string `env:"STORAGE_GC_POLLING"`
	StorageGCTimeout               string `env:"STORAGE_GC_TIMEOUT"`
	StorageCompactionMaxFanIn      int    `env:"STORAGE_COMPACTION_MAX_FANIN,default=10"`
	StorageFileSetsMaxOpen         int    `env:"STORAGE_FILESETS_MAX_OPEN,default=50"`
	StorageDiskCacheSize           int    `env:"STORAGE_DISK_CACHE_SIZE,default=100"`
	StorageMemoryCacheSize         int    `env:"STORAGE_MEMORY_CACHE_SIZE,default=100"`
}

// WorkerFullConfiguration contains the full worker configuration.
type WorkerFullConfiguration struct {
	GlobalConfiguration
	WorkerSpecificConfiguration
}

// WorkerSpecificConfiguration contains the worker specific configuration.
type WorkerSpecificConfiguration struct {
	// Worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`
	// The name of this pod
	PodName string `env:"PPS_POD_NAME,required"`
}

// FeatureFlags contains the configuration for feature flags.  XXX: if you're
// adding a new feature flag then you need to make sure it gets propagated to
// the workers and their sidecars, this should be done in:
// src/server/pps/server/worker_rc.go in the workerPodSpec func.
type FeatureFlags struct {
	DisableCommitProgressCounter bool `env:"DISABLE_COMMIT_PROGRESS_COUNTER,default=false"`
	LokiLogging                  bool `env:"LOKI_LOGGING,default=false"`
	IdentityServerEnabled        bool `env:"IDENTITY_SERVER_ENABLED,default=false"`
}

// NewConfiguration creates a generic configuration from a specific type of configuration.
func NewConfiguration(config interface{}) *Configuration {
	configuration := &Configuration{}
	switch v := config.(type) {
	case *GlobalConfiguration:
		configuration.GlobalConfiguration = v
		return configuration
	case *PachdFullConfiguration:
		configuration.GlobalConfiguration = &v.GlobalConfiguration
		configuration.PachdSpecificConfiguration = &v.PachdSpecificConfiguration
		return configuration
	case *WorkerFullConfiguration:
		configuration.GlobalConfiguration = &v.GlobalConfiguration
		configuration.WorkerSpecificConfiguration = &v.WorkerSpecificConfiguration
		return configuration
	default:
		return nil
	}
}
<commit_msg>Make PG Bouncer Env Vars required (#6737)<commit_after>package serviceenv

// Configuration is the generic configuration structure used to access configuration fields.
type Configuration struct {
	*GlobalConfiguration
	*PachdSpecificConfiguration
	*WorkerSpecificConfiguration
}

// GlobalConfiguration contains the global configuration.
type GlobalConfiguration struct {
	FeatureFlags
	EtcdHost                       string `env:"ETCD_SERVICE_HOST,required"`
	EtcdPort                       string `env:"ETCD_SERVICE_PORT,required"`
	PPSWorkerPort                  uint16 `env:"PPS_WORKER_GRPC_PORT,default=1080"`
	Port                           uint16 `env:"PORT,default=1650"`
	PrometheusPort                 uint16 `env:"PROMETHEUS_PORT,default=1656"`
	PeerPort                       uint16 `env:"PEER_PORT,default=1653"`
	S3GatewayPort                  uint16 `env:"S3GATEWAY_PORT,default=1600"`
	PPSEtcdPrefix                  string `env:"PPS_ETCD_PREFIX,default=pachyderm_pps"`
	Namespace                      string `env:"PACH_NAMESPACE,default=default"`
	StorageRoot                    string `env:"PACH_ROOT,default=/pach"`
	GCPercent                      int    `env:"GC_PERCENT,default=50"`
	LokiHost                       string `env:"LOKI_SERVICE_HOST"`
	LokiPort                       string `env:"LOKI_SERVICE_PORT"`
	OidcPort                       uint16 `env:"OIDC_PORT,default=1657"`
	PGBouncerHost                  string `env:"PG_BOUNCER_HOST,required"`
	PGBouncerPort                  int    `env:"PG_BOUNCER_PORT,required"`
	PostgresSSL                    string `env:"POSTGRES_SSL,default=disable"`
	PostgresHost                   string `env:"POSTGRES_HOST"`
	PostgresPort                   int    `env:"POSTGRES_PORT"`
	PostgresDBName                 string `env:"POSTGRES_DATABASE,required"`
	PostgresUser                   string `env:"POSTGRES_USER,required"`
	PostgresPassword               string `env:"POSTGRES_PASSWORD"`
	PostgresMaxOpenConns           int    `env:"POSTGRES_MAX_OPEN_CONNS,default=10"`
	PostgresMaxIdleConns           int    `env:"POSTGRES_MAX_IDLE_CONNS,default=2"`
	PGBouncerMaxOpenConns          int    `env:"PG_BOUNCER_MAX_OPEN_CONNS,default=100"`
	PGBouncerMaxIdleConns          int    `env:"PG_BOUNCER_MAX_IDLE_CONNS,default=20"`
	PostgresConnMaxLifetimeSeconds int    `env:"POSTGRES_CONN_MAX_LIFETIME_SECONDS,default=0"`
	PostgresConnMaxIdleSeconds     int    `env:"POSTGRES_CONN_MAX_IDLE_SECONDS,default=0"`
	PachdServiceHost               string `env:"PACHD_SERVICE_HOST"`
	PachdServicePort               string `env:"PACHD_SERVICE_PORT"`

	EtcdPrefix           string `env:"ETCD_PREFIX,default="`
	DeploymentID         string `env:"CLUSTER_DEPLOYMENT_ID,default="`
	LogLevel             string `env:"LOG_LEVEL,default=info"`
	EnterpriseEtcdPrefix string `env:"PACHYDERM_ENTERPRISE_ETCD_PREFIX,default=pachyderm_enterprise"`
	Metrics              bool   `env:"METRICS,default=true"`
	MetricsEndpoint      string `env:"METRICS_ENDPOINT,default="`

	// SessionDurationMinutes it how long auth tokens are valid for, defaults to 30 days (30 * 24 * 60)
	SessionDurationMinutes int `env:"SESSION_DURATION_MINUTES,default=43200"`

	IdentityServerDatabase string `env:"IDENTITY_SERVER_DATABASE,default=dex"`

	// PPSSpecCommitID and PPSPipelineName are only set for workers and sidecar
	// pachd instances. Because both pachd and worker need to know the spec commit
	// (the worker so that it can avoid jobs for other versions of the same pipelines
	// and the sidecar so that it can serve the S3 gateway) it's stored in the
	// GlobalConfiguration, but it isn't set in a cluster's main pachd containers.
	PPSSpecCommitID string `env:"PPS_SPEC_COMMIT"`
	// The name of the pipeline that this worker belongs to
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`

	// If set to the name of a GCP project, enable GCP-specific continuous profiling and send
	// profiles to that project: https://cloud.google.com/profiler/docs.  Requires that pachd
	// has google application credentials (through environment variables or workload identity),
	// and that the service account associated with the credentials has 'cloudprofiler.agent' on
	// the target project.  If set on a pachd pod, propagates to workers and sidecars (which
	// also need permission).
	GoogleCloudProfilerProject string `env:"GOOGLE_CLOUD_PROFILER_PROJECT"`
}

// PachdFullConfiguration contains the full pachd configuration.
type PachdFullConfiguration struct {
	GlobalConfiguration
	PachdSpecificConfiguration
}

// PachdSpecificConfiguration contains the pachd specific configuration.
type PachdSpecificConfiguration struct {
	StorageConfiguration
	StorageBackend             string `env:"STORAGE_BACKEND,required"`
	StorageHostPath            string `env:"STORAGE_HOST_PATH,default="`
	PFSEtcdPrefix              string `env:"PFS_ETCD_PREFIX,default=pachyderm_pfs"`
	KubeAddress                string `env:"KUBERNETES_PORT_443_TCP_ADDR,required"`
	Init                       bool   `env:"INIT,default=false"`
	WorkerImage                string `env:"WORKER_IMAGE,default="`
	WorkerSidecarImage         string `env:"WORKER_SIDECAR_IMAGE,default="`
	WorkerImagePullPolicy      string `env:"WORKER_IMAGE_PULL_POLICY,default="`
	ImagePullSecret            string `env:"IMAGE_PULL_SECRET,default="`
	MemoryRequest              string `env:"PACHD_MEMORY_REQUEST,default=1T"`
	WorkerUsesRoot             bool   `env:"WORKER_USES_ROOT,default=false"`
	RequireCriticalServersOnly bool   `env:"REQUIRE_CRITICAL_SERVERS_ONLY,default=false"`
	// TODO: Merge this with the worker specific pod name (PPS_POD_NAME) into a global configuration pod name.
	PachdPodName string `env:"PACHD_POD_NAME,required"`
}

// StorageConfiguration contains the storage configuration.
type StorageConfiguration struct {
	StorageMemoryThreshold         int64  `env:"STORAGE_MEMORY_THRESHOLD"`
	StorageShardThreshold          int64  `env:"STORAGE_SHARD_THRESHOLD"`
	StorageLevelFactor             int64  `env:"STORAGE_LEVEL_FACTOR"`
	StorageUploadConcurrencyLimit  int    `env:"STORAGE_UPLOAD_CONCURRENCY_LIMIT,default=100"`
	StoragePutFileConcurrencyLimit int    `env:"STORAGE_PUT_FILE_CONCURRENCY_LIMIT,default=100"`
	StorageGCPolling               string `env:"STORAGE_GC_POLLING"`
	StorageGCTimeout               string `env:"STORAGE_GC_TIMEOUT"`
	StorageCompactionMaxFanIn      int    `env:"STORAGE_COMPACTION_MAX_FANIN,default=10"`
	StorageFileSetsMaxOpen         int    `env:"STORAGE_FILESETS_MAX_OPEN,default=50"`
	StorageDiskCacheSize           int    `env:"STORAGE_DISK_CACHE_SIZE,default=100"`
	StorageMemoryCacheSize         int    `env:"STORAGE_MEMORY_CACHE_SIZE,default=100"`
}

// WorkerFullConfiguration contains the full worker configuration.
type WorkerFullConfiguration struct {
	GlobalConfiguration
	WorkerSpecificConfiguration
}

// WorkerSpecificConfiguration contains the worker specific configuration.
type WorkerSpecificConfiguration struct {
	// Worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`
	// The name of this pod
	PodName string `env:"PPS_POD_NAME,required"`
}

// FeatureFlags contains the configuration for feature flags.  XXX: if you're
// adding a new feature flag then you need to make sure it gets propagated to
// the workers and their sidecars, this should be done in:
// src/server/pps/server/worker_rc.go in the workerPodSpec func.
type FeatureFlags struct {
	DisableCommitProgressCounter bool `env:"DISABLE_COMMIT_PROGRESS_COUNTER,default=false"`
	LokiLogging                  bool `env:"LOKI_LOGGING,default=false"`
	IdentityServerEnabled        bool `env:"IDENTITY_SERVER_ENABLED,default=false"`
}

// NewConfiguration creates a generic configuration from a specific type of configuration.
func NewConfiguration(config interface{}) *Configuration {
	configuration := &Configuration{}
	switch v := config.(type) {
	case *GlobalConfiguration:
		configuration.GlobalConfiguration = v
		return configuration
	case *PachdFullConfiguration:
		configuration.GlobalConfiguration = &v.GlobalConfiguration
		configuration.PachdSpecificConfiguration = &v.PachdSpecificConfiguration
		return configuration
	case *WorkerFullConfiguration:
		configuration.GlobalConfiguration = &v.GlobalConfiguration
		configuration.WorkerSpecificConfiguration = &v.WorkerSpecificConfiguration
		return configuration
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>package countmin

import (
	"hash/fnv"
	"math"
	"math/rand"
)

// Max int 64 size
const MAXINT64 = 1<<63 - 1

// CountMin holds count table and hashes for calculating
// frequencies from a given input (usually a stream of data)
type CountMin struct {
	depth      int
	width      int
	size       int64
	eps        float64
	confidence float64
	table      [][]int64
	hashes     []int64
}

// New CountMin given a depth and width of the frequency
// table to use. Returns the newly created CountMin
func New(depth, width int) *CountMin {
	cm := &CountMin{
		depth:      depth,
		width:      width,
		eps:        2.0 / float64(width),
		confidence: 1.0 - 1.0/math.Pow(2, float64(depth)),
	}
	cm.initTable()
	return cm
}

// NewWithEpsCount initializes a CountMin given a confidence
// to ensure between 0 < 1.0 and an epsilon between 0 and 1
// with a smaller value for higher precision as it affects
// the width of the table
func NewWithEpsCount(confidence, eps float64) *CountMin {
	if confidence >= 1.0 {
		confidence = 0.99999
	}
	cm := &CountMin{
		eps:        eps,
		confidence: confidence,
		width:      int(math.Ceil(float64(2.0) / eps)),
		depth:      int(math.Ceil(-math.Log(1-confidence) / math.Log(2))),
	}
	cm.initTable()
	return cm
}

func (cm *CountMin) initTable() {
	cm.table = make([][]int64, cm.depth)
	for i := range cm.table {
		cm.table[i] = make([]int64, cm.width)
	}
	cm.hashes = make([]int64, cm.depth)
	for i := range cm.hashes {
		cm.hashes[i] = rand.Int63()
	}
}

// RelativeError returns epsilon of CountMin
func (cm *CountMin) RelativeError() float64 {
	return cm.eps
}

// Confidence returns confidence of CountMin
func (cm *CountMin) Confidence() float64 {
	return cm.confidence
}

// Size returns total size of CountMin
func (cm *CountMin) Size() int64 {
	return cm.size
}

// Add inserts an item in bytes and a count associated
// which will also increment size of CountMin
func (cm *CountMin) Add(item []byte, count int64) {
	if count < 0 {
		return
	}
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		cm.table[i][hashed[i]] += count
	}
	cm.size += count
}

// Count gets an associated total count for an item
// in the CountMin
func (cm *CountMin) Count(item []byte) int64 {
	var answer int64 = MAXINT64
	var val int64
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		val = cm.table[i][hashed[i]]
		if val < answer {
			answer = val
		}
	}
	return answer
}

func (cm *CountMin) hasher(item []byte) []int64 {
	total := cm.depth
	max := int64(cm.width)
	result := make([]int64, total)
	hash := fnv.New32a()
	hash.Write(item)
	sum := int64(hash.Sum32())
	for i := 0; i < total; i++ {
		result[i] = int64(math.Abs(float64((sum * int64(i)) % max)))
	}
	return result
}
<commit_msg>Better commenting<commit_after>package countmin

import (
	"hash/fnv"
	"math"
	"math/rand"
)

// Max int 64 size
const MAXINT64 = 1<<63 - 1

// CountMin holds count table and hashes for calculating
// frequencies from a given input (usually a stream of data)
type CountMin struct {
	depth      int
	width      int
	size       int64
	eps        float64
	confidence float64
	table      [][]int64
	hashes     []int64
}

// New CountMin given a depth and width of the frequency
// table to use. Returns the newly created CountMin
func New(depth, width int) *CountMin {
	cm := &CountMin{
		depth:      depth,
		width:      width,
		eps:        2.0 / float64(width),
		confidence: 1.0 - 1.0/math.Pow(2, float64(depth)),
	}
	cm.initTable()
	return cm
}

// NewWithEpsCount initializes a CountMin given a confidence
// to ensure between 0 < 1.0 which effects number of hashes used
// and an epsilon between 0 and 1 with a smaller value for
// higher precision as it affects the width of the table
func NewWithEpsCount(confidence, eps float64) *CountMin {
	if confidence >= 1.0 {
		confidence = 0.99999
	}
	cm := &CountMin{
		eps:        eps,
		confidence: confidence,
		width:      int(math.Ceil(float64(2.0) / eps)),
		depth:      int(math.Ceil(-math.Log(1-confidence) / math.Log(2))),
	}
	cm.initTable()
	return cm
}

func (cm *CountMin) initTable() {
	cm.table = make([][]int64, cm.depth)
	for i := range cm.table {
		cm.table[i] = make([]int64, cm.width)
	}
	cm.hashes = make([]int64, cm.depth)
	for i := range cm.hashes {
		cm.hashes[i] = rand.Int63()
	}
}

// RelativeError returns epsilon of CountMin
func (cm *CountMin) RelativeError() float64 {
	return cm.eps
}

// Confidence returns confidence of CountMin
func (cm *CountMin) Confidence() float64 {
	return cm.confidence
}

// Size returns total size of CountMin
func (cm *CountMin) Size() int64 {
	return cm.size
}

// Add inserts an item in bytes and a count associated
// which will also increment size of CountMin
func (cm *CountMin) Add(item []byte, count int64) {
	if count < 0 {
		return
	}
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		cm.table[i][hashed[i]] += count
	}
	cm.size += count
}

// Count gets an associated total count for an item
// in the CountMin
func (cm *CountMin) Count(item []byte) int64 {
	var answer int64 = MAXINT64
	var val int64
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		val = cm.table[i][hashed[i]]
		if val < answer {
			answer = val
		}
	}
	return answer
}

func (cm *CountMin) hasher(item []byte) []int64 {
	total := cm.depth
	max := int64(cm.width)
	result := make([]int64, total)
	hash := fnv.New32a()
	hash.Write(item)
	sum := int64(hash.Sum32())
	for i := 0; i < total; i++ {
		result[i] = int64(math.Abs(float64((sum * int64(i)) % max)))
	}
	return result
}
<|endoftext|>


<commit_before>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * cqlstore.go: a quick & dirty Cassandra CQL backend for Gorilla sessions
 *
 */

import (
	"encoding/gob"
	"fmt"
	"github.com/gocql/gocql"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	"log"
	"net/http"
	"time"
)

type CQLStore struct {
	Cass    *gocql.Session       // connected gocql cassandra session
	Codecs  []securecookie.Codec // session codecs
	Options *sessions.Options    // default configuration
}

func init() {
	gob.Register(time.Now())
}

func NewCQLStore(cass *gocql.Session, keyPairs ...[]byte) *CQLStore {
	return &CQLStore{
		Cass:   cass,
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
	}
}

func (cs *CQLStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(cs, name)
}

func (cs *CQLStore) New(r *http.Request, name string) (sess *sessions.Session, err error) {
	sess = sessions.NewSession(cs, name)
	sess.IsNew = true
	opts := *cs.Options // make a copy
	sess.Options = &opts

	// load the cookie (if it exists)
	c, err := r.Cookie(name)
	if err == nil {
		err = securecookie.DecodeMulti(name, c.Value, &sess.ID, cs.Codecs...)
		if err != nil {
			log.Printf("Cookie decode failed: %s\n", err)
			return
		}
	}

	var data []byte
	var created, modified time.Time
	query := `SELECT data, created, modified FROM sessions WHERE id=?`
	iq := cs.Cass.Query(query, sess.ID).Iter()
	ok := iq.Scan(&data, &created, &modified)
	if ok {
		sess.IsNew = false
		sess.Values["created"] = created
		sess.Values["modified"] = modified
	}

	return
}

func (cs *CQLStore) Save(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	now := time.Now()

	// generate a uuid if there isn't an id already
	if sess.ID == "" {
		var id gocql.UUID
		id, err = gocql.RandomUUID()
		if err != nil {
			return
		}
		sess.ID = id.String()
	}

	// serialize the session for storage in cassandra
	blob, err := securecookie.EncodeMulti(sess.Name(), sess.Values, cs.Codecs...)
	if err != nil {
		log.Printf("Failed to encode session for storage in Cassandra: %s\n", err)
		return
	}

	var created, modified time.Time
	if sess.IsNew {
		created = now
	} else {
		created = sess.Values["created"].(time.Time)
	}
	modified = now

	query := `INSERT INTO sessions (id, data, created, modified) VALUES (?, ?, ?, ?)`
	err = cs.Cass.Query(query, sess.ID, blob, created, modified).Exec()
	if err != nil {
		fmt.Printf("Failed to save session to Cassandra: %s\n", err)
		return
	}

	// update the cookie
	http.SetCookie(w, sessions.NewCookie(sess.Name(), blob, sess.Options))

	return nil
}

func (cs *CQLStore) Delete(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	// overwrite the cookie with a negative max age so the browser expires it immediately
	opts := *sess.Options
	opts.MaxAge = -1
	http.SetCookie(w, sessions.NewCookie(sess.Name(), "", &opts))

	// delete the session from the DB
	err = cs.Cass.Query(`DELETE FROM sessions WHERE id=?`, sess.ID).Exec()
	return
}
<commit_msg>Fix bug in cookie decoding.<commit_after>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * cqlstore.go: a quick & dirty Cassandra CQL backend for Gorilla sessions
 *
 */

import (
	"encoding/gob"
	"fmt"
	"github.com/gocql/gocql"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	"log"
	"net/http"
	"time"
)

type CQLStore struct {
	Cass    *gocql.Session       // connected gocql cassandra session
	Codecs  []securecookie.Codec // session codecs
	Options *sessions.Options    // default configuration
}

func init() {
	gob.Register(time.Now())
}

func NewCQLStore(cass *gocql.Session, keyPairs ...[]byte) *CQLStore {
	return &CQLStore{
		Cass:   cass,
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
	}
}

func (cs *CQLStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(cs, name)
}

func (cs *CQLStore) New(r *http.Request, name string) (sess *sessions.Session, err error) {
	sess = sessions.NewSession(cs, name)
	sess.IsNew = true
	opts := *cs.Options // make a copy
	sess.Options = &opts

	// load the cookie (if it exists)
	c, err := r.Cookie(name)
	if err == nil {
		err = securecookie.DecodeMulti(name, c.Value, &sess.Values, cs.Codecs...)
		if err != nil {
			log.Printf("cqlstore: Cookie decode failed: %s\n", err)
			return
		}
	}

	var data []byte
	var created, modified time.Time
	query := `SELECT data, created, modified FROM sessions WHERE id=?`
	iq := cs.Cass.Query(query, sess.ID).Iter()
	ok := iq.Scan(&data, &created, &modified)
	if ok {
		sess.IsNew = false
		sess.Values["created"] = created
		sess.Values["modified"] = modified
	}

	return
}

func (cs *CQLStore) Save(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	now := time.Now()

	// generate a uuid if there isn't an id already
	if sess.ID == "" {
		var id gocql.UUID
		id, err = gocql.RandomUUID()
		if err != nil {
			return
		}
		sess.ID = id.String()
	}

	// serialize the session for storage in cassandra
	blob, err := securecookie.EncodeMulti(sess.Name(), sess.Values, cs.Codecs...)
	if err != nil {
		log.Printf("cqlstore: Failed to encode session for storage in Cassandra: %s\n", err)
		return
	}

	var created, modified time.Time
	if sess.IsNew {
		created = now
	} else {
		created = sess.Values["created"].(time.Time)
	}
	modified = now

	query := `INSERT INTO sessions (id, data, created, modified) VALUES (?, ?, ?, ?)`
	err = cs.Cass.Query(query, sess.ID, blob, created, modified).Exec()
	if err != nil {
		fmt.Printf("Failed to save session to Cassandra: %s\n", err)
		return
	}

	// update the cookie
	http.SetCookie(w, sessions.NewCookie(sess.Name(), blob, sess.Options))

	return nil
}

func (cs *CQLStore) Delete(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	// overwrite the cookie with a negative max age so the browser expires it immediately
	opts := *sess.Options
	opts.MaxAge = -1
	http.SetCookie(w, sessions.NewCookie(sess.Name(), "", &opts))

	// delete the session from the DB
	err = cs.Cass.Query(`DELETE FROM sessions WHERE id=?`, sess.ID).Exec()
	return
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"testing"
	"time"

	"github.com/TheThingsNetwork/ttn/core"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
	"github.com/brocaar/lorawan"
)

type routerEntryShape struct {
	Until   time.Time
	DevAddr lorawan.DevAddr
	Address string
}

func TestStorageExpiration(t *testing.T) {
	tests := []struct {
		Desc            string
		ExistingEntries []routerEntryShape
		Store           *routerEntryShape
		Lookup          lorawan.DevAddr
		WantEntry       *routerEntryShape
		WantError       []error
	}{
		{
			Desc:            "No entry, Lookup address",
			ExistingEntries: nil,
			Store:           nil,
			Lookup:          lorawan.DevAddr([4]byte{0, 0, 0, 1}),
			WantEntry:       nil,
			WantError:       []error{ErrNotFound},
		},
	}

	for _, test := range tests {
		// Describe
		Desc(t, test.Desc)

		// Build
		db := genFilledRouterStorage(test.ExistingEntries)
		cherr := make(chan interface{}, 2)

		// Operate
		storeRouter(db, test.Store, cherr)
		got := lookupRouter(db, test.Lookup, cherr)

		// Check
		checkChErrors(t, test.WantError, cherr)
		checkRouterEntries(t, test.WantEntry, got)
	}
}

// ----- BUILD utilities
func genFilledRouterStorage(setup []routerEntryShape) RouterStorage {
	db, err := NewRouterStorage()
	if err != nil {
		panic(err)
	}

	if err := db.Reset(); err != nil {
		panic(err)
	}

	for i, shape := range setup {
		entry := routerEntry{
			until: shape.Until,
			Recipient: core.Recipient{
				Address: shape.Address,
				Id:      i,
			},
		}
		if err := db.Store(shape.DevAddr, entry); err != nil {
			panic(err)
		}
	}

	return db
}

// ----- OPERATE utilities
func storeRouter(db RouterStorage, entry *routerEntryShape, cherr chan interface{}) {

}

func lookupRouter(db RouterStorage, devAddr lorawan.DevAddr, cherr chan interface{}) routerEntry {
	return routerEntry{}
}

// ----- CHECK utilities
func checkRouterEntries(t *testing.T, want *routerEntryShape, got routerEntry) {
}
<commit_msg>[tests.router] Implements test utilities for router storage testing<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"reflect"
	"testing"
	"time"

	"github.com/TheThingsNetwork/ttn/core"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
	"github.com/brocaar/lorawan"
)

type routerEntryShape struct {
	DevAddr lorawan.DevAddr
	Address string
}

func TestStorageExpiration(t *testing.T) {
	tests := []struct {
		Desc            string
		ExistingEntries []routerEntryShape
		ExpiryDelay     time.Duration
		Store           *routerEntryShape
		Lookup          lorawan.DevAddr
		WantEntry       *routerEntryShape
		WantError       []error
	}{
		{
			Desc:            "No entry, Lookup address",
			ExistingEntries: nil,
			ExpiryDelay:     time.Minute,
			Store:           nil,
			Lookup:          lorawan.DevAddr([4]byte{0, 0, 0, 1}),
			WantEntry:       nil,
			WantError:       []error{ErrNotFound},
		},
	}

	for _, test := range tests {
		// Describe
		Desc(t, test.Desc)

		// Build
		db := genFilledRouterStorage(test.ExistingEntries)
		cherr := make(chan interface{}, 2)
		go func() {
			time.After(time.Millisecond * 250)
			close(cherr)
		}()

		// Operate
		storeRouter(db, test.Store, cherr)
		got := lookupRouter(db, test.Lookup, cherr)

		// Check
		checkChErrors(t, test.WantError, cherr)
		checkRouterEntries(t, test.WantEntry, got)

		// Clean
		db.Close()
	}
}

// ----- BUILD utilities
func genFilledRouterStorage(setup []routerEntryShape) RouterStorage {
	db, err := NewRouterStorage()
	if err != nil {
		panic(err)
	}

	if err := db.Reset(); err != nil {
		panic(err)
	}

	for i, shape := range setup {
		entry := routerEntry{
			Recipient: core.Recipient{
				Address: shape.Address,
				Id:      i,
			},
		}
		if err := db.Store(shape.DevAddr, entry); err != nil {
			panic(err)
		}
	}

	return db
}

// ----- OPERATE utilities
func storeRouter(db RouterStorage, entry *routerEntryShape, cherr chan interface{}) {
	if entry == nil {
		return
	}

	err := db.Store(entry.DevAddr, routerEntry{
		Recipient: core.Recipient{
			Address: entry.Address,
			Id:      "LikeICare",
		},
	})

	if err != nil {
		cherr <- err
	}
}

func lookupRouter(db RouterStorage, devAddr lorawan.DevAddr, cherr chan interface{}) routerEntry {
	entry, err := db.Lookup(devAddr)
	if err != nil {
		cherr <- err
	}
	return entry
}

// ----- CHECK utilities
func checkRouterEntries(t *testing.T, want *routerEntryShape, got routerEntry) {
	if want != nil {
		addr, ok := got.Recipient.Address.(string)

		if !ok {
			Ko(t, "Unexpected recipient address format: %+v", got.Recipient.Address)
			return
		}

		if addr != want.Address {
			Ko(t, "The retrieved address [%s] does not match expected [%s]", addr, want.Address)
			return
		}
	} else {
		if !reflect.DeepEqual(routerEntry{}, got) {
			Ko(t, "No entry was exected but got: %v", got)
			return
		}
	}
	Ok(t, "Check router entries")
}
<|endoftext|>


<commit_before>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package data

import (
	"bytes"
	"testing"

	"github.com/issue9/assert"
)

func TestLoadTemplate(t *testing.T) {
	a := assert.New(t)

	d := &Data{Config: &Config{Theme: "t1"}}
	a.NotError(d.loadTemplate("./testdata/themes")).NotNil(d.Template)

	w := bytes.NewBufferString("")
	a.NotError(d.Template.ExecuteTemplate(w, "header.html", map[string]string{"val": "123"}))
	a.Equal(w.String(), "
123
\n")
}

func TestGetThemesName(t *testing.T) {
	a := assert.New(t)

	themes, err := getThemesName("./testdata/themes")
	a.NotError(err).NotNil(themes).Equal(2, len(themes))
}

func TestData_longDataFormat(t *testing.T) {
	a := assert.New(t)

	d := &Data{Config: &Config{LongDateFormat: "2006-01-02T15:04:05-0700"}}
	a.Equal(d.longDateFormat(1456324895), "2016-02-24T22:41:35+0800")
}

func TestData_shortDataFormat(t *testing.T) {
	a := assert.New(t)

	d := &Data{Config: &Config{ShortDateFormat: "2006-01-02"}}
	a.Equal(d.shortDateFormat(1456324895), "2016-02-24")
}

func TestStripTags(t *testing.T) {
	a := assert.New(t)

	tests := map[string]string{
		"
str
":        "str",
		"str
":             "str",
		"

str

": "str",
	}

	for expr, val := range tests {
		a.Equal(stripTags(expr), val, "测试[%v]时出错", expr)
	}
}
<commit_msg>去掉与时区相关的测试代码<commit_after>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package data

import (
	"bytes"
	"testing"

	"github.com/issue9/assert"
)

func TestLoadTemplate(t *testing.T) {
	a := assert.New(t)

	d := &Data{Config: &Config{Theme: "t1"}}
	a.NotError(d.loadTemplate("./testdata/themes")).NotNil(d.Template)

	w := bytes.NewBufferString("")
	a.NotError(d.Template.ExecuteTemplate(w, "header.html", map[string]string{"val": "123"}))
	a.Equal(w.String(), "
123
\n")
}

func TestGetThemesName(t *testing.T) {
	a := assert.New(t)

	themes, err := getThemesName("./testdata/themes")
	a.NotError(err).NotNil(themes).Equal(2, len(themes))
}

/*func TestData_longDataFormat(t *testing.T) {
	a := assert.New(t)

	d := &Data{Config: &Config{LongDateFormat: "2006-01-02T15:04:05-0700"}}
	a.Equal(d.longDateFormat(1456324895), "2016-02-24T22:41:35+0800")
}

func TestData_shortDataFormat(t *testing.T) {
	a := assert.New(t)

	d := &Data{Config: &Config{ShortDateFormat: "2006-01-02"}}
	a.Equal(d.shortDateFormat(1456324895), "2016-02-24")
}*/

func TestStripTags(t *testing.T) {
	a := assert.New(t)

	tests := map[string]string{
		"
str
":        "str",
		"str
":             "str",
		"

str

": "str",
	}

	for expr, val := range tests {
		a.Equal(stripTags(expr), val, "测试[%v]时出错", expr)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package description

import (
	"time"

	"github.com/juju/errors"
	"github.com/juju/schema"
)

type cloudimagemetadata struct {
	Version  int       `yaml:"version"`
	CloudImageMetadatas_ []*cloudimagemetadata `yaml:"cloudimagemetadata"`
}

type cloudimagemetadata struct {
	Id_         string                 `yaml:"id"`
	Receiver_   string                 `yaml:"receiver"`
	Name_       string                 `yaml:"name"`
	Parameters_ map[string]interface{} `yaml:"parameters"`
	Enqueued_   time.Time              `yaml:"enqueued"`
	// Can't use omitempty with time.Time, it just doesn't work
	// (nothing is serialised), so use a pointer in the struct.
	Started_   *time.Time             `yaml:"started,omitempty"`
	Completed_ *time.Time             `yaml:"completed,omitempty"`
	Status_    string                 `yaml:"status"`
	Message_   string                 `yaml:"message"`
	Results_   map[string]interface{} `yaml:"results"`
}

// Id implements CloudImageMetadata.
func (i *cloudimagemetadata) Id() string {
	return i.Id_
}

// Receiver implements CloudImageMetadata.
func (i *cloudimagemetadata) Receiver() string {
	return i.Receiver_
}

// Name implements CloudImageMetadata.
func (i *cloudimagemetadata) Name() string {
	return i.Name_
}

// Parameters implements CloudImageMetadata.
func (i *cloudimagemetadata) Parameters() map[string]interface{} {
	return i.Parameters_
}

// Enqueued implements CloudImageMetadata.
func (i *cloudimagemetadata) Enqueued() time.Time {
	return i.Enqueued_
}

// Started implements CloudImageMetadata.
func (i *cloudimagemetadata) Started() time.Time {
	var zero time.Time
	if i.Started_ == nil {
		return zero
	}
	return *i.Started_
}

// Completed implements CloudImageMetadata.
func (i *cloudimagemetadata) Completed() time.Time {
	var zero time.Time
	if i.Completed_ == nil {
		return zero
	}
	return *i.Completed_
}

// Status implements CloudImageMetadata.
func (i *cloudimagemetadata) Status() string {
	return i.Status_
}

// Message implements CloudImageMetadata.
func (i *cloudimagemetadata) Message() string {
	return i.Message_
}

// Results implements CloudImageMetadata.
func (i *cloudimagemetadata) Results() map[string]interface{} {
	return i.Results_
}

// CloudImageMetadataArgs is an argument struct used to create a
// new internal cloudimagemetadata type that supports the CloudImageMetadata interface.
type CloudImageMetadataArgs struct {
	Id         string
	Receiver   string
	Name       string
	Parameters map[string]interface{}
	Enqueued   time.Time
	Started    time.Time
	Completed  time.Time
	Status     string
	Message    string
	Results    map[string]interface{}
}

func newCloudImageMetadata(args CloudImageMetadataArgs) *cloudimagemetadata {
	cloudimagemetadata := &cloudimagemetadata{
		Receiver_:   args.Receiver,
		Name_:       args.Name,
		Parameters_: args.Parameters,
		Enqueued_:   args.Enqueued,
		Status_:     args.Status,
		Message_:    args.Message,
		Id_:         args.Id,
		Results_:    args.Results,
	}
	if !args.Started.IsZero() {
		value := args.Started
		cloudimagemetadata.Started_ = &value
	}
	if !args.Completed.IsZero() {
		value := args.Completed
		cloudimagemetadata.Completed_ = &value
	}
	return cloudimagemetadata
}

func importCloudImageMetadatas(source map[string]interface{}) ([]*cloudimagemetadata, error) {
	checker := versionedChecker("cloudimagemetadata")
	coerced, err := checker.Coerce(source, nil)
	if err != nil {
		return nil, errors.Annotatef(err, "cloudimagemetadata version schema check failed")
	}
	valid := coerced.(map[string]interface{})

	version := int(valid["version"].(int64))
	importFunc, ok := cloudimagemetadataDeserializationFuncs[version]
	if !ok {
		return nil, errors.NotValidf("version %d", version)
	}
	sourceList := valid["cloudimagemetadata"].([]interface{})
	return importCloudImageMetadataList(sourceList, importFunc)
}

func importCloudImageMetadataList(sourceList []interface{}, importFunc cloudimagemetadataDeserializationFunc) ([]*cloudimagemetadata, error) {
	result := make([]*cloudimagemetadata, 0, len(sourceList))
	for i, value := range sourceList {
		source, ok := value.(map[string]interface{})
		if !ok {
			return nil, errors.Errorf("unexpected value for cloudimagemetadata %d, %T", i, value)
		}
		cloudimagemetadata, err := importFunc(source)
		if err != nil {
			return nil, errors.Annotatef(err, "cloudimagemetadata %d", i)
		}
		result = append(result, cloudimagemetadata)
	}
	return result, nil
}

type cloudimagemetadataDeserializationFunc func(map[string]interface{}) (*cloudimagemetadata, error)

var cloudimagemetadataDeserializationFuncs = map[int]cloudimagemetadataDeserializationFunc{
	1: importCloudImageMetadataV1,
}

func importCloudImageMetadataV1(source map[string]interface{}) (*cloudimagemetadata, error) {
	fields := schema.Fields{
		"receiver":   schema.String(),
		"name":       schema.String(),
		"parameters": schema.StringMap(schema.Any()),
		"enqueued":   schema.Time(),
		"started":    schema.Time(),
		"completed":  schema.Time(),
		"status":     schema.String(),
		"message":    schema.String(),
		"results":    schema.StringMap(schema.Any()),
		"id":         schema.String(),
	}
	// Some values don't have to be there.
	defaults := schema.Defaults{
		"started":   time.Time{},
		"completed": time.Time{},
	}
	checker := schema.FieldMap(fields, defaults)

	coerced, err := checker.Coerce(source, nil)
	if err != nil {
		return nil, errors.Annotatef(err, "cloudimagemetadata v1 schema check failed")
	}
	valid := coerced.(map[string]interface{})
	cloudimagemetadata := &cloudimagemetadata{
		Id_:         valid["id"].(string),
		Receiver_:   valid["receiver"].(string),
		Name_:       valid["name"].(string),
		Status_:     valid["status"].(string),
		Message_:    valid["message"].(string),
		Parameters_: valid["parameters"].(map[string]interface{}),
		Enqueued_:   valid["enqueued"].(time.Time),
		Results_:    valid["results"].(map[string]interface{}),
	}

	started := valid["started"].(time.Time)
	if !started.IsZero() {
		cloudimagemetadata.Started_ = &started
	}
	completed := valid["completed"].(time.Time)
	if !started.IsZero() {
		cloudimagemetadata.Completed_ = &completed
	}
	return cloudimagemetadata, nil
}
<commit_msg>go fmt<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package description

import (
	"time"

	"github.com/juju/errors"
	"github.com/juju/schema"
)

type cloudimagemetadata struct {
	Version              int                   `yaml:"version"`
	CloudImageMetadatas_ []*cloudimagemetadata `yaml:"cloudimagemetadata"`
}

type cloudimagemetadata struct {
	Id_         string                 `yaml:"id"`
	Receiver_   string                 `yaml:"receiver"`
	Name_       string                 `yaml:"name"`
	Parameters_ map[string]interface{} `yaml:"parameters"`
	Enqueued_   time.Time              `yaml:"enqueued"`
	// Can't use omitempty with time.Time, it just doesn't work
	// (nothing is serialised), so use a pointer in the struct.
	Started_   *time.Time             `yaml:"started,omitempty"`
	Completed_ *time.Time             `yaml:"completed,omitempty"`
	Status_    string                 `yaml:"status"`
	Message_   string                 `yaml:"message"`
	Results_   map[string]interface{} `yaml:"results"`
}

// Id implements CloudImageMetadata.
func (i *cloudimagemetadata) Id() string {
	return i.Id_
}

// Receiver implements CloudImageMetadata.
func (i *cloudimagemetadata) Receiver() string {
	return i.Receiver_
}

// Name implements CloudImageMetadata.
func (i *cloudimagemetadata) Name() string {
	return i.Name_
}

// Parameters implements CloudImageMetadata.
func (i *cloudimagemetadata) Parameters() map[string]interface{} {
	return i.Parameters_
}

// Enqueued implements CloudImageMetadata.
func (i *cloudimagemetadata) Enqueued() time.Time {
	return i.Enqueued_
}

// Started implements CloudImageMetadata.
func (i *cloudimagemetadata) Started() time.Time {
	var zero time.Time
	if i.Started_ == nil {
		return zero
	}
	return *i.Started_
}

// Completed implements CloudImageMetadata.
func (i *cloudimagemetadata) Completed() time.Time {
	var zero time.Time
	if i.Completed_ == nil {
		return zero
	}
	return *i.Completed_
}

// Status implements CloudImageMetadata.
func (i *cloudimagemetadata) Status() string {
	return i.Status_
}

// Message implements CloudImageMetadata.
func (i *cloudimagemetadata) Message() string {
	return i.Message_
}

// Results implements CloudImageMetadata.
func (i *cloudimagemetadata) Results() map[string]interface{} {
	return i.Results_
}

// CloudImageMetadataArgs is an argument struct used to create a
// new internal cloudimagemetadata type that supports the CloudImageMetadata interface.
type CloudImageMetadataArgs struct {
	Id         string
	Receiver   string
	Name       string
	Parameters map[string]interface{}
	Enqueued   time.Time
	Started    time.Time
	Completed  time.Time
	Status     string
	Message    string
	Results    map[string]interface{}
}

func newCloudImageMetadata(args CloudImageMetadataArgs) *cloudimagemetadata {
	cloudimagemetadata := &cloudimagemetadata{
		Receiver_:   args.Receiver,
		Name_:       args.Name,
		Parameters_: args.Parameters,
		Enqueued_:   args.Enqueued,
		Status_:     args.Status,
		Message_:    args.Message,
		Id_:         args.Id,
		Results_:    args.Results,
	}
	if !args.Started.IsZero() {
		value := args.Started
		cloudimagemetadata.Started_ = &value
	}
	if !args.Completed.IsZero() {
		value := args.Completed
		cloudimagemetadata.Completed_ = &value
	}
	return cloudimagemetadata
}

func importCloudImageMetadatas(source map[string]interface{}) ([]*cloudimagemetadata, error) {
	checker := versionedChecker("cloudimagemetadata")
	coerced, err := checker.Coerce(source, nil)
	if err != nil {
		return nil, errors.Annotatef(err, "cloudimagemetadata version schema check failed")
	}
	valid := coerced.(map[string]interface{})

	version := int(valid["version"].(int64))
	importFunc, ok := cloudimagemetadataDeserializationFuncs[version]
	if !ok {
		return nil, errors.NotValidf("version %d", version)
	}
	sourceList := valid["cloudimagemetadata"].([]interface{})
	return importCloudImageMetadataList(sourceList, importFunc)
}

func importCloudImageMetadataList(sourceList []interface{}, importFunc cloudimagemetadataDeserializationFunc) ([]*cloudimagemetadata, error) {
	result := make([]*cloudimagemetadata, 0, len(sourceList))
	for i, value := range sourceList {
		source, ok := value.(map[string]interface{})
		if !ok {
			return nil, errors.Errorf("unexpected value for cloudimagemetadata %d, %T", i, value)
		}
		cloudimagemetadata, err := importFunc(source)
		if err != nil {
			return nil, errors.Annotatef(err, "cloudimagemetadata %d", i)
		}
		result = append(result, cloudimagemetadata)
	}
	return result, nil
}

type cloudimagemetadataDeserializationFunc func(map[string]interface{}) (*cloudimagemetadata, error)

var cloudimagemetadataDeserializationFuncs = map[int]cloudimagemetadataDeserializationFunc{
	1: importCloudImageMetadataV1,
}

func importCloudImageMetadataV1(source map[string]interface{}) (*cloudimagemetadata, error) {
	fields := schema.Fields{
		"receiver":   schema.String(),
		"name":       schema.String(),
		"parameters": schema.StringMap(schema.Any()),
		"enqueued":   schema.Time(),
		"started":    schema.Time(),
		"completed":  schema.Time(),
		"status":     schema.String(),
		"message":    schema.String(),
		"results":    schema.StringMap(schema.Any()),
		"id":         schema.String(),
	}
	// Some values don't have to be there.
	defaults := schema.Defaults{
		"started":   time.Time{},
		"completed": time.Time{},
	}
	checker := schema.FieldMap(fields, defaults)

	coerced, err := checker.Coerce(source, nil)
	if err != nil {
		return nil, errors.Annotatef(err, "cloudimagemetadata v1 schema check failed")
	}
	valid := coerced.(map[string]interface{})
	cloudimagemetadata := &cloudimagemetadata{
		Id_:         valid["id"].(string),
		Receiver_:   valid["receiver"].(string),
		Name_:       valid["name"].(string),
		Status_:     valid["status"].(string),
		Message_:    valid["message"].(string),
		Parameters_: valid["parameters"].(map[string]interface{}),
		Enqueued_:   valid["enqueued"].(time.Time),
		Results_:    valid["results"].(map[string]interface{}),
	}

	started := valid["started"].(time.Time)
	if !started.IsZero() {
		cloudimagemetadata.Started_ = &started
	}
	completed := valid["completed"].(time.Time)
	if !started.IsZero() {
		cloudimagemetadata.Completed_ = &completed
	}
	return cloudimagemetadata, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"os"
)

// askURL provide an i/o process asking for a valid URL
func askURL() string {
	var url string
	fmt.Print("URL: ")
	fmt.Scanf("%s", &url)

	if (url == "exit") {
		os.Exit(1)
	}

	if (url == "") ||
		((!strings.HasPrefix(url, "http:")) ||
			(!strings.HasPrefix(url, "https:"))) {
		fmt.Println("Please type in a valid url or use http://")
		url = askURL()
	}

	return url
}

func main() {
	url := askURL()

	data, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	} else {
		fmt.Println("Fetching url...")
		defer data.Body.Close()
		contents, err := ioutil.ReadAll(data.Body)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("%s\n", string(contents))
	}
}<commit_msg>Fix error<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"os"
)

// askURL provide an i/o process asking for a valid URL
func askURL() string {
	var url string
	fmt.Print("URL: ")
	fmt.Scanf("%s", &url)

	if (url == "exit") {
		os.Exit(1)
	}

	if (url == "") ||
		((!strings.HasPrefix(url, "http:")) ||
			(!strings.HasPrefix(url, "http:"))) {
		fmt.Println("Please type in a valid url or use http://")
		url = askURL()
	}

	return url
}

func main() {
	url := askURL()

	data, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	} else {
		fmt.Println("Fetching url...")
		defer data.Body.Close()
		contents, err := ioutil.ReadAll(data.Body)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("%s\n", string(contents))
	}
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"ht"
	"liftca"
	"liftca/cmd/liftca/handlers"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

func myHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprint(w, "plop")
}

func main() {
	quit := make(chan bool)

	var addressArg string
	var storeFileArg string

	flag.StringVar(&addressArg, "a", ":8080", "listen address")
	flag.StringVar(&storeFileArg, "s", "store.gob", "path to state storage file")
	flag.Parse()

	storeFile := filepath.Clean(storeFileArg)
	backingFile, err := os.OpenFile(storeFile, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}
	store := liftca.NewStore(backingFile)
	r := ht.NewRouter()
	fileServer := ht.NewLoggingFileServer(http.Dir("static"))

	r.Handle("GET", "/ca", ht.NewHandler(store, handlers.GetCAs))
	r.Handle("POST", "/ca", ht.NewHandler(store, handlers.PostCA))
	r.Handle("GET", "/ca/{ca_id}-certificate.cer", ht.NewHandler(store, handlers.GetCACertificateCER))
	r.Handle("GET", "/ca/{ca_id}-crl.crl", ht.NewHandler(store, handlers.GetCACRLCER))
	r.Handle("GET", "/ca/{ca_id}-private-key.cer", ht.NewHandler(store, handlers.GetCAPrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem", ht.NewHandler(store, handlers.GetCACertificatePEM))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCACertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem", ht.NewHandler(store, handlers.GetCAPrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCAPrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}-crl.pem", ht.NewHandler(store, handlers.GetCACRLPEM))
	r.Handle("GET", "/ca/{ca_id}-crl.pem.txt", ht.NewHandler(store, handlers.GetCACRLPEMTXT))
	r.Handle("GET", "/ca/{ca_id}", ht.NewHandler(store, handlers.GetCA))
	r.Handle("GET", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.GetCerts))
	r.Handle("POST", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.PostCert))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem", ht.NewHandler(store, handlers.GetCertificatePEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.cer", ht.NewHandler(store, handlers.GetCertificatePrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.cer", ht.NewHandler(store, handlers.GetCertificateCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}", ht.NewHandler(store, handlers.GetCert))
	r.Handle("POST", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.PostCRL))
	r.Handle("GET", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.GetCRL))
	r.Handle("DELETE", "/ca/{ca_id}/crl/{cert_id}", ht.NewHandler(store, handlers.DeleteCRL))
	r.Handle("GET", "/", fileServer)
	r.Handle("GET", "/{f}", fileServer)
	r.Handle("GET", "/js/{f}", fileServer)
	r.Handle("GET", "/fonts/{f}", fileServer)
	r.Handle("GET", "/css/{f}", fileServer)
	r.Handle("GET", "/img/{f}", fileServer)
	r.Handle("GET", "/partials/{f}", fileServer)

	s := &http.Server{
		Addr:           addressArg,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
		Handler:        r,
	}
	go s.ListenAndServe()

	log.Printf("liftCA engaged at '%v', data file '%v'", addressArg, storeFile)
	<-quit
}
<commit_msg>Remove old test code<commit_after>package main

import (
	"flag"
	"fmt"
	"ht"
	"liftca"
	"liftca/cmd/liftca/handlers"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

func main() {
	quit := make(chan bool)

	var addressArg string
	var storeFileArg string

	flag.StringVar(&addressArg, "a", ":8080", "listen address")
	flag.StringVar(&storeFileArg, "s", "store.gob", "path to state storage file")
	flag.Parse()

	storeFile := filepath.Clean(storeFileArg)
	backingFile, err := os.OpenFile(storeFile, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}
	store := liftca.NewStore(backingFile)
	r := ht.NewRouter()
	fileServer := ht.NewLoggingFileServer(http.Dir("static"))

	r.Handle("GET", "/ca", ht.NewHandler(store, handlers.GetCAs))
	r.Handle("POST", "/ca", ht.NewHandler(store, handlers.PostCA))
	r.Handle("GET", "/ca/{ca_id}-certificate.cer", ht.NewHandler(store, handlers.GetCACertificateCER))
	r.Handle("GET", "/ca/{ca_id}-crl.crl", ht.NewHandler(store, handlers.GetCACRLCER))
	r.Handle("GET", "/ca/{ca_id}-private-key.cer", ht.NewHandler(store, handlers.GetCAPrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem", ht.NewHandler(store, handlers.GetCACertificatePEM))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCACertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem", ht.NewHandler(store, handlers.GetCAPrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCAPrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}-crl.pem", ht.NewHandler(store, handlers.GetCACRLPEM))
	r.Handle("GET", "/ca/{ca_id}-crl.pem.txt", ht.NewHandler(store, handlers.GetCACRLPEMTXT))
	r.Handle("GET", "/ca/{ca_id}", ht.NewHandler(store, handlers.GetCA))
	r.Handle("GET", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.GetCerts))
	r.Handle("POST", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.PostCert))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem", ht.NewHandler(store, handlers.GetCertificatePEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.cer", ht.NewHandler(store, handlers.GetCertificatePrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.cer", ht.NewHandler(store, handlers.GetCertificateCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}", ht.NewHandler(store, handlers.GetCert))
	r.Handle("POST", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.PostCRL))
	r.Handle("GET", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.GetCRL))
	r.Handle("DELETE", "/ca/{ca_id}/crl/{cert_id}", ht.NewHandler(store, handlers.DeleteCRL))
	r.Handle("GET", "/", fileServer)
	r.Handle("GET", "/{f}", fileServer)
	r.Handle("GET", "/js/{f}", fileServer)
	r.Handle("GET", "/fonts/{f}", fileServer)
	r.Handle("GET", "/css/{f}", fileServer)
	r.Handle("GET", "/img/{f}", fileServer)
	r.Handle("GET", "/partials/{f}", fileServer)

	s := &http.Server{
		Addr:           addressArg,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
		Handler:        r,
	}
	go s.ListenAndServe()

	log.Printf("liftCA engaged at '%v', data file '%v'", addressArg, storeFile)
	<-quit
}
<|endoftext|>


<commit_before><commit_msg>Use test case pattern for bit-mask tests<commit_after><|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSesReceiptRuleSet() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSesReceiptRuleSetCreate,
		Read:   resourceAwsSesReceiptRuleSetRead,
		Delete: resourceAwsSesReceiptRuleSetDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"rule_set_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsSesReceiptRuleSetCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	ruleSetName := d.Get("rule_set_name").(string)

	createOpts := &ses.CreateReceiptRuleSetInput{
		RuleSetName: aws.String(ruleSetName),
	}

	_, err := conn.CreateReceiptRuleSet(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating SES rule set: %s", err)
	}

	d.SetId(ruleSetName)

	return resourceAwsSesReceiptRuleSetRead(d, meta)
}

func resourceAwsSesReceiptRuleSetRead(d *schema.ResourceData, meta interface{}) error {
	ruleSetExists, err := findRuleSet(d.Id(), nil, meta)

	if !ruleSetExists {
		log.Printf("[WARN] SES Receipt Rule Set (%s) not found", d.Id())
		d.SetId("")
		return nil
	}

	if err != nil {
		return err
	}

	d.Set("rule_set_name", d.Id())

	return nil
}

func resourceAwsSesReceiptRuleSetDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	log.Printf("[DEBUG] SES Delete Receipt Rule Set: %s", d.Id())
	_, err := conn.DeleteReceiptRuleSet(&ses.DeleteReceiptRuleSetInput{
		RuleSetName: aws.String(d.Id()),
	})

	return err
}

func findRuleSet(name string, token *string, meta interface{}) (bool, error) {
	conn := meta.(*AWSClient).sesConn

	ruleSetExists := false

	listOpts := &ses.ListReceiptRuleSetsInput{
		NextToken: token,
	}

	response, err := conn.ListReceiptRuleSets(listOpts)
	for _, element := range response.RuleSets {
		if *element.Name == name {
			ruleSetExists = true
		}
	}

	if err != nil && !ruleSetExists && response.NextToken != nil {
		ruleSetExists, err = findRuleSet(name, response.NextToken, meta)
	}

	if err != nil {
		return false, err
	}

	return ruleSetExists, nil
}
<commit_msg>resource/aws_ses_receipt_rule_set: Prevent missing Terraform state for newly created resources<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSesReceiptRuleSet() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSesReceiptRuleSetCreate,
		Read:   resourceAwsSesReceiptRuleSetRead,
		Delete: resourceAwsSesReceiptRuleSetDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"rule_set_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsSesReceiptRuleSetCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	ruleSetName := d.Get("rule_set_name").(string)

	createOpts := &ses.CreateReceiptRuleSetInput{
		RuleSetName: aws.String(ruleSetName),
	}

	_, err := conn.CreateReceiptRuleSet(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating SES rule set: %s", err)
	}

	d.SetId(ruleSetName)

	return resourceAwsSesReceiptRuleSetRead(d, meta)
}

func resourceAwsSesReceiptRuleSetRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	input := &ses.DescribeReceiptRuleSetInput{
		RuleSetName: aws.String(d.Id()),
	}

	_, err := conn.DescribeReceiptRuleSet(input)

	if isAWSErr(err, ses.ErrCodeRuleSetDoesNotExistException, "") {
		log.Printf("[WARN] SES Receipt Rule Set (%s) not found, removing from state", d.Id())
		d.SetId("")
		return nil
	}

	if err != nil {
		return fmt.Errorf("error describing SES Receipt Rule Set (%s): %s", d.Id(), err)
	}

	d.Set("rule_set_name", d.Id())

	return nil
}

func resourceAwsSesReceiptRuleSetDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	log.Printf("[DEBUG] SES Delete Receipt Rule Set: %s", d.Id())
	_, err := conn.DeleteReceiptRuleSet(&ses.DeleteReceiptRuleSetInput{
		RuleSetName: aws.String(d.Id()),
	})

	return err
}
<|endoftext|>