<commit_before>package fingerprint

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/stretchr/testify/require"
)

func TestEnvAWSFingerprint_nonAws(t *testing.T) {
	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = "http://127.0.0.1/latest"

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)
	require.Empty(t, response.Attributes)
}

func TestEnvAWSFingerprint_aws(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	keys := []string{
		"platform.aws.ami-id",
		"unique.platform.aws.hostname",
		"unique.platform.aws.instance-id",
		"platform.aws.instance-type",
		"unique.platform.aws.local-hostname",
		"unique.platform.aws.local-ipv4",
		"unique.platform.aws.public-hostname",
		"unique.platform.aws.public-ipv4",
		"platform.aws.placement.availability-zone",
		"unique.network.ip-address",
	}

	for _, k := range keys {
		assertNodeAttributeContains(t, response.Attributes, k)
	}

	require.NotEmpty(t, response.Links)

	// confirm we have at least instance-id and ami-id
	for _, k := range []string{"aws.ec2"} {
		assertNodeLinksContains(t, response.Links, k)
	}
}

func TestNetworkFingerprint_AWS(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	assertNodeAttributeContains(t, response.Attributes, "unique.network.ip-address")

	require.NotNil(t, response.NodeResources)
	require.Len(t, response.NodeResources.Networks, 1)

	// Test at least the first Network Resource
	net := response.NodeResources.Networks[0]
	require.NotEmpty(t, net.IP, "Expected Network Resource to have an IP")
	require.NotEmpty(t, net.CIDR, "Expected Network Resource to have a CIDR")
	require.NotEmpty(t, net.Device, "Expected Network Resource to have a Device Name")
}

func TestNetworkFingerprint_AWS_network(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	{
		node := &structs.Node{
			Attributes: make(map[string]string),
		}

		request := &FingerprintRequest{Config: &config.Config{}, Node: node}
		var response FingerprintResponse
		err := f.Fingerprint(request, &response)
		require.NoError(t, err)

		require.True(t, response.Detected, "expected response to be applicable")

		assertNodeAttributeContains(t, response.Attributes, "unique.network.ip-address")

		require.NotNil(t, response.NodeResources)
		require.Len(t, response.NodeResources.Networks, 1)

		// Test at least the first Network Resource
		net := response.NodeResources.Networks[0]
		require.NotEmpty(t, net.IP, "Expected Network Resource to have an IP")
		require.NotEmpty(t, net.CIDR, "Expected Network Resource to have a CIDR")
		require.NotEmpty(t, net.Device, "Expected Network Resource to have a Device Name")
		require.Equal(t, 1000, net.MBits)
	}

	// Try again this time setting a network speed in the config
	{
		node := &structs.Node{
			Attributes: make(map[string]string),
		}

		cfg := &config.Config{
			NetworkSpeed: 10,
		}

		request := &FingerprintRequest{Config: cfg, Node: node}
		var response FingerprintResponse
		err := f.Fingerprint(request, &response)
		require.NoError(t, err)

		assertNodeAttributeContains(t, response.Attributes, "unique.network.ip-address")

		require.NotNil(t, response.NodeResources)
		require.Len(t, response.NodeResources.Networks, 1)

		// Test at least the first Network Resource
		net := response.NodeResources.Networks[0]
		require.NotEmpty(t, net.IP, "Expected Network Resource to have an IP")
		require.NotEmpty(t, net.CIDR, "Expected Network Resource to have a CIDR")
		require.NotEmpty(t, net.Device, "Expected Network Resource to have a Device Name")
		require.Equal(t, 10, net.MBits)
	}
}

func TestNetworkFingerprint_AWS_NoNetwork(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, noNetworkAWSStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	require.True(t, response.Detected, "expected response to be applicable")

	require.Equal(t, "ami-1234", response.Attributes["platform.aws.ami-id"])

	require.Nil(t, response.NodeResources)
}

func TestNetworkFingerprint_AWS_IncompleteImmitation(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, incompleteAWSImmitationStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	require.False(t, response.Detected, "expected response not to be applicable")

	require.NotContains(t, response.Attributes, "platform.aws.ami-id")
	require.Nil(t, response.NodeResources)
}

/// Utility functions for tests

func startFakeEC2Metadata(t *testing.T, endpoints []endpoint) (endpoint string, cleanup func()) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for _, e := range endpoints {
			if r.RequestURI == e.Uri {
				w.Header().Set("Content-Type", e.ContentType)
				fmt.Fprintln(w, e.Body)
			}
		}
	}))

	return ts.URL + "/latest", ts.Close
}

type routes struct {
	Endpoints []*endpoint `json:"endpoints"`
}

type endpoint struct {
	Uri         string
	ContentType string
	Body        string
}

// awsStubs mimics normal EC2 instance metadata
var awsStubs = []endpoint{
	{
		Uri:         "/latest/meta-data/ami-id",
		ContentType: "text/plain",
		Body:        "ami-1234",
	},
	{
		Uri:         "/latest/meta-data/hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/placement/availability-zone",
		ContentType: "text/plain",
		Body:        "us-west-2a",
	},
	{
		Uri:         "/latest/meta-data/instance-id",
		ContentType: "text/plain",
		Body:        "i-b3ba3875",
	},
	{
		Uri:         "/latest/meta-data/instance-type",
		ContentType: "text/plain",
		Body:        "m3.2xlarge",
	},
	{
		Uri:         "/latest/meta-data/local-hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/local-ipv4",
		ContentType: "text/plain",
		Body:        "10.0.0.207",
	},
	{
		Uri:         "/latest/meta-data/public-hostname",
		ContentType: "text/plain",
		Body:        "ec2-54-191-117-175.us-west-2.compute.amazonaws.com",
	},
	{
		Uri:         "/latest/meta-data/public-ipv4",
		ContentType: "text/plain",
		Body:        "54.191.117.175",
	},
}

// noNetworkAWSStubs mimics an EC2 instance but without local ip address
// may happen in environments with odd EC2 Metadata emulation
var noNetworkAWSStubs = []endpoint{
	{
		Uri:         "/latest/meta-data/ami-id",
		ContentType: "text/plain",
		Body:        "ami-1234",
	},
	{
		Uri:         "/latest/meta-data/hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/placement/availability-zone",
		ContentType: "text/plain",
		Body:        "us-west-2a",
	},
	{
		Uri:         "/latest/meta-data/instance-id",
		ContentType: "text/plain",
		Body:        "i-b3ba3875",
	},
	{
		Uri:         "/latest/meta-data/instance-type",
		ContentType: "text/plain",
		Body:        "m3.2xlarge",
	},
	{
		Uri:         "/latest/meta-data/local-hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/local-ipv4",
		ContentType: "text/plain",
		Body:        "",
	},
	{
		Uri:         "/latest/meta-data/public-hostname",
		ContentType: "text/plain",
		Body:        "ec2-54-191-117-175.us-west-2.compute.amazonaws.com",
	},
	{
		Uri:         "/latest/meta-data/public-ipv4",
		ContentType: "text/plain",
		Body:        "54.191.117.175",
	},
}

// incompleteAWSImmitationsStub mimics environments where some AWS endpoints
// return empty, namely Hetzner
var incompleteAWSImmitationStubs = []endpoint{
	{
		Uri:         "/latest/meta-data/hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/instance-id",
		ContentType: "text/plain",
		Body:        "i-b3ba3875",
	},
	{
		Uri:         "/latest/meta-data/local-ipv4",
		ContentType: "text/plain",
		Body:        "",
	},
	{
		Uri:         "/latest/meta-data/public-ipv4",
		ContentType: "text/plain",
		Body:        "54.191.117.175",
	},
}
<commit_msg>fixup! tests: Add tests for EC2 Metadata immitation cases<commit_after>package fingerprint

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/stretchr/testify/require"
)

func TestEnvAWSFingerprint_nonAws(t *testing.T) {
	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = "http://127.0.0.1/latest"

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)
	require.Empty(t, response.Attributes)
}

func TestEnvAWSFingerprint_aws(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	keys := []string{
		"platform.aws.ami-id",
		"unique.platform.aws.hostname",
		"unique.platform.aws.instance-id",
		"platform.aws.instance-type",
		"unique.platform.aws.local-hostname",
		"unique.platform.aws.local-ipv4",
		"unique.platform.aws.public-hostname",
		"unique.platform.aws.public-ipv4",
		"platform.aws.placement.availability-zone",
		"unique.network.ip-address",
	}

	for _, k := range keys {
		assertNodeAttributeContains(t, response.Attributes, k)
	}

	require.NotEmpty(t, response.Links)

	// confirm we have at least instance-id and ami-id
	for _, k := range []string{"aws.ec2"} {
		assertNodeLinksContains(t, response.Links, k)
	}
}

func TestNetworkFingerprint_AWS(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	assertNodeAttributeContains(t, response.Attributes, "unique.network.ip-address")

	require.NotNil(t, response.NodeResources)
	require.Len(t, response.NodeResources.Networks, 1)

	// Test at least the first Network Resource
	net := response.NodeResources.Networks[0]
	require.NotEmpty(t, net.IP, "Expected Network Resource to have an IP")
	require.NotEmpty(t, net.CIDR, "Expected Network Resource to have a CIDR")
	require.NotEmpty(t, net.Device, "Expected Network Resource to have a Device Name")
}

func TestNetworkFingerprint_AWS_network(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	{
		node := &structs.Node{
			Attributes: make(map[string]string),
		}

		request := &FingerprintRequest{Config: &config.Config{}, Node: node}
		var response FingerprintResponse
		err := f.Fingerprint(request, &response)
		require.NoError(t, err)

		require.True(t, response.Detected, "expected response to be applicable")

		assertNodeAttributeContains(t, response.Attributes, "unique.network.ip-address")

		require.NotNil(t, response.NodeResources)
		require.Len(t, response.NodeResources.Networks, 1)

		// Test at least the first Network Resource
		net := response.NodeResources.Networks[0]
		require.NotEmpty(t, net.IP, "Expected Network Resource to have an IP")
		require.NotEmpty(t, net.CIDR, "Expected Network Resource to have a CIDR")
		require.NotEmpty(t, net.Device, "Expected Network Resource to have a Device Name")
		require.Equal(t, 1000, net.MBits)
	}

	// Try again this time setting a network speed in the config
	{
		node := &structs.Node{
			Attributes: make(map[string]string),
		}

		cfg := &config.Config{
			NetworkSpeed: 10,
		}

		request := &FingerprintRequest{Config: cfg, Node: node}
		var response FingerprintResponse
		err := f.Fingerprint(request, &response)
		require.NoError(t, err)

		assertNodeAttributeContains(t, response.Attributes, "unique.network.ip-address")

		require.NotNil(t, response.NodeResources)
		require.Len(t, response.NodeResources.Networks, 1)

		// Test at least the first Network Resource
		net := response.NodeResources.Networks[0]
		require.NotEmpty(t, net.IP, "Expected Network Resource to have an IP")
		require.NotEmpty(t, net.CIDR, "Expected Network Resource to have a CIDR")
		require.NotEmpty(t, net.Device, "Expected Network Resource to have a Device Name")
		require.Equal(t, 10, net.MBits)
	}
}

func TestNetworkFingerprint_AWS_NoNetwork(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, noNetworkAWSStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	require.True(t, response.Detected, "expected response to be applicable")

	require.Equal(t, "ami-1234", response.Attributes["platform.aws.ami-id"])

	require.Nil(t, response.NodeResources)
}

func TestNetworkFingerprint_AWS_IncompleteImitation(t *testing.T) {
	endpoint, cleanup := startFakeEC2Metadata(t, incompleteAWSImitationStubs)
	defer cleanup()

	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
	f.(*EnvAWSFingerprint).endpoint = endpoint

	node := &structs.Node{
		Attributes: make(map[string]string),
	}

	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
	var response FingerprintResponse
	err := f.Fingerprint(request, &response)
	require.NoError(t, err)

	require.False(t, response.Detected, "expected response not to be applicable")

	require.NotContains(t, response.Attributes, "platform.aws.ami-id")
	require.Nil(t, response.NodeResources)
}

/// Utility functions for tests

func startFakeEC2Metadata(t *testing.T, endpoints []endpoint) (endpoint string, cleanup func()) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for _, e := range endpoints {
			if r.RequestURI == e.Uri {
				w.Header().Set("Content-Type", e.ContentType)
				fmt.Fprintln(w, e.Body)
			}
		}
	}))

	return ts.URL + "/latest", ts.Close
}

type routes struct {
	Endpoints []*endpoint `json:"endpoints"`
}

type endpoint struct {
	Uri         string `json:"uri"`
	ContentType string `json:"content-type"`
	Body        string `json:"body"`
}

// awsStubs mimics normal EC2 instance metadata
var awsStubs = []endpoint{
	{
		Uri:         "/latest/meta-data/ami-id",
		ContentType: "text/plain",
		Body:        "ami-1234",
	},
	{
		Uri:         "/latest/meta-data/hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/placement/availability-zone",
		ContentType: "text/plain",
		Body:        "us-west-2a",
	},
	{
		Uri:         "/latest/meta-data/instance-id",
		ContentType: "text/plain",
		Body:        "i-b3ba3875",
	},
	{
		Uri:         "/latest/meta-data/instance-type",
		ContentType: "text/plain",
		Body:        "m3.2xlarge",
	},
	{
		Uri:         "/latest/meta-data/local-hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/local-ipv4",
		ContentType: "text/plain",
		Body:        "10.0.0.207",
	},
	{
		Uri:         "/latest/meta-data/public-hostname",
		ContentType: "text/plain",
		Body:        "ec2-54-191-117-175.us-west-2.compute.amazonaws.com",
	},
	{
		Uri:         "/latest/meta-data/public-ipv4",
		ContentType: "text/plain",
		Body:        "54.191.117.175",
	},
}

// noNetworkAWSStubs mimics an EC2 instance but without local ip address
// may happen in environments with odd EC2 Metadata emulation
var noNetworkAWSStubs = []endpoint{
	{
		Uri:         "/latest/meta-data/ami-id",
		ContentType: "text/plain",
		Body:        "ami-1234",
	},
	{
		Uri:         "/latest/meta-data/hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/placement/availability-zone",
		ContentType: "text/plain",
		Body:        "us-west-2a",
	},
	{
		Uri:         "/latest/meta-data/instance-id",
		ContentType: "text/plain",
		Body:        "i-b3ba3875",
	},
	{
		Uri:         "/latest/meta-data/instance-type",
		ContentType: "text/plain",
		Body:        "m3.2xlarge",
	},
	{
		Uri:         "/latest/meta-data/local-hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/local-ipv4",
		ContentType: "text/plain",
		Body:        "",
	},
	{
		Uri:         "/latest/meta-data/public-hostname",
		ContentType: "text/plain",
		Body:        "ec2-54-191-117-175.us-west-2.compute.amazonaws.com",
	},
	{
		Uri:         "/latest/meta-data/public-ipv4",
		ContentType: "text/plain",
		Body:        "54.191.117.175",
	},
}

// incompleteAWSImitationsStub mimics environments where some AWS endpoints
// return empty, namely Hetzner
var incompleteAWSImitationStubs = []endpoint{
	{
		Uri:         "/latest/meta-data/hostname",
		ContentType: "text/plain",
		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
	},
	{
		Uri:         "/latest/meta-data/instance-id",
		ContentType: "text/plain",
		Body:        "i-b3ba3875",
	},
	{
		Uri:         "/latest/meta-data/local-ipv4",
		ContentType: "text/plain",
		Body:        "",
	},
	{
		Uri:         "/latest/meta-data/public-ipv4",
		ContentType: "text/plain",
		Body:        "54.191.117.175",
	},
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"github.com/NaySoftware/go-fcm"
	"math/rand"
	"strconv"
	"time"
)

type Game struct {
	ID          string
	Name        string
	Players     []Player
	Initialized bool
	Public      bool

	Hints                int
	MaxHints             int
	Bombs                int
	Deck                 []Card
	Discard              []Card
	Piles                []int
	PileCards            []Card
	CurrentPlayerIndex   int
	CurrentPlayer        string
	State                int
	Turn                 int
	TurnsLeft            int
	CardsLeft            int
	CardsLastModified    []int
	Type                 int
	StartTime            int64
	Mode                 int
	Colors               []string
	CurrentScore         int
	HighestPossibleScore int
}

func (g *Game) Initialize(public bool, gameMode int, startingHints int, maxHints int, startingBombs int) string {
	g.State = StateNotStarted

	// validate input

	if maxHints < 1 {
		maxHints = DefaultMaxHints
	}
	if startingHints < 1 || startingHints > maxHints {
		startingHints = maxHints
	}
	if startingBombs < 1 {
		startingBombs = DefaultStartingBombs
	}

	if gameMode != ModeNormal && gameMode != ModeRainbow && gameMode != ModeWildcard && gameMode != ModeHard {
		gameMode = ModeNormal
	}

	// figure out how many cards are in the Deck
	if gameMode == ModeNormal {
		g.Colors = normalColors[:]
	} else {
		g.Colors = rainbowColors[:]
	}
	g.Mode = gameMode
	maxCards := g.MaxCards()

	// populate the Deck, Discard, and Piles
	g.Deck = make([]Card, maxCards, maxCards)
	g.PopulateDeck()
	g.Discard = make([]Card, 0, maxCards)
	g.Piles = make([]int, len(g.Colors), len(g.Colors))

	// set starting values
	g.Public = public
	g.Mode = gameMode
	g.Hints = startingHints
	g.MaxHints = maxHints
	g.Bombs = startingBombs
	g.TurnsLeft = -1
	g.Turn = 0

	// start with no Players
	g.Players = make([]Player, 0, len(cardsInHand)-1)
	g.Initialized = true
	g.HighestPossibleScore = g.GetHighestPossibleScore()
	return ""
}

func (g *Game) AddPlayer(id string, name string) string {
	if !g.Initialized {
		return "Attempting to add player before game has been fully initialized."
	}
	if g.State != StateNotStarted {
		return "Attempting to add players after game has started."
	}
	if len(g.Players) >= MaxPlayers {
		return "Attempted to add a player to a full game."
	}

	g.Players = append(g.Players, Player{GoogleID: id, Name: name})
	g.Turn++
	return ""
}

func (g *Game) Start() string {
	if !g.Initialized {
		return "Attempting to start before game has been fully Initialized."
	}
	if g.State != StateNotStarted {
		return "Attempting to start a game that has already been started."
	}

	numPlayers := len(g.Players)
	if numPlayers >= len(cardsInHand) || cardsInHand[numPlayers] == 0 {
		return "Attempted to start game with invalid number of players."
	}

	// create hands
	for index, _ := range g.Players {
		g.Players[index].Initialize(cardsInHand[numPlayers])
		for i := 0; i < cardsInHand[numPlayers]; i++ {
			err := g.Players[index].AddCard(g.DrawCard())
			if err != "" {
				return "Error initializing player's hand: " + err
			}
		}
	}

	// let's do it
	g.CurrentPlayerIndex = rand.Intn(numPlayers)
	g.CurrentPlayer = g.Players[g.CurrentPlayerIndex].GoogleID
	g.State = StateStarted
	g.StartTime = time.Now().Unix()
	g.Turn++
	g.SendCurrentPlayerNotification()

	return ""
}

func (g *Game) ProcessMove(mp *Message) string {
	m := *mp
	if g.State != StateStarted {
		return "Attempting to process a move for a non-ongoing game."
	}
	if m.Player != g.CurrentPlayer {
		return "Attempting to process a move for out-of-turn player."
	}
	p := g.GetPlayerByGoogleID(m.Player)
	if p == nil {
		return "Attempting to process a move for a nonexistent player."
	}

	// TODO: more checking that it's a valid move

	var cardsModified []int

	if m.MoveType == MovePlay {
		card, err := p.RemoveCard(m.CardIndex)
		if err != "" {
			return "Error removing card from player's hand to play: " + err
		}
		cardsModified = append(cardsModified, card.ID)
		if g.PlayCard(card) {
			// play was successful!
			mp.Result = ResultPlay
			if card.Number == 5 {
				g.Hints++
				if g.Hints > g.MaxHints {
					g.Hints = g.MaxHints
				}
			}
			if g.PilesComplete() {
				g.State = StatePerfect
			}
			p.LastMove = "played " + card.Color + " " + strconv.Itoa(card.Number)
		} else {
			// play was unsuccessful :(
			mp.Result = ResultBomb
			g.Bombs--
			if g.Bombs == 0 {
				g.State = StateBombedOut
			}
			g.Discard = append(g.Discard, card)
			p.LastMove = "bombed " + card.Color + " " + strconv.Itoa(card.Number)
		}
	} else if m.MoveType == MoveDiscard {
		card, err := p.RemoveCard(m.CardIndex)
		if err != "" {
			return "Error removing card from player's hand to discard: " + err
		}
		cardsModified = append(cardsModified, card.ID)
		g.Discard = append(g.Discard, card)
		g.Hints++
		if g.Hints > g.MaxHints {
			g.Hints = g.MaxHints
		}
		p.LastMove = "discarded " + card.Color + " " + strconv.Itoa(card.Number)
	} else if m.MoveType == MoveHint {
		if g.Hints <= 0 {
			return "Attempting to hint with no hints remaining."
		}
		hintReceiver := g.GetPlayerByGoogleID(m.HintPlayer)
		if hintReceiver == nil {
			return "Attempting to give hint to a nonexistent player."
		}
		cardsHinted, err := hintReceiver.ReceiveHint(m.CardIndex, m.HintInfoType, m.HintColor, g.Mode)
		if err != "" {
			return "Error giving hint: " + err
		}
		cardsModified = append(cardsModified, cardsHinted...)
		g.Hints--

		p.LastMove = "➡ " + hintReceiver.Name + " "
		hintedCard := hintReceiver.GetCardByID(cardsModified[0])
		if m.HintInfoType == HintNumber {
			p.LastMove += strconv.Itoa(hintedCard.Number) + "s"
		} else if (g.Mode == ModeWildcard || g.Mode == ModeHard) && hintedCard.Color == "rainbow" {
			p.LastMove += mp.HintColor + "s"
		} else {
			p.LastMove += hintedCard.Color + "s"
		}

	} else {
		return "Attempting to process unknown move type."
	}

	if m.MoveType == MovePlay || m.MoveType == MoveDiscard {
		if len(g.Deck) > 0 {
			drawnCard := g.DrawCard()
			err := p.AddCard(drawnCard)
			if err != "" {
				return "Error drawing card: " + err
			}
			cardsModified = append(cardsModified, drawnCard.ID)
		}
	}

	g.Turn++
	if g.TurnsLeft > 0 {
		g.TurnsLeft--
	}
	if g.State == StateStarted && g.TurnsLeft == 0 {
		g.State = StateDeckEmpty
	}

	if len(g.Deck) == 0 && g.TurnsLeft == -1 {
		// Deck is empty, start the countdown
		g.TurnsLeft = len(g.Players)
	}

	g.CurrentPlayerIndex = (g.CurrentPlayerIndex + 1) % len(g.Players)
	g.CurrentPlayer = g.Players[g.CurrentPlayerIndex].GoogleID
	g.SendCurrentPlayerNotification()
	g.CardsLastModified = cardsModified

	if g.State == StateStarted && !g.AnyPlayableCards() {
		g.State = StateNoPlays
	}

	g.CurrentScore = g.Score()
	g.HighestPossibleScore = g.GetHighestPossibleScore()

	return ""
}

func (g *Game) CreateState(playerid string) Game {
	p := g.GetPlayerByGoogleID(playerid)

	gCopy := Game{}
	gCopy = *g

	// clear the Deck (could be used to determine your hand)
	gCopy.CardsLeft = len(gCopy.Deck)
	gCopy.Deck = make([]Card, 0, 0)

	// clear your hand, except for revealed info
	newPlayers := make([]Player, len(g.Players), len(g.Players))
	for playerIndex, player := range gCopy.Players {
		if p.GoogleID == player.GoogleID {
			newHand := make([]Card, len(player.Cards), len(player.Cards))
			for cardIndex, card := range player.Cards {
				card.Color = ""
				card.Number = 0
				newHand[cardIndex] = card
			}
			player.Cards = newHand
		}
		newPlayers[playerIndex] = player
	}
	gCopy.Players = newPlayers
	return gCopy
}

func (g *Game) GetPlayerByGoogleID(id string) *Player {
	var p *Player
	if g.Players == nil {
		return p
	}
	for index, _ := range g.Players {
		if g.Players[index].GoogleID == id {
			return &g.Players[index]
		}
	}
	return p
}

func (g *Game) SendCurrentPlayerNotification() {
	token := g.GetPlayerByGoogleID(g.CurrentPlayer).PushToken
	if token == "" {
		return
	}

	data := map[string]string{
		"msg": "Other players are waiting! Take your turn.",
		"sum": "Fireworks - it's your turn!",
	}

	c := fcm.NewFcmClient(PushServerKey)
	c.NewFcmRegIdsMsg([]string{token}, data)

	n := fcm.NotificationPayload{}
	n.Title = "Fireworks - it's your turn!"
	n.Body = "Other players are waiting! Take your turn."
	n.ClickAction = "https://ryanschoen.com/fireworks/#!/games/" + g.ID
	n.Icon = "https://ryanschoen.com/images/icons/fireworks128.png"
	c.SetNotificationPayload(&n)

	status, err := c.Send()

	if err == nil {
		status.PrintResults()
	} else {
		fmt.Println(err)
	}
}
<commit_msg>Small push notification fixes<commit_after>package lib

import (
	"fmt"
	"github.com/NaySoftware/go-fcm"
	"math/rand"
	"strconv"
	"time"
)

type Game struct {
	ID          string
	Name        string
	Players     []Player
	Initialized bool
	Public      bool

	Hints                int
	MaxHints             int
	Bombs                int
	Deck                 []Card
	Discard              []Card
	Piles                []int
	PileCards            []Card
	CurrentPlayerIndex   int
	CurrentPlayer        string
	State                int
	Turn                 int
	TurnsLeft            int
	CardsLeft            int
	CardsLastModified    []int
	Type                 int
	StartTime            int64
	Mode                 int
	Colors               []string
	CurrentScore         int
	HighestPossibleScore int
}

func (g *Game) Initialize(public bool, gameMode int, startingHints int, maxHints int, startingBombs int) string {
	g.State = StateNotStarted

	// validate input

	if maxHints < 1 {
		maxHints = DefaultMaxHints
	}
	if startingHints < 1 || startingHints > maxHints {
		startingHints = maxHints
	}
	if startingBombs < 1 {
		startingBombs = DefaultStartingBombs
	}

	if gameMode != ModeNormal && gameMode != ModeRainbow && gameMode != ModeWildcard && gameMode != ModeHard {
		gameMode = ModeNormal
	}

	// figure out how many cards are in the Deck
	if gameMode == ModeNormal {
		g.Colors = normalColors[:]
	} else {
		g.Colors = rainbowColors[:]
	}
	g.Mode = gameMode
	maxCards := g.MaxCards()

	// populate the Deck, Discard, and Piles
	g.Deck = make([]Card, maxCards, maxCards)
	g.PopulateDeck()
	g.Discard = make([]Card, 0, maxCards)
	g.Piles = make([]int, len(g.Colors), len(g.Colors))

	// set starting values
	g.Public = public
	g.Mode = gameMode
	g.Hints = startingHints
	g.MaxHints = maxHints
	g.Bombs = startingBombs
	g.TurnsLeft = -1
	g.Turn = 0

	// start with no Players
	g.Players = make([]Player, 0, len(cardsInHand)-1)
	g.Initialized = true
	g.HighestPossibleScore = g.GetHighestPossibleScore()
	return ""
}

func (g *Game) AddPlayer(id string, name string) string {
	if !g.Initialized {
		return "Attempting to add player before game has been fully initialized."
	}
	if g.State != StateNotStarted {
		return "Attempting to add players after game has started."
	}
	if len(g.Players) >= MaxPlayers {
		return "Attempted to add a player to a full game."
	}

	g.Players = append(g.Players, Player{GoogleID: id, Name: name})
	g.Turn++
	return ""
}

func (g *Game) Start() string {
	if !g.Initialized {
		return "Attempting to start before game has been fully Initialized."
	}
	if g.State != StateNotStarted {
		return "Attempting to start a game that has already been started."
	}

	numPlayers := len(g.Players)
	if numPlayers >= len(cardsInHand) || cardsInHand[numPlayers] == 0 {
		return "Attempted to start game with invalid number of players."
	}

	// create hands
	for index, _ := range g.Players {
		g.Players[index].Initialize(cardsInHand[numPlayers])
		for i := 0; i < cardsInHand[numPlayers]; i++ {
			err := g.Players[index].AddCard(g.DrawCard())
			if err != "" {
				return "Error initializing player's hand: " + err
			}
		}
	}

	// let's do it
	g.CurrentPlayerIndex = rand.Intn(numPlayers)
	g.CurrentPlayer = g.Players[g.CurrentPlayerIndex].GoogleID
	g.State = StateStarted
	g.StartTime = time.Now().Unix()
	g.Turn++
	g.SendCurrentPlayerNotification()

	return ""
}

func (g *Game) ProcessMove(mp *Message) string {
	m := *mp
	if g.State != StateStarted {
		return "Attempting to process a move for a non-ongoing game."
	}
	if m.Player != g.CurrentPlayer {
		return "Attempting to process a move for out-of-turn player."
	}
	p := g.GetPlayerByGoogleID(m.Player)
	if p == nil {
		return "Attempting to process a move for a nonexistent player."
	}

	// TODO: more checking that it's a valid move

	var cardsModified []int

	if m.MoveType == MovePlay {
		card, err := p.RemoveCard(m.CardIndex)
		if err != "" {
			return "Error removing card from player's hand to play: " + err
		}
		cardsModified = append(cardsModified, card.ID)
		if g.PlayCard(card) {
			// play was successful!
			mp.Result = ResultPlay
			if card.Number == 5 {
				g.Hints++
				if g.Hints > g.MaxHints {
					g.Hints = g.MaxHints
				}
			}
			if g.PilesComplete() {
				g.State = StatePerfect
			}
			p.LastMove = "played " + card.Color + " " + strconv.Itoa(card.Number)
		} else {
			// play was unsuccessful :(
			mp.Result = ResultBomb
			g.Bombs--
			if g.Bombs == 0 {
				g.State = StateBombedOut
			}
			g.Discard = append(g.Discard, card)
			p.LastMove = "bombed " + card.Color + " " + strconv.Itoa(card.Number)
		}
	} else if m.MoveType == MoveDiscard {
		card, err := p.RemoveCard(m.CardIndex)
		if err != "" {
			return "Error removing card from player's hand to discard: " + err
		}
		cardsModified = append(cardsModified, card.ID)
		g.Discard = append(g.Discard, card)
		g.Hints++
		if g.Hints > g.MaxHints {
			g.Hints = g.MaxHints
		}
		p.LastMove = "discarded " + card.Color + " " + strconv.Itoa(card.Number)
	} else if m.MoveType == MoveHint {
		if g.Hints <= 0 {
			return "Attempting to hint with no hints remaining."
		}
		hintReceiver := g.GetPlayerByGoogleID(m.HintPlayer)
		if hintReceiver == nil {
			return "Attempting to give hint to a nonexistent player."
		}
		cardsHinted, err := hintReceiver.ReceiveHint(m.CardIndex, m.HintInfoType, m.HintColor, g.Mode)
		if err != "" {
			return "Error giving hint: " + err
		}
		cardsModified = append(cardsModified, cardsHinted...)
		g.Hints--

		p.LastMove = "➡ " + hintReceiver.Name + " "
		hintedCard := hintReceiver.GetCardByID(cardsModified[0])
		if m.HintInfoType == HintNumber {
			p.LastMove += strconv.Itoa(hintedCard.Number) + "s"
		} else if (g.Mode == ModeWildcard || g.Mode == ModeHard) && hintedCard.Color == "rainbow" {
			p.LastMove += mp.HintColor + "s"
		} else {
			p.LastMove += hintedCard.Color + "s"
		}

	} else {
		return "Attempting to process unknown move type."
	}

	if m.MoveType == MovePlay || m.MoveType == MoveDiscard {
		if len(g.Deck) > 0 {
			drawnCard := g.DrawCard()
			err := p.AddCard(drawnCard)
			if err != "" {
				return "Error drawing card: " + err
			}
			cardsModified = append(cardsModified, drawnCard.ID)
		}
	}

	g.Turn++
	if g.TurnsLeft > 0 {
		g.TurnsLeft--
	}
	if g.State == StateStarted && g.TurnsLeft == 0 {
		g.State = StateDeckEmpty
	}

	if len(g.Deck) == 0 && g.TurnsLeft == -1 {
		// Deck is empty, start the countdown
		g.TurnsLeft = len(g.Players)
	}

	g.CurrentPlayerIndex = (g.CurrentPlayerIndex + 1) % len(g.Players)
	g.CurrentPlayer = g.Players[g.CurrentPlayerIndex].GoogleID
	g.SendCurrentPlayerNotification()
	g.CardsLastModified = cardsModified

	if g.State == StateStarted && !g.AnyPlayableCards() {
		g.State = StateNoPlays
	}

	g.CurrentScore = g.Score()
	g.HighestPossibleScore = g.GetHighestPossibleScore()

	return ""
}

func (g *Game) CreateState(playerid string) Game {
	p := g.GetPlayerByGoogleID(playerid)

	gCopy := Game{}
	gCopy = *g

	// clear the Deck (could be used to determine your hand)
	gCopy.CardsLeft = len(gCopy.Deck)
	gCopy.Deck = make([]Card, 0, 0)

	// clear your hand, except for revealed info
	newPlayers := make([]Player, len(g.Players), len(g.Players))
	for playerIndex, player := range gCopy.Players {
		if p.GoogleID == player.GoogleID {
			newHand := make([]Card, len(player.Cards), len(player.Cards))
			for cardIndex, card := range player.Cards {
				card.Color = ""
				card.Number = 0
				newHand[cardIndex] = card
			}
			player.Cards = newHand
		}
		newPlayers[playerIndex] = player
	}
	gCopy.Players = newPlayers
	return gCopy
}

func (g *Game) GetPlayerByGoogleID(id string) *Player {
	var p *Player
	if g.Players == nil {
		return p
	}
	for index, _ := range g.Players {
		if g.Players[index].GoogleID == id {
			return &g.Players[index]
		}
	}
	return p
}

func (g *Game) SendCurrentPlayerNotification() {
	token := g.GetPlayerByGoogleID(g.CurrentPlayer).PushToken
	if token == "" {
		return
	}

	data := map[string]string{
		"msg": "Other players are waiting! Take your turn.",
		"sum": "Fireworks - it's your turn!",
	}

	c := fcm.NewFcmClient(PushServerKey)
	c.NewFcmRegIdsMsg([]string{token}, data)

	n := fcm.NotificationPayload{}
	n.Title = "Fireworks - it's your turn!"
	n.Body = "Other players are waiting! Take your turn."
	n.ClickAction = "https://ryanschoen.com/fireworks/#!/games/" + g.ID
	n.Icon = "https://ryanschoen.com/fireworks/images/icons/fireworks128.png"
	c.SetNotificationPayload(&n)

	_, err := c.Send()

	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package bloom

import (
	"hash/fnv"
	"math"
)

type BitSetProvider interface {
	Set([]uint) error
	Test([]uint) (bool, error)
}

type BloomFilter struct {
	m      uint
	k      uint
	bitSet BitSetProvider
}

// Creates a new Bloom filter for about n items with fp false positive rate
func New(n uint, fp float64, bitSet BitSetProvider) *BloomFilter {
	m, k := estimateParameters(n, fp)
	return &BloomFilter{m: m, k: k, bitSet: bitSet}
}

// Used with permission from https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go
func estimateParameters(n uint, p float64) (uint, uint) {
	m := uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2)))
	k := uint(math.Ceil(math.Log(2) * float64(m) / float64(n)))
	return m, k
}

func (f *BloomFilter) Add(data []byte) error {
	locations := f.getLocations(data)
	err := f.bitSet.Set(locations)
	if err != nil {
		return err
	}
	return nil
}

func (f *BloomFilter) Exists(data []byte) (bool, error) {
	locations := f.getLocations(data)
	isSet, err := f.bitSet.Test(locations)
	if err != nil {
		return false, err
	}
	if !isSet {
		return false, nil
	}

	return true, nil
}

func (f *BloomFilter) getLocations(data []byte) []uint {
	hashValue := getHash(data)
	locations := make([]uint, f.k)
	for i := uint(0); i < f.k; i++ {
		locations[i] = uint((hashValue * uint64(i+1)) % uint64(f.m))
	}
	return locations
}

func getHash(data []byte) uint64 {
	hasher := fnv.New64()
	hasher.Write(data)
	return hasher.Sum64()
}
<commit_msg>hash byte + k to get uniq hash<commit_after>package bloom

import (
	"hash/fnv"
	"math"
	"encoding/binary"
)

type BitSetProvider interface {
	Set([]uint) error
	Test([]uint) (bool, error)
}

type BloomFilter struct {
	m      uint
	k      uint
	bitSet BitSetProvider
}

// Creates a new Bloom filter for about n items with fp false positive rate
func New(n uint, fp float64, bitSet BitSetProvider) *BloomFilter {
	m, k := estimateParameters(n, fp)
	return &BloomFilter{m: m, k: k, bitSet: bitSet}
}

// Used with permission from https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go
func estimateParameters(n uint, p float64) (uint, uint) {
	m := uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2)))
	k := uint(math.Ceil(math.Log(2) * float64(m) / float64(n)))
	return m, k
}

func (f *BloomFilter) Add(data []byte) error {
	locations := f.getLocations(data)
	err := f.bitSet.Set(locations)
	if err != nil {
		return err
	}
	return nil
}

func (f *BloomFilter) Exists(data []byte) (bool, error) {
	locations := f.getLocations(data)
	isSet, err := f.bitSet.Test(locations)
	if err != nil {
		return false, err
	}
	if !isSet {
		return false, nil
	}

	return true, nil
}

func (f *BloomFilter) getLocations(data []byte) []uint {
	locations := make([]uint, f.k)
	for i := uint(0); i < f.k; i++ {
		hashValue := getHash(data, i)
		locations[i] = uint((hashValue * uint64(i+1)) % uint64(f.m))
	}
	return locations
}

func getHash(data []byte, k uint) uint64 {
	hasher := fnv.New64()
	hasher.Write(data)
	a := make([]byte, 4)
	binary.LittleEndian.PutUint32(a, uint32(k))
	hasher.Write(a)
	return hasher.Sum64()
}<|endoftext|>


<commit_before>package docker

import (
	"github.com/ActiveState/log"
	"logyard/clients/common"
	"logyard/clients/docker_events"
	"sync"
)

const ID_LENGTH = 12

type dockerListener struct {
	waiters map[string]chan bool
	mux     sync.Mutex
}

var DockerListener *dockerListener

func init() {
	DockerListener = new(dockerListener)
	DockerListener.waiters = make(map[string]chan bool)
}

func (l *dockerListener) BlockUntilContainerStops(id string) {
	var total int
	ch := make(chan bool)
	id = id[:ID_LENGTH]

	if len(id) != ID_LENGTH {
		common.Fatal("Invalid docker ID length: %v", len(id))
	}

	// Add a wait channel
	func() {
		l.mux.Lock()
		defer l.mux.Unlock()
		if _, ok := l.waiters[id]; ok {
			panic("already added")
		}
		l.waiters[id] = ch
		total = len(l.waiters)
	}()

	// Wait
	log.Infof("Waiting for container %v to exit (total waiters: %d)", id, total)
	<-ch
}

func (l *dockerListener) Listen() {
	for evt := range docker_events.Stream() {
		id := evt.Id[:ID_LENGTH]
		if len(id) != ID_LENGTH {
			common.Fatal("Invalid docker ID length: %v (orig: %v)", len(id), len(evt.Id))
		}

		// Notify container stop events by closing the appropriate ch.
		if !(evt.Status == "die" || evt.Status == "kill") {
			continue
		}
		l.mux.Lock()
		if ch, ok := l.waiters[id]; ok {
			close(ch)
			delete(l.waiters, evt.Id)
		}
		l.mux.Unlock()
	}
}
<commit_msg>fix close-of-closed-ch panic introduced by 5125f11d<commit_after>package docker

import (
	"github.com/ActiveState/log"
	"logyard/clients/common"
	"logyard/clients/docker_events"
	"sync"
)

const ID_LENGTH = 12

type dockerListener struct {
	waiters map[string]chan bool
	mux     sync.Mutex
}

var DockerListener *dockerListener

func init() {
	DockerListener = new(dockerListener)
	DockerListener.waiters = make(map[string]chan bool)
}

func (l *dockerListener) BlockUntilContainerStops(id string) {
	var total int
	ch := make(chan bool)
	id = id[:ID_LENGTH]

	if len(id) != ID_LENGTH {
		common.Fatal("Invalid docker ID length: %v", len(id))
	}

	// Add a wait channel
	func() {
		l.mux.Lock()
		defer l.mux.Unlock()
		if _, ok := l.waiters[id]; ok {
			panic("already added")
		}
		l.waiters[id] = ch
		total = len(l.waiters)
	}()

	// Wait
	log.Infof("Waiting for container %v to exit (total waiters: %d)", id, total)
	<-ch
}

func (l *dockerListener) Listen() {
	for evt := range docker_events.Stream() {
		id := evt.Id[:ID_LENGTH]
		if len(id) != ID_LENGTH {
			common.Fatal("Invalid docker ID length: %v (orig: %v)", len(id), len(evt.Id))
		}

		// Notify container stop events by closing the appropriate ch.
		if !(evt.Status == "die" || evt.Status == "kill") {
			continue
		}
		l.mux.Lock()
		if ch, ok := l.waiters[id]; ok {
			close(ch)
			delete(l.waiters, id)
		}
		l.mux.Unlock()
	}
}
<|endoftext|>


<commit_before>package periodic

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/heap"
    "github.com/Lupino/periodic/driver"
    "github.com/Lupino/periodic/protocol"
)

type Sched struct {
    jobTimer   *time.Timer
    grabQueue  *GrabQueue
    procQueue  map[int64]driver.Job
    revertPQ   PriorityQueue
    revTimer   *time.Timer
    entryPoint string
    JobLocker  *sync.Mutex
    stats      map[string]*FuncStat
    FuncLocker *sync.Mutex
    driver     driver.StoreDriver
    jobPQ      map[string]*PriorityQueue
    PQLocker   *sync.Mutex
    timeout    time.Duration
    alive      bool
    cacheItem  *Item
}

func NewSched(entryPoint string, store driver.StoreDriver, timeout time.Duration) *Sched {
    sched := new(Sched)
    sched.jobTimer = time.NewTimer(1 * time.Hour)
    sched.revTimer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = NewGrabQueue()
    sched.procQueue = make(map[int64]driver.Job)
    sched.revertPQ = make(PriorityQueue, 0)
    heap.Init(&sched.revertPQ)
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.PQLocker = new(sync.Mutex)
    sched.FuncLocker = new(sync.Mutex)
    sched.stats = make(map[string]*FuncStat)
    sched.driver = store
    sched.jobPQ = make(map[string]*PriorityQueue)
    sched.timeout = timeout
    sched.alive = true
    sched.cacheItem = nil
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.loadJobQueue()
    go sched.handleJobPQ()
    go sched.handleRevertPQ()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("Periodic task system started on %s\n", sched.entryPoint)
    for {
        if !sched.alive {
            break
        }
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        if sched.timeout > 0 {
            conn.SetDeadline(time.Now().Add(sched.timeout * time.Second))
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) NotifyJobTimer() {
    sched.jobTimer.Reset(time.Millisecond)
}

func (sched *Sched) NotifyRevertTimer() {
    sched.revTimer.Reset(time.Millisecond)
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch protocol.ClientType(payload[0]) {
    case protocol.TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case protocol.TYPE_WORKER:
        worker := NewWorker(sched, c)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.NotifyJobTimer()
    defer sched.NotifyRevertTimer()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if _, ok := sched.procQueue[jobId]; ok {
        delete(sched.procQueue, jobId)
    }
    job, err := sched.driver.Get(jobId)
    if err == nil {
        sched.driver.Delete(jobId)
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
        sched.removeRevertPQ(job)
    }
    return
}

func (sched *Sched) SubmitJob(grabItem GrabItem, job driver.Job) bool {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        sched.driver.Delete(job.Id)
        return true
    }
    if _, ok := sched.procQueue[job.Id]; ok {
        return true
    }

    if !grabItem.w.alive {
        return false
    }
    if err := grabItem.w.HandleDo(grabItem.msgId, job); err != nil {
        grabItem.w.alive = false
        return false
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = driver.JOB_STATUS_PROC
    job.RunAt = current
    sched.driver.Save(&job)
    sched.IncrStatProc(job)
    sched.pushRevertPQ(job)
    sched.NotifyRevertTimer()
    sched.procQueue[job.Id] = job
    sched.grabQueue.Remove(grabItem)
    return true
}

func (sched *Sched) clearCacheItem() {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    sched.cacheItem = nil
}

func (sched *Sched) lessItem() (lessItem *Item) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if sched.cacheItem != nil {
        return sched.cacheItem
    }
    maybeItem := make(map[string]*Item)
    for Func, stat := range sched.stats {
        if stat.Worker == 0 {
            continue
        }
        pq, ok := sched.jobPQ[Func]
        if !ok || pq.Len() == 0 {
            continue
        }

        item := heap.Pop(pq).(*Item)

        maybeItem[Func] = item

    }

    if len(maybeItem) == 0 {
        return nil
    }

    var lessFunc string

    for Func, item := range maybeItem {
        if lessItem == nil {
            lessItem = item
            lessFunc = Func
            continue
        }
        if lessItem.priority > item.priority {
            lessItem = item
            lessFunc = Func
        }
    }

    for Func, item := range maybeItem {
        if Func == lessFunc {
            continue
        }
        pq := sched.jobPQ[Func]
        heap.Push(pq, item)
    }
    sched.cacheItem = lessItem
    return
}

func (sched *Sched) handleJobPQ() {
    var current time.Time
    var timestamp int64
    for {
        if !sched.alive {
            break
        }
        if sched.grabQueue.Len() == 0 {
            sched.jobTimer.Reset(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        lessItem := sched.lessItem()

        if lessItem == nil {
            sched.jobTimer.Reset(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        schedJob, err := sched.driver.Get(lessItem.value)

        if err != nil {
            sched.clearCacheItem()
            log.Printf("handleJobPQ error job: %d %v\n", lessItem.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.jobTimer.Reset(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.jobTimer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                sched.pushJobPQ(schedJob)
                continue
            }
        }

        grabItem, err := sched.grabQueue.Get(schedJob.Func)
        if err == nil {
            if sched.SubmitJob(grabItem, schedJob) {
                sched.clearCacheItem()
            } else {
                sched.pushJobPQ(schedJob)
            }
        } else {
            sched.pushJobPQ(schedJob)
        }
    }
}

func (sched *Sched) handleRevertPQ() {
    var current time.Time
    var timestamp int64
    for {
        if !sched.alive {
            break
        }
        if sched.revertPQ.Len() == 0 {
            sched.revTimer.Reset(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        sched.PQLocker.Lock()
        item := heap.Pop(&sched.revertPQ).(*Item)
        sched.PQLocker.Unlock()

        if item == nil {
            sched.revTimer.Reset(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        revertJob, err := sched.driver.Get(item.value)

        if err != nil {
            log.Printf("handleRevertPQ error: job: %d %v\n", item.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if item.priority > timestamp {
            sched.revTimer.Reset(time.Second * time.Duration(item.priority - timestamp))
            current =<-sched.revTimer.C
            timestamp = int64(current.Unix())
            if item.priority > timestamp {
                sched.pushRevertPQ(revertJob)
                continue
            }
        }

        sched.DecrStatProc(revertJob)
        revertJob.Status = driver.JOB_STATUS_READY
        sched.driver.Save(&revertJob)
        sched.pushJobPQ(revertJob)
        if _, ok := sched.procQueue[revertJob.Id]; ok {
            delete(sched.procQueue, revertJob.Id)
        }
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.NotifyJobTimer()
    defer sched.NotifyRevertTimer()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if _, ok := sched.procQueue[jobId]; ok {
        delete(sched.procQueue, jobId)
    }
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = driver.JOB_STATUS_READY
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) getFuncStat(Func string) *FuncStat {
    defer sched.FuncLocker.Unlock()
    sched.FuncLocker.Lock()
    stat, ok := sched.stats[Func]
    if !ok {
        stat = new(FuncStat)
        sched.stats[Func] = stat
    }
    return stat
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Incr()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Decr()
}

func (sched *Sched) IncrStatJob(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Incr()
}

func (sched *Sched) DecrStatJob(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Decr()
}

func (sched *Sched) IncrStatProc(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == driver.JOB_STATUS_PROC {
        stat.Processing.Incr()
    }
}

func (sched *Sched) DecrStatProc(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == driver.JOB_STATUS_PROC {
        stat.Processing.Decr()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.NotifyJobTimer()
    defer sched.NotifyRevertTimer()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if _, ok := sched.procQueue[jobId]; ok {
        delete(sched.procQueue, jobId)
    }
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = driver.JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) pushJobPQ(job driver.Job) bool {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == driver.JOB_STATUS_READY {
        item := &Item{
            value: job.Id,
            priority: job.SchedAt,
        }
        if sched.cacheItem != nil && item.priority < sched.cacheItem.priority {
            if job.Id == sched.cacheItem.value {
                return true
            }
            job, _ = sched.driver.Get(sched.cacheItem.value)
            sched.cacheItem = item
            if job.Id <= 0 || job.Status != driver.JOB_STATUS_READY {
                return false
            }
        }
        pq, ok := sched.jobPQ[job.Func]
        if !ok {
            pq1 := make(PriorityQueue, 0)
            pq = &pq1
            sched.jobPQ[job.Func] = pq
            heap.Init(pq)
        }
        heap.Push(pq, item)
        return true
    }
    return false
}

func (sched *Sched) pushRevertPQ(job driver.Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == driver.JOB_STATUS_PROC && job.Timeout > 0 {
        runAt := job.RunAt
        if runAt == 0 {
            runAt = job.SchedAt
        }
        item := &Item{
            value: job.Id,
            priority: runAt + job.Timeout,
        }
        heap.Push(&sched.revertPQ, item)
    }
}

func (sched *Sched) removeRevertPQ(job driver.Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == driver.JOB_STATUS_PROC && job.Timeout > 0 {
        for _, item := range sched.revertPQ {
            if item.value == job.Id {
                heap.Remove(&sched.revertPQ, item.index)
                break
            }
        }
    }
}

func (sched *Sched) loadJobQueue() {
    updateQueue := make([]driver.Job, 0)
    removeQueue := make([]driver.Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    iter := sched.driver.NewIterator(nil)
    for {
        if !iter.Next() {
            break
        }
        job := iter.Value()
        if job.Name == "" {
            removeQueue = append(removeQueue, job)
            continue
        }
        sched.IncrStatJob(job)
        sched.pushJobPQ(job)
        runAt := job.RunAt
        if runAt < job.SchedAt {
            runAt = job.SchedAt
        }
        if runAt + job.Timeout < current {
            updateQueue = append(updateQueue, job)
        } else {
            sched.procQueue[job.Id] = job
            sched.IncrStatProc(job)
            sched.pushRevertPQ(job)
        }
    }

    iter.Close()

    for _, job := range updateQueue {
        job.Status = driver.JOB_STATUS_READY
        sched.driver.Save(&job)
    }

    for _, job := range removeQueue {
        sched.driver.Delete(job.Id)
    }
}

func (sched *Sched) Close() {
    sched.alive = false
    sched.driver.Close()
    log.Printf("Periodic task system shutdown\n")
}
<commit_msg>Fix. timer rest data race<commit_after>package periodic

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/heap"
    "github.com/Lupino/periodic/driver"
    "github.com/Lupino/periodic/protocol"
)

type Sched struct {
    jobTimer   *time.Timer
    grabQueue  *GrabQueue
    procQueue  map[int64]driver.Job
    revertPQ   PriorityQueue
    revTimer   *time.Timer
    entryPoint string
    JobLocker  *sync.Mutex
    TimerLocker *sync.Mutex
    stats      map[string]*FuncStat
    FuncLocker *sync.Mutex
    driver     driver.StoreDriver
    jobPQ      map[string]*PriorityQueue
    PQLocker   *sync.Mutex
    timeout    time.Duration
    alive      bool
    cacheItem  *Item
}

func NewSched(entryPoint string, store driver.StoreDriver, timeout time.Duration) *Sched {
    sched := new(Sched)
    sched.jobTimer = time.NewTimer(1 * time.Hour)
    sched.revTimer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = NewGrabQueue()
    sched.procQueue = make(map[int64]driver.Job)
    sched.revertPQ = make(PriorityQueue, 0)
    heap.Init(&sched.revertPQ)
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.PQLocker = new(sync.Mutex)
    sched.FuncLocker = new(sync.Mutex)
    sched.TimerLocker = new(sync.Mutex)
    sched.stats = make(map[string]*FuncStat)
    sched.driver = store
    sched.jobPQ = make(map[string]*PriorityQueue)
    sched.timeout = timeout
    sched.alive = true
    sched.cacheItem = nil
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.loadJobQueue()
    go sched.handleJobPQ()
    go sched.handleRevertPQ()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("Periodic task system started on %s\n", sched.entryPoint)
    for {
        if !sched.alive {
            break
        }
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        if sched.timeout > 0 {
            conn.SetDeadline(time.Now().Add(sched.timeout * time.Second))
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) NotifyJobTimer() {
    sched.resetJobTimer(time.Millisecond)
}

func (sched *Sched) resetJobTimer(d time.Duration) {
    defer sched.TimerLocker.Unlock()
    sched.TimerLocker.Lock()
    sched.jobTimer.Reset(d)
}

func (sched *Sched) NotifyRevertTimer() {
    sched.resetRevertTimer(time.Millisecond)
}

func (sched *Sched) resetRevertTimer(d time.Duration) {
    defer sched.TimerLocker.Unlock()
    sched.TimerLocker.Lock()
    sched.revTimer.Reset(d)
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch protocol.ClientType(payload[0]) {
    case protocol.TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case protocol.TYPE_WORKER:
        worker := NewWorker(sched, c)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.NotifyJobTimer()
    defer sched.NotifyRevertTimer()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if _, ok := sched.procQueue[jobId]; ok {
        delete(sched.procQueue, jobId)
    }
    job, err := sched.driver.Get(jobId)
    if err == nil {
        sched.driver.Delete(jobId)
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
        sched.removeRevertPQ(job)
    }
    return
}

func (sched *Sched) SubmitJob(grabItem GrabItem, job driver.Job) bool {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        sched.driver.Delete(job.Id)
        return true
    }
    if _, ok := sched.procQueue[job.Id]; ok {
        return true
    }

    if !grabItem.w.alive {
        return false
    }
    if err := grabItem.w.HandleDo(grabItem.msgId, job); err != nil {
        grabItem.w.alive = false
        return false
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = driver.JOB_STATUS_PROC
    job.RunAt = current
    sched.driver.Save(&job)
    sched.IncrStatProc(job)
    sched.pushRevertPQ(job)
    sched.NotifyRevertTimer()
    sched.procQueue[job.Id] = job
    sched.grabQueue.Remove(grabItem)
    return true
}

func (sched *Sched) clearCacheItem() {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    sched.cacheItem = nil
}

func (sched *Sched) lessItem() (lessItem *Item) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if sched.cacheItem != nil {
        return sched.cacheItem
    }
    maybeItem := make(map[string]*Item)
    for Func, stat := range sched.stats {
        if stat.Worker == 0 {
            continue
        }
        pq, ok := sched.jobPQ[Func]
        if !ok || pq.Len() == 0 {
            continue
        }

        item := heap.Pop(pq).(*Item)

        maybeItem[Func] = item

    }

    if len(maybeItem) == 0 {
        return nil
    }

    var lessFunc string

    for Func, item := range maybeItem {
        if lessItem == nil {
            lessItem = item
            lessFunc = Func
            continue
        }
        if lessItem.priority > item.priority {
            lessItem = item
            lessFunc = Func
        }
    }

    for Func, item := range maybeItem {
        if Func == lessFunc {
            continue
        }
        pq := sched.jobPQ[Func]
        heap.Push(pq, item)
    }
    sched.cacheItem = lessItem
    return
}

func (sched *Sched) handleJobPQ() {
    var current time.Time
    var timestamp int64
    for {
        if !sched.alive {
            break
        }
        if sched.grabQueue.Len() == 0 {
            sched.resetJobTimer(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        lessItem := sched.lessItem()

        if lessItem == nil {
            sched.resetJobTimer(time.Minute)
            current =<-sched.jobTimer.C
            continue
        }

        schedJob, err := sched.driver.Get(lessItem.value)

        if err != nil {
            sched.clearCacheItem()
            log.Printf("handleJobPQ error job: %d %v\n", lessItem.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.resetJobTimer(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.jobTimer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                sched.pushJobPQ(schedJob)
                continue
            }
        }

        grabItem, err := sched.grabQueue.Get(schedJob.Func)
        if err == nil {
            if sched.SubmitJob(grabItem, schedJob) {
                sched.clearCacheItem()
            } else {
                sched.pushJobPQ(schedJob)
            }
        } else {
            sched.pushJobPQ(schedJob)
        }
    }
}

func (sched *Sched) handleRevertPQ() {
    var current time.Time
    var timestamp int64
    for {
        if !sched.alive {
            break
        }
        if sched.revertPQ.Len() == 0 {
            sched.resetRevertTimer(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        sched.PQLocker.Lock()
        item := heap.Pop(&sched.revertPQ).(*Item)
        sched.PQLocker.Unlock()

        if item == nil {
            sched.resetRevertTimer(time.Minute)
            current =<-sched.revTimer.C
            continue
        }

        revertJob, err := sched.driver.Get(item.value)

        if err != nil {
            log.Printf("handleRevertPQ error: job: %d %v\n", item.value, err)
            continue
        }

        timestamp = int64(time.Now().Unix())

        if item.priority > timestamp {
            sched.resetRevertTimer(time.Second * time.Duration(item.priority - timestamp))
            current =<-sched.revTimer.C
            timestamp = int64(current.Unix())
            if item.priority > timestamp {
                sched.pushRevertPQ(revertJob)
                continue
            }
        }

        sched.DecrStatProc(revertJob)
        revertJob.Status = driver.JOB_STATUS_READY
        sched.driver.Save(&revertJob)
        sched.pushJobPQ(revertJob)
        if _, ok := sched.procQueue[revertJob.Id]; ok {
            delete(sched.procQueue, revertJob.Id)
        }
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.NotifyJobTimer()
    defer sched.NotifyRevertTimer()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if _, ok := sched.procQueue[jobId]; ok {
        delete(sched.procQueue, jobId)
    }
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = driver.JOB_STATUS_READY
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) getFuncStat(Func string) *FuncStat {
    defer sched.FuncLocker.Unlock()
    sched.FuncLocker.Lock()
    stat, ok := sched.stats[Func]
    if !ok {
        stat = new(FuncStat)
        sched.stats[Func] = stat
    }
    return stat
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Incr()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat := sched.getFuncStat(Func)
    stat.Worker.Decr()
}

func (sched *Sched) IncrStatJob(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Incr()
}

func (sched *Sched) DecrStatJob(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    stat.Job.Decr()
}

func (sched *Sched) IncrStatProc(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == driver.JOB_STATUS_PROC {
        stat.Processing.Incr()
    }
}

func (sched *Sched) DecrStatProc(job driver.Job) {
    stat := sched.getFuncStat(job.Func)
    if job.Status == driver.JOB_STATUS_PROC {
        stat.Processing.Decr()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.NotifyJobTimer()
    defer sched.NotifyRevertTimer()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if _, ok := sched.procQueue[jobId]; ok {
        delete(sched.procQueue, jobId)
    }
    job, _ := sched.driver.Get(jobId)
    sched.DecrStatProc(job)
    sched.removeRevertPQ(job)
    job.Status = driver.JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    sched.driver.Save(&job)
    sched.pushJobPQ(job)
    return
}

func (sched *Sched) pushJobPQ(job driver.Job) bool {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == driver.JOB_STATUS_READY {
        item := &Item{
            value: job.Id,
            priority: job.SchedAt,
        }
        if sched.cacheItem != nil && item.priority < sched.cacheItem.priority {
            if job.Id == sched.cacheItem.value {
                return true
            }
            job, _ = sched.driver.Get(sched.cacheItem.value)
            sched.cacheItem = item
            if job.Id <= 0 || job.Status != driver.JOB_STATUS_READY {
                return false
            }
        }
        pq, ok := sched.jobPQ[job.Func]
        if !ok {
            pq1 := make(PriorityQueue, 0)
            pq = &pq1
            sched.jobPQ[job.Func] = pq
            heap.Init(pq)
        }
        heap.Push(pq, item)
        return true
    }
    return false
}

func (sched *Sched) pushRevertPQ(job driver.Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == driver.JOB_STATUS_PROC && job.Timeout > 0 {
        runAt := job.RunAt
        if runAt == 0 {
            runAt = job.SchedAt
        }
        item := &Item{
            value: job.Id,
            priority: runAt + job.Timeout,
        }
        heap.Push(&sched.revertPQ, item)
    }
}

func (sched *Sched) removeRevertPQ(job driver.Job) {
    defer sched.PQLocker.Unlock()
    sched.PQLocker.Lock()
    if job.Status == driver.JOB_STATUS_PROC && job.Timeout > 0 {
        for _, item := range sched.revertPQ {
            if item.value == job.Id {
                heap.Remove(&sched.revertPQ, item.index)
                break
            }
        }
    }
}

func (sched *Sched) loadJobQueue() {
    updateQueue := make([]driver.Job, 0)
    removeQueue := make([]driver.Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    iter := sched.driver.NewIterator(nil)
    for {
        if !iter.Next() {
            break
        }
        job := iter.Value()
        if job.Name == "" {
            removeQueue = append(removeQueue, job)
            continue
        }
        sched.IncrStatJob(job)
        sched.pushJobPQ(job)
        runAt := job.RunAt
        if runAt < job.SchedAt {
            runAt = job.SchedAt
        }
        if runAt + job.Timeout < current {
            updateQueue = append(updateQueue, job)
        } else {
            sched.procQueue[job.Id] = job
            sched.IncrStatProc(job)
            sched.pushRevertPQ(job)
        }
    }

    iter.Close()

    for _, job := range updateQueue {
        job.Status = driver.JOB_STATUS_READY
        sched.driver.Save(&job)
    }

    for _, job := range removeQueue {
        sched.driver.Delete(job.Id)
    }
}

func (sched *Sched) Close() {
    sched.alive = false
    sched.driver.Close()
    log.Printf("Periodic task system shutdown\n")
}
<|endoftext|>


<commit_before>package main

import (
    "log"
    "net"
    "time"
    "sync"
    "container/list"
    "huabot-sched/db"
)

type Sched struct {
    workerCount int
    timer *time.Timer
    grabQueue *list.List
    jobQueue *list.List
    sockFile string
    locker   *sync.Mutex
}

func NewSched(sockFile string) *Sched {
    sched = new(Sched)
    sched.workerCount = 0
    sched.timer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = list.New()
    sched.jobQueue = list.New()
    sched.sockFile = sockFile
    sched.locker = new(sync.Mutex)
    return sched
}

func (sched *Sched) Serve() {
    sockCheck(sched.sockFile)
    sched.checkJobQueue()
    go sched.handle()
    listen, err := net.Listen("unix", sched.sockFile)
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("huabot-sched started on %s\n", sched.sockFile)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.timer.Reset(time.Millisecond)
}

func (sched *Sched) DieWorker(worker *Worker, lock... bool) {
    if len(lock) == 0 || lock[0] {
        defer sched.locker.Unlock()
        sched.locker.Lock()
    }
    defer sched.Notify()
    sched.workerCount -= 1
    log.Printf("Total worker: %d\n", sched.workerCount)
    sched.removeGrabQueue(worker)
    worker.Close()
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    worker := NewWorker(sched, Conn{Conn: conn})
    sched.workerCount += 1
    log.Printf("Total worker: %d\n", sched.workerCount)
    go worker.Handle()
}

func (sched *Sched) Done(jobId int) {
    defer sched.Notify()
    defer sched.locker.Unlock()
    sched.locker.Lock()
    removeListJob(sched.jobQueue, jobId)
    db.DelJob(jobId)
    return
}

func (sched *Sched) isDoJob(job db.Job) bool {
    now := time.Now()
    current := int(now.Unix())
    ret := false
    for e := sched.jobQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(db.Job)
        if chk.Timeout > 0 && chk.SchedAt + chk.Timeout > current {
            newJob, _ := db.GetJob(chk.Id)
            if newJob.Status == "doing" {
                newJob.Status = "ready"
                newJob.Save()
            }
            sched.jobQueue.Remove(e)
            continue
        }
        if chk.Id == job.Id {
            old := e.Value.(db.Job)
            if old.Timeout > 0 && old.SchedAt + old.Timeout < int(now.Unix()) {
                ret = false
            } else {
                ret = true
            }
        }
    }
    return ret
}

func (sched *Sched) SubmitJob(worker *Worker, job db.Job) {
    defer sched.locker.Unlock()
    sched.locker.Lock()
    if job.Name == "" {
        job.Delete()
        return
    }
    if sched.isDoJob(job) {
        return
    }
    if !worker.alive {
        return
    }
    if err := worker.HandleDo(job); err != nil {
        worker.alive = false
        sched.DieWorker(worker, false)
        return
    }
    job.Status = "doing"
    job.Save()
    sched.jobQueue.PushBack(job)
    sched.removeGrabQueue(worker)
}

func (sched *Sched) handle() {
    var current time.Time
    var timestamp int
    for {
        for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
            worker := e.Value.(*Worker)
            jobs, err := db.RangeSchedJob("ready", 0, 0)
            if err != nil || len(jobs) == 0 {
                sched.timer.Reset(time.Minute)
                current =<-sched.timer.C
                continue
            }
            timestamp = int(time.Now().Unix())
            if jobs[0].SchedAt < timestamp {
                sched.SubmitJob(worker, jobs[0])
            } else {
                sched.timer.Reset(time.Second * time.Duration(jobs[0].SchedAt - timestamp))
                current =<-sched.timer.C
                timestamp = int(current.Unix())
                if jobs[0].SchedAt <= timestamp {
                    sched.SubmitJob(worker, jobs[0])
                }
            }
        }
        if sched.grabQueue.Len() == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
        }
    }
}

func (sched *Sched) Fail(jobId int) {
    defer sched.Notify()
    defer sched.locker.Unlock()
    sched.locker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    job.Status = "ready"
    job.Save()
    return
}

func (sched *Sched) SchedLater(jobId int, delay int) {
    defer sched.Notify()
    defer sched.locker.Unlock()
    sched.locker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    job.Status = "ready"
    var now = time.Now()
    job.SchedAt = int(now.Unix()) + delay
    job.Save()
    return
}

func (sched *Sched) removeGrabQueue(worker *Worker) {
    for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
        if e.Value.(*Worker) == worker {
            sched.grabQueue.Remove(e)
        }
    }
}

func (sched *Sched) checkJobQueue() {
    start := 0
    limit := 20
    total, _ := db.CountSchedJob("doing")
    updateQueue := make([]db.Job, 0)
    removeQueue := make([]db.Job, 0)
    var now = time.Now()
    current := int(now.Unix())

    for start = 0; start < total; start += limit {
        jobs, _ := db.RangeSchedJob("doing", start, start + limit)
        for _, job := range jobs {
            if job.Name == "" {
                removeQueue = append(removeQueue, job)
                continue
            }
            if job.SchedAt + job.Timeout < current {
                updateQueue = append(updateQueue, job)
            } else {
                sched.jobQueue.PushBack(job)
            }
        }
    }

    for _, job := range updateQueue {
        job.Status = "ready"
        job.Save()
    }

    for _, job := range removeQueue {
        job.Delete()
    }
}

func (sched *Sched) Close() {
}
<commit_msg>dont lock when DieWorker<commit_after>package main

import (
    "log"
    "net"
    "time"
    "sync"
    "container/list"
    "huabot-sched/db"
)

type Sched struct {
    workerCount int
    timer *time.Timer
    grabQueue *list.List
    jobQueue *list.List
    sockFile string
    locker   *sync.Mutex
}

func NewSched(sockFile string) *Sched {
    sched = new(Sched)
    sched.workerCount = 0
    sched.timer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = list.New()
    sched.jobQueue = list.New()
    sched.sockFile = sockFile
    sched.locker = new(sync.Mutex)
    return sched
}

func (sched *Sched) Serve() {
    sockCheck(sched.sockFile)
    sched.checkJobQueue()
    go sched.handle()
    listen, err := net.Listen("unix", sched.sockFile)
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("huabot-sched started on %s\n", sched.sockFile)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.timer.Reset(time.Millisecond)
}

func (sched *Sched) DieWorker(worker *Worker) {
    defer sched.Notify()
    sched.workerCount -= 1
    log.Printf("Total worker: %d\n", sched.workerCount)
    sched.removeGrabQueue(worker)
    worker.Close()
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    worker := NewWorker(sched, Conn{Conn: conn})
    sched.workerCount += 1
    log.Printf("Total worker: %d\n", sched.workerCount)
    go worker.Handle()
}

func (sched *Sched) Done(jobId int) {
    defer sched.Notify()
    defer sched.locker.Unlock()
    sched.locker.Lock()
    removeListJob(sched.jobQueue, jobId)
    db.DelJob(jobId)
    return
}

func (sched *Sched) isDoJob(job db.Job) bool {
    now := time.Now()
    current := int(now.Unix())
    ret := false
    for e := sched.jobQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(db.Job)
        if chk.Timeout > 0 && chk.SchedAt + chk.Timeout > current {
            newJob, _ := db.GetJob(chk.Id)
            if newJob.Status == "doing" {
                newJob.Status = "ready"
                newJob.Save()
            }
            sched.jobQueue.Remove(e)
            continue
        }
        if chk.Id == job.Id {
            old := e.Value.(db.Job)
            if old.Timeout > 0 && old.SchedAt + old.Timeout < int(now.Unix()) {
                ret = false
            } else {
                ret = true
            }
        }
    }
    return ret
}

func (sched *Sched) SubmitJob(worker *Worker, job db.Job) {
    defer sched.locker.Unlock()
    sched.locker.Lock()
    if job.Name == "" {
        job.Delete()
        return
    }
    if sched.isDoJob(job) {
        return
    }
    if !worker.alive {
        return
    }
    if err := worker.HandleDo(job); err != nil {
        worker.alive = false
        sched.DieWorker(worker)
        return
    }
    job.Status = "doing"
    job.Save()
    sched.jobQueue.PushBack(job)
    sched.removeGrabQueue(worker)
}

func (sched *Sched) handle() {
    var current time.Time
    var timestamp int
    for {
        for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
            worker := e.Value.(*Worker)
            jobs, err := db.RangeSchedJob("ready", 0, 0)
            if err != nil || len(jobs) == 0 {
                sched.timer.Reset(time.Minute)
                current =<-sched.timer.C
                continue
            }
            timestamp = int(time.Now().Unix())
            if jobs[0].SchedAt < timestamp {
                sched.SubmitJob(worker, jobs[0])
            } else {
                sched.timer.Reset(time.Second * time.Duration(jobs[0].SchedAt - timestamp))
                current =<-sched.timer.C
                timestamp = int(current.Unix())
                if jobs[0].SchedAt <= timestamp {
                    sched.SubmitJob(worker, jobs[0])
                }
            }
        }
        if sched.grabQueue.Len() == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
        }
    }
}

func (sched *Sched) Fail(jobId int) {
    defer sched.Notify()
    defer sched.locker.Unlock()
    sched.locker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    job.Status = "ready"
    job.Save()
    return
}

func (sched *Sched) SchedLater(jobId int, delay int) {
    defer sched.Notify()
    defer sched.locker.Unlock()
    sched.locker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    job.Status = "ready"
    var now = time.Now()
    job.SchedAt = int(now.Unix()) + delay
    job.Save()
    return
}

func (sched *Sched) removeGrabQueue(worker *Worker) {
    for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
        if e.Value.(*Worker) == worker {
            sched.grabQueue.Remove(e)
        }
    }
}

func (sched *Sched) checkJobQueue() {
    start := 0
    limit := 20
    total, _ := db.CountSchedJob("doing")
    updateQueue := make([]db.Job, 0)
    removeQueue := make([]db.Job, 0)
    var now = time.Now()
    current := int(now.Unix())

    for start = 0; start < total; start += limit {
        jobs, _ := db.RangeSchedJob("doing", start, start + limit)
        for _, job := range jobs {
            if job.Name == "" {
                removeQueue = append(removeQueue, job)
                continue
            }
            if job.SchedAt + job.Timeout < current {
                updateQueue = append(updateQueue, job)
            } else {
                sched.jobQueue.PushBack(job)
            }
        }
    }

    for _, job := range updateQueue {
        job.Status = "ready"
        job.Save()
    }

    for _, job := range removeQueue {
        job.Delete()
    }
}

func (sched *Sched) Close() {
}
<|endoftext|>


<commit_before>package entrypoint

import (
	"context"
	"fmt"

	"github.com/datawire/ambassador/cmd/ambex"
	"github.com/datawire/ambassador/pkg/consulwatch"
	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/ambassador/pkg/snapshot/v1"
)

func makeEndpoints(ctx context.Context, ksnap *snapshot.KubernetesSnapshot, consulEndpoints map[string]consulwatch.Endpoints) *ambex.Endpoints {
	k8sServices := map[string]*kates.Service{}
	for _, svc := range ksnap.Services {
		k8sServices[key(svc)] = svc
	}

	result := map[string][]*ambex.Endpoint{}

	for _, k8sEp := range ksnap.Endpoints {
		svc, ok := k8sServices[key(k8sEp)]
		if !ok {
			continue
		}
		for _, ep := range k8sEndpointsToAmbex(k8sEp, svc) {
			result[ep.ClusterName] = append(result[ep.ClusterName], ep)
		}
	}

	for _, consulEp := range consulEndpoints {
		for _, ep := range consulEndpointsToAmbex(ctx, consulEp) {
			result[ep.ClusterName] = append(result[ep.ClusterName], ep)
		}
	}

	return &ambex.Endpoints{Entries: result}
}

func key(resource kates.Object) string {
	return fmt.Sprintf("%s:%s", resource.GetNamespace(), resource.GetName())
}

func k8sEndpointsToAmbex(ep *kates.Endpoints, svc *kates.Service) (result []*ambex.Endpoint) {
	portmap := map[string][]string{}
	for _, p := range svc.Spec.Ports {
		port := fmt.Sprintf("%d", p.Port)
		targetPort := p.TargetPort.String()
		if targetPort == "" {
			targetPort = fmt.Sprintf("%d", p.Port)
		}

		portmap[targetPort] = append(portmap[targetPort], port)
		if p.Name != "" {
			portmap[targetPort] = append(portmap[targetPort], p.Name)
			portmap[p.Name] = append(portmap[p.Name], port)
		}
		if len(svc.Spec.Ports) == 1 {
			portmap[targetPort] = append(portmap[targetPort], "")
			portmap[""] = append(portmap[""], port)
			portmap[""] = append(portmap[""], "")
		}
	}

	for _, subset := range ep.Subsets {
		for _, port := range subset.Ports {
			if port.Protocol == kates.ProtocolTCP || port.Protocol == kates.ProtocolUDP {
				portNames := map[string]bool{}
				candidates := []string{fmt.Sprintf("%d", port.Port), port.Name, ""}
				for _, c := range candidates {
					if pns, ok := portmap[c]; ok {
						for _, pn := range pns {
							portNames[pn] = true
						}
					}
				}
				for _, addr := range subset.Addresses {
					for pn := range portNames {
						sep := "/"
						if pn == "" {
							sep = ""
						}
						result = append(result, &ambex.Endpoint{
							ClusterName: fmt.Sprintf("k8s/%s/%s%s%s", ep.Namespace, ep.Name, sep, pn),
							Ip:          addr.IP,
							Port:        uint32(port.Port),
							Protocol:    string(port.Protocol),
						})
					}
				}
			}
		}
	}

	return
}

func consulEndpointsToAmbex(ctx context.Context, endpoints consulwatch.Endpoints) (result []*ambex.Endpoint) {
	for _, ep := range endpoints.Endpoints {
		result = append(result, &ambex.Endpoint{
			ClusterName: fmt.Sprintf("consul/%s/%s", endpoints.Id, endpoints.Service),
			Ip:          ep.Address,
			Port:        uint32(ep.Port),
			Protocol:    "TCP",
		})
	}

	return
}
<commit_msg>(from AES) Lookup consul addresses.<commit_after>package entrypoint

import (
	"context"
	"fmt"
	"net"

	"github.com/datawire/ambassador/cmd/ambex"
	"github.com/datawire/ambassador/pkg/consulwatch"
	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/ambassador/pkg/snapshot/v1"
	"github.com/datawire/dlib/dlog"
)

func makeEndpoints(ctx context.Context, ksnap *snapshot.KubernetesSnapshot, consulEndpoints map[string]consulwatch.Endpoints) *ambex.Endpoints {
	k8sServices := map[string]*kates.Service{}
	for _, svc := range ksnap.Services {
		k8sServices[key(svc)] = svc
	}

	result := map[string][]*ambex.Endpoint{}

	for _, k8sEp := range ksnap.Endpoints {
		svc, ok := k8sServices[key(k8sEp)]
		if !ok {
			continue
		}
		for _, ep := range k8sEndpointsToAmbex(k8sEp, svc) {
			result[ep.ClusterName] = append(result[ep.ClusterName], ep)
		}
	}

	for _, consulEp := range consulEndpoints {
		for _, ep := range consulEndpointsToAmbex(ctx, consulEp) {
			result[ep.ClusterName] = append(result[ep.ClusterName], ep)
		}
	}

	return &ambex.Endpoints{Entries: result}
}

func key(resource kates.Object) string {
	return fmt.Sprintf("%s:%s", resource.GetNamespace(), resource.GetName())
}

func k8sEndpointsToAmbex(ep *kates.Endpoints, svc *kates.Service) (result []*ambex.Endpoint) {
	portmap := map[string][]string{}
	for _, p := range svc.Spec.Ports {
		port := fmt.Sprintf("%d", p.Port)
		targetPort := p.TargetPort.String()
		if targetPort == "" {
			targetPort = fmt.Sprintf("%d", p.Port)
		}

		portmap[targetPort] = append(portmap[targetPort], port)
		if p.Name != "" {
			portmap[targetPort] = append(portmap[targetPort], p.Name)
			portmap[p.Name] = append(portmap[p.Name], port)
		}
		if len(svc.Spec.Ports) == 1 {
			portmap[targetPort] = append(portmap[targetPort], "")
			portmap[""] = append(portmap[""], port)
			portmap[""] = append(portmap[""], "")
		}
	}

	for _, subset := range ep.Subsets {
		for _, port := range subset.Ports {
			if port.Protocol == kates.ProtocolTCP || port.Protocol == kates.ProtocolUDP {
				portNames := map[string]bool{}
				candidates := []string{fmt.Sprintf("%d", port.Port), port.Name, ""}
				for _, c := range candidates {
					if pns, ok := portmap[c]; ok {
						for _, pn := range pns {
							portNames[pn] = true
						}
					}
				}
				for _, addr := range subset.Addresses {
					for pn := range portNames {
						sep := "/"
						if pn == "" {
							sep = ""
						}
						result = append(result, &ambex.Endpoint{
							ClusterName: fmt.Sprintf("k8s/%s/%s%s%s", ep.Namespace, ep.Name, sep, pn),
							Ip:          addr.IP,
							Port:        uint32(port.Port),
							Protocol:    string(port.Protocol),
						})
					}
				}
			}
		}
	}

	return
}

func consulEndpointsToAmbex(ctx context.Context, endpoints consulwatch.Endpoints) (result []*ambex.Endpoint) {
	for _, ep := range endpoints.Endpoints {
		addrs, err := net.LookupHost(ep.Address)
		if err != nil {
			dlog.Errorf(ctx, "error resolving consul address %s: %+v", ep.Address, err)
			continue
		}
		for _, addr := range addrs {
			result = append(result, &ambex.Endpoint{
				ClusterName: fmt.Sprintf("consul/%s/%s", endpoints.Id, endpoints.Service),
				Ip:          addr,
				Port:        uint32(ep.Port),
				Protocol:    "TCP",
			})
		}
	}

	return
}
<|endoftext|>