<commit_before>package rest

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestOrdersAll(t *testing.T) {
	httpDo := func(_ *http.Client, req *http.Request) (*http.Response, error) {
		msg := `
				[
					[4419360502,null,83283216761,"tIOTBTC",1508281683000,1508281731000,63938,63938,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000843,0,0,0,null,null,null,0,0,null],
					[4419354239,null,83265164211,"tIOTBTC",1508281665000,1508281674000,63976,63976,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.00008425,0,0,0,null,null,null,0,0,null],
					[4419339620,null,83217673277,"tIOTBTC",1508281618000,1508281653000,64014,64014,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000842,0,0,0,null,null,null,0,0,null]
				]`
		resp := http.Response{
			Body:       ioutil.NopCloser(bytes.NewBufferString(msg)),
			StatusCode: 200,
		}
		return &resp, nil
	}

	orders, err := NewClientWithHttpDo(httpDo).Orders.All()

	if err != nil {
		t.Error(err)
	}

	if len(orders.Snapshot) != 3 {
		t.Fatalf("expected three orders but got %d", len(orders.Snapshot))
	}
}

func TestOrdersHistory(t *testing.T) {
	httpDo := func(_ *http.Client, req *http.Request) (*http.Response, error) {
		msg := `
				[
					[4419360502,null,83283216761,"tIOTBTC",1508281683000,1508281731000,63938,63938,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000843,0,0,0,null,null,null,0,0,null],
					[4419354239,null,83265164211,"tIOTBTC",1508281665000,1508281674000,63976,63976,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.00008425,0,0,0,null,null,null,0,0,null],
					[4419339620,null,83217673277,"tIOTBTC",1508281618000,1508281653000,64014,64014,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000842,0,0,0,null,null,null,0,0,null]
				]`
		resp := http.Response{
			Body:       ioutil.NopCloser(bytes.NewBufferString(msg)),
			StatusCode: 200,
		}
		return &resp, nil
	}

	orders, err := NewClientWithHttpDo(httpDo).Orders.GetHistoryBySymbol(bitfinex.TradingPrefix + bitfinex.IOTBTC)

	if err != nil {
		t.Error(err)
	}

	if len(orders.Snapshot) != 3 {
		t.Errorf("expected three orders but got %d", len(orders.Snapshot))
	}
}

func TestCancelOrderMulti(t *testing.T) {
	t.Run("calls correct resource with correct payload", func(t *testing.T) {
		handler := func(w http.ResponseWriter, r *http.Request) {
			assert.Equal(t, "/auth/w/order/cancel/multi", r.RequestURI)
			assert.Equal(t, "POST", r.Method)

			gotReqPld := CancelOrderMultiArgs{}
			err := json.NewDecoder(r.Body).Decode(&gotReqPld)
			require.Nil(t, err)

			expectedReqPld := CancelOrderMultiArgs{
				ID:  []int{123},
				GID: []int{234},
				All: 1,
			}
			assert.Equal(t, expectedReqPld, gotReqPld)

			respMock := []interface{}{1568711312683}
			payload, _ := json.Marshal(respMock)
			_, err = w.Write(payload)
			require.Nil(t, err)
		}

		server := httptest.NewServer(http.HandlerFunc(handler))
		defer server.Close()

		c := NewClientWithURL(server.URL)
		pld := CancelOrderMultiArgs{
			ID:  []int{123},
			GID: []int{234},
			All: 1,
		}
		rsp, err := c.Orders.CancelOrderMulti(pld)
		require.Nil(t, err)

		rspFlt, err := convert.F64Slice(rsp)
		require.Nil(t, err)
		assert.Equal(t, []float64{1568711312683}, rspFlt)
	})
}

func TestCancelOrders(t *testing.T) {
	t.Run("calls correct resource with correct payload", func(t *testing.T) {
		handler := func(w http.ResponseWriter, r *http.Request) {
			assert.Equal(t, "/auth/w/order/multi", r.RequestURI)
			assert.Equal(t, "POST", r.Method)

			respMock := []interface{}{1568711312683}
			payload, _ := json.Marshal(respMock)
			_, err := w.Write(payload)
			require.Nil(t, err)
		}

		server := httptest.NewServer(http.HandlerFunc(handler))
		defer server.Close()

		c := NewClientWithURL(server.URL)
		rsp, err := c.Orders.CancelOrders([]int{1189428429, 1189428430})
		require.Nil(t, err)

		rspFlt, err := convert.F64Slice(rsp)
		require.Nil(t, err)
		assert.Equal(t, []float64{1568711312683}, rspFlt)
	})
}
<commit_msg>updating orders tests<commit_after>package rest

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestOrdersAll(t *testing.T) {
	httpDo := func(_ *http.Client, req *http.Request) (*http.Response, error) {
		msg := `
				[
					[4419360502,null,83283216761,"tIOTBTC",1508281683000,1508281731000,63938,63938,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000843,0,0,0,null,null,null,0,0,null],
					[4419354239,null,83265164211,"tIOTBTC",1508281665000,1508281674000,63976,63976,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.00008425,0,0,0,null,null,null,0,0,null],
					[4419339620,null,83217673277,"tIOTBTC",1508281618000,1508281653000,64014,64014,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000842,0,0,0,null,null,null,0,0,null]
				]`
		resp := http.Response{
			Body:       ioutil.NopCloser(bytes.NewBufferString(msg)),
			StatusCode: 200,
		}
		return &resp, nil
	}

	orders, err := NewClientWithHttpDo(httpDo).Orders.All()

	if err != nil {
		t.Error(err)
	}

	if len(orders.Snapshot) != 3 {
		t.Fatalf("expected three orders but got %d", len(orders.Snapshot))
	}
}

func TestOrdersHistory(t *testing.T) {
	httpDo := func(_ *http.Client, req *http.Request) (*http.Response, error) {
		msg := `
				[
					[4419360502,null,83283216761,"tIOTBTC",1508281683000,1508281731000,63938,63938,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000843,0,0,0,null,null,null,0,0,null],
					[4419354239,null,83265164211,"tIOTBTC",1508281665000,1508281674000,63976,63976,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.00008425,0,0,0,null,null,null,0,0,null],
					[4419339620,null,83217673277,"tIOTBTC",1508281618000,1508281653000,64014,64014,"EXCHANGE LIMIT",null,null,null,null,"CANCELED",null,null,0.0000842,0,0,0,null,null,null,0,0,null]
				]`
		resp := http.Response{
			Body:       ioutil.NopCloser(bytes.NewBufferString(msg)),
			StatusCode: 200,
		}
		return &resp, nil
	}

	orders, err := NewClientWithHttpDo(httpDo).Orders.GetHistoryBySymbol(bitfinex.TradingPrefix + bitfinex.IOTBTC)

	if err != nil {
		t.Error(err)
	}

	if len(orders.Snapshot) != 3 {
		t.Errorf("expected three orders but got %d", len(orders.Snapshot))
	}
}

func TestCancelOrderMulti(t *testing.T) {
	t.Run("calls correct resource with correct payload", func(t *testing.T) {
		handler := func(w http.ResponseWriter, r *http.Request) {
			assert.Equal(t, "/auth/w/order/cancel/multi", r.RequestURI)
			assert.Equal(t, "POST", r.Method)

			gotReqPld := CancelOrderMultiArgs{}
			err := json.NewDecoder(r.Body).Decode(&gotReqPld)
			require.Nil(t, err)

			expectedReqPld := CancelOrderMultiArgs{
				OrderIDs:      OrderIDs{123},
				GroupOrderIDs: GroupOrderIDs{234},
				All:           1,
			}
			assert.Equal(t, expectedReqPld, gotReqPld)

			respMock := []interface{}{1568711312683}
			payload, _ := json.Marshal(respMock)
			_, err = w.Write(payload)
			require.Nil(t, err)
		}

		server := httptest.NewServer(http.HandlerFunc(handler))
		defer server.Close()

		c := NewClientWithURL(server.URL)
		pld := CancelOrderMultiArgs{
			OrderIDs:      OrderIDs{123},
			GroupOrderIDs: GroupOrderIDs{234},
			All:           1,
		}
		rsp, err := c.Orders.CancelOrderMulti(pld)
		require.Nil(t, err)

		rspFlt, err := convert.F64Slice(rsp)
		require.Nil(t, err)
		assert.Equal(t, []float64{1568711312683}, rspFlt)
	})
}

func TestCancelOrders(t *testing.T) {
	t.Run("calls correct resource with correct payload", func(t *testing.T) {
		handler := func(w http.ResponseWriter, r *http.Request) {
			assert.Equal(t, "/auth/w/order/multi", r.RequestURI)
			assert.Equal(t, "POST", r.Method)

			respMock := []interface{}{1568711312683}
			payload, _ := json.Marshal(respMock)
			_, err := w.Write(payload)
			require.Nil(t, err)
		}

		server := httptest.NewServer(http.HandlerFunc(handler))
		defer server.Close()

		c := NewClientWithURL(server.URL)
		rsp, err := c.Orders.CancelOrders(OrderIDs{1189428429, 1189428430})
		require.Nil(t, err)

		rspFlt, err := convert.F64Slice(rsp)
		require.Nil(t, err)
		assert.Equal(t, []float64{1568711312683}, rspFlt)
	})
}
<|endoftext|>


<commit_before>package notifications

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/bwmarrin/discordgo"
	"github.com/globalsign/mgo/bson"
)

// _noti list
func handleList(session *discordgo.Session, msg *discordgo.Message) {
	guild, err := helpers.GetGuild(msg.GuildID)
	helpers.Relax(err)

	var entryBucket []models.NotificationsEntry
	err = helpers.MDbIter(helpers.MdbCollection(models.NotificationsTable).Find(bson.M{
		"userid":  msg.Author.ID,
		"guildid": bson.M{"$in": []string{msg.GuildID, "global"}},
	}).Sort("-triggered")).All(&entryBucket)
	helpers.Relax(err)

	if entryBucket == nil || len(entryBucket) <= 0 {
		helpers.SendMessage(msg.ChannelID, helpers.GetTextF("plugins.notifications.keyword-list-no-keywords-error", msg.Author.ID))
		return
	}

	resultMessage := fmt.Sprintf("Enabled keywords for the server: `%s`:\n", guild.Name)
	for _, entry := range entryBucket {
		resultMessage += fmt.Sprintf("`%s` (triggered `%d` times)", entry.Keyword, entry.Triggered)

		if len(entry.IgnoredGuildIDs) > 0 {
			resultMessage += " [Ignored in these Guild(s): "
			for _, ignoredGuildID := range entry.IgnoredGuildIDs {
				guild, err := helpers.GetGuildWithoutApi(ignoredGuildID)
				if err != nil {
					resultMessage += "N/A"
				} else {
					resultMessage += "`" + guild.Name + "`"
				}
				resultMessage += ", "
			}
			resultMessage = strings.TrimRight(resultMessage, ", ")
			resultMessage += "]"
		}

		if len(entry.IgnoredChannelIDs) > 0 {
			resultMessage += " [Ignored in these Channel(s): "
			for _, ignoredChannelID := range entry.IgnoredChannelIDs {
				resultMessage += "<#" + ignoredChannelID + ">, "
			}
			resultMessage = strings.TrimRight(resultMessage, ", ")
			resultMessage += "]"
		}

		if entry.GuildID == "global" {
			resultMessage += " [Global Keyword] :globe_with_meridians:"
		}

		resultMessage += "\n"
	}
	resultMessage += fmt.Sprintf("Found **%d** Keywords in total.", len(entryBucket))

	dmChannel, err := session.UserChannelCreate(msg.Author.ID)
	helpers.Relax(err)

	_, err = helpers.SendMessage(msg.ChannelID, helpers.GetTextF("bot.check-your-dms", msg.Author.ID))
	helpers.RelaxMessage(err, msg.ChannelID, msg.ID)

	_, err = helpers.SendMessage(dmChannel.ID, resultMessage)
	helpers.RelaxMessage(err, msg.ChannelID, msg.ID)
}

// _noti ignore <keyword(s)> [<#channel or channel id>]
func handleIgnore(session *discordgo.Session, content string, msg *discordgo.Message, args []string) {
	if len(args) < 2 {
		helpers.SendMessage(msg.ChannelID, helpers.GetText("bot.arguments.too-few"))
		return
	}

	session.ChannelTyping(msg.ChannelID)

	var targetChannelID string
	targetChannel, err := helpers.GetChannelFromMention(msg, args[len(args)-1])
	if err == nil {
		targetChannelID = targetChannel.ID
	}

	// trim commands
	keywords := strings.TrimSpace(strings.Replace(content, args[0], "", 1))
	if targetChannelID != "" {
		// trim target channel ID
		keywords = strings.TrimSpace(
			keywords[0:strings.LastIndex(keywords, args[len(args)-1])],
		)
	}

	added, err := ignoreKeywordInGuildOrChannel(msg.Author.ID, keywords, msg.GuildID, targetChannelID)
	if err != nil {
		switch err {
		case KeywordsNotFoundError:
			helpers.SendMessage(msg.ChannelID, helpers.GetText("plugins.notifications.keyword-ignore-not-found-error"))
			return
		}
		helpers.Relax(err)
	}

	var message string
	if added {
		if targetChannelID == "" {
			message = helpers.GetText("plugins.notifications.keyword-ignore-guild-added")
		} else {
			message = helpers.GetTextF("plugins.notifications.keyword-ignore-channel-added", targetChannel.Mention())
		}
	} else {
		if targetChannelID == "" {
			message = helpers.GetText("plugins.notifications.keyword-ignore-guild-removed")
		} else {
			message = helpers.GetTextF("plugins.notifications.keyword-ignore-channel-removed", targetChannel.Mention())
		}
	}

	_, err = helpers.SendMessage(msg.ChannelID, message)
	helpers.RelaxMessage(err, msg.ChannelID, msg.ID)
}

func ignoreKeywordInGuildOrChannel(userID, keywords, guildID, channelID string) (bool, error) {
	var added bool
	var entryBucket models.NotificationsEntry

	findArgs := bson.M{"userid": userID,
		"keyword": bson.M{"$regex": bson.RegEx{Pattern: "^" + regexp.QuoteMeta(keywords) + "$", Options: "i"}}}
	if channelID == "" {
		// ignore guild? look for global keywords
		findArgs["guildid"] = "global"
	} else {
		// ignore channels? look for global keywords, or keywords on this guild
		findArgs["$or"] = []bson.M{{"guildid": "global"}, {"guildid": guildID}}
	}

	err := helpers.MdbOne(
		helpers.MdbCollection(models.NotificationsTable).Find(findArgs),
		&entryBucket,
	)
	if err != nil {
		if helpers.IsMdbNotFound(err) {
			return added, KeywordsNotFoundError
		}
		return added, err
	}

	if channelID == "" {
		// ignore Guild
		ignoredGuildIDsWithout := make([]string, 0)
		for _, alreadyIgnoredGuildID := range entryBucket.IgnoredGuildIDs {
			if alreadyIgnoredGuildID != guildID {
				ignoredGuildIDsWithout = append(ignoredGuildIDsWithout, alreadyIgnoredGuildID)
			}
		}

		if len(ignoredGuildIDsWithout) != len(entryBucket.IgnoredGuildIDs) {
			entryBucket.IgnoredGuildIDs = ignoredGuildIDsWithout
			added = false
		} else {
			entryBucket.IgnoredGuildIDs = append(entryBucket.IgnoredGuildIDs, guildID)
			added = true
		}
	} else {
		// ignore Channel
		ignoredChannelIDsWithout := make([]string, 0)
		for _, alreadyIgnoredChannelID := range entryBucket.IgnoredChannelIDs {
			if alreadyIgnoredChannelID != channelID {
				ignoredChannelIDsWithout = append(ignoredChannelIDsWithout, alreadyIgnoredChannelID)
			}
		}

		if len(ignoredChannelIDsWithout) != len(entryBucket.IgnoredChannelIDs) {
			entryBucket.IgnoredChannelIDs = ignoredChannelIDsWithout
			added = false
		} else {
			entryBucket.IgnoredChannelIDs = append(entryBucket.IgnoredChannelIDs, channelID)
			added = true
		}
	}

	err = helpers.MDbUpdate(models.NotificationsTable, entryBucket.ID, entryBucket)
	if err != nil {
		return added, err
	}

	asyncRefresh()

	return added, nil
}
<commit_msg>[notifications] delete command message for ignoring keywords 🕵<commit_after>package notifications

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/bwmarrin/discordgo"
	"github.com/globalsign/mgo/bson"
)

// _noti list
func handleList(session *discordgo.Session, msg *discordgo.Message) {
	guild, err := helpers.GetGuild(msg.GuildID)
	helpers.Relax(err)

	var entryBucket []models.NotificationsEntry
	err = helpers.MDbIter(helpers.MdbCollection(models.NotificationsTable).Find(bson.M{
		"userid":  msg.Author.ID,
		"guildid": bson.M{"$in": []string{msg.GuildID, "global"}},
	}).Sort("-triggered")).All(&entryBucket)
	helpers.Relax(err)

	if entryBucket == nil || len(entryBucket) <= 0 {
		helpers.SendMessage(msg.ChannelID, helpers.GetTextF("plugins.notifications.keyword-list-no-keywords-error", msg.Author.ID))
		return
	}

	resultMessage := fmt.Sprintf("Enabled keywords for the server: `%s`:\n", guild.Name)
	for _, entry := range entryBucket {
		resultMessage += fmt.Sprintf("`%s` (triggered `%d` times)", entry.Keyword, entry.Triggered)

		if len(entry.IgnoredGuildIDs) > 0 {
			resultMessage += " [Ignored in these Guild(s): "
			for _, ignoredGuildID := range entry.IgnoredGuildIDs {
				guild, err := helpers.GetGuildWithoutApi(ignoredGuildID)
				if err != nil {
					resultMessage += "N/A"
				} else {
					resultMessage += "`" + guild.Name + "`"
				}
				resultMessage += ", "
			}
			resultMessage = strings.TrimRight(resultMessage, ", ")
			resultMessage += "]"
		}

		if len(entry.IgnoredChannelIDs) > 0 {
			resultMessage += " [Ignored in these Channel(s): "
			for _, ignoredChannelID := range entry.IgnoredChannelIDs {
				resultMessage += "<#" + ignoredChannelID + ">, "
			}
			resultMessage = strings.TrimRight(resultMessage, ", ")
			resultMessage += "]"
		}

		if entry.GuildID == "global" {
			resultMessage += " [Global Keyword] :globe_with_meridians:"
		}

		resultMessage += "\n"
	}
	resultMessage += fmt.Sprintf("Found **%d** Keywords in total.", len(entryBucket))

	dmChannel, err := session.UserChannelCreate(msg.Author.ID)
	helpers.Relax(err)

	_, err = helpers.SendMessage(msg.ChannelID, helpers.GetTextF("bot.check-your-dms", msg.Author.ID))
	helpers.RelaxMessage(err, msg.ChannelID, msg.ID)

	_, err = helpers.SendMessage(dmChannel.ID, resultMessage)
	helpers.RelaxMessage(err, msg.ChannelID, msg.ID)
}

// _noti ignore <keyword(s)> [<#channel or channel id>]
func handleIgnore(session *discordgo.Session, content string, msg *discordgo.Message, args []string) {
	if len(args) < 2 {
		helpers.SendMessage(msg.ChannelID, helpers.GetText("bot.arguments.too-few"))
		return
	}

	session.ChannelMessageDelete(msg.ChannelID, msg.ID)
	session.ChannelTyping(msg.ChannelID)

	var targetChannelID string
	targetChannel, err := helpers.GetChannelFromMention(msg, args[len(args)-1])
	if err == nil {
		targetChannelID = targetChannel.ID
	}

	// trim commands
	keywords := strings.TrimSpace(strings.Replace(content, args[0], "", 1))
	if targetChannelID != "" {
		// trim target channel ID
		keywords = strings.TrimSpace(
			keywords[0:strings.LastIndex(keywords, args[len(args)-1])],
		)
	}

	added, err := ignoreKeywordInGuildOrChannel(msg.Author.ID, keywords, msg.GuildID, targetChannelID)
	if err != nil {
		switch err {
		case KeywordsNotFoundError:
			helpers.SendMessage(msg.ChannelID, helpers.GetText("plugins.notifications.keyword-ignore-not-found-error"))
			return
		}
		helpers.Relax(err)
	}

	var message string
	if added {
		if targetChannelID == "" {
			message = helpers.GetText("plugins.notifications.keyword-ignore-guild-added")
		} else {
			message = helpers.GetTextF("plugins.notifications.keyword-ignore-channel-added", targetChannel.Mention())
		}
	} else {
		if targetChannelID == "" {
			message = helpers.GetText("plugins.notifications.keyword-ignore-guild-removed")
		} else {
			message = helpers.GetTextF("plugins.notifications.keyword-ignore-channel-removed", targetChannel.Mention())
		}
	}

	_, err = helpers.SendMessage(msg.ChannelID, message)
	helpers.RelaxMessage(err, msg.ChannelID, msg.ID)
}

func ignoreKeywordInGuildOrChannel(userID, keywords, guildID, channelID string) (bool, error) {
	var added bool
	var entryBucket models.NotificationsEntry

	findArgs := bson.M{"userid": userID,
		"keyword": bson.M{"$regex": bson.RegEx{Pattern: "^" + regexp.QuoteMeta(keywords) + "$", Options: "i"}}}
	if channelID == "" {
		// ignore guild? look for global keywords
		findArgs["guildid"] = "global"
	} else {
		// ignore channels? look for global keywords, or keywords on this guild
		findArgs["$or"] = []bson.M{{"guildid": "global"}, {"guildid": guildID}}
	}

	err := helpers.MdbOne(
		helpers.MdbCollection(models.NotificationsTable).Find(findArgs),
		&entryBucket,
	)
	if err != nil {
		if helpers.IsMdbNotFound(err) {
			return added, KeywordsNotFoundError
		}
		return added, err
	}

	if channelID == "" {
		// ignore Guild
		ignoredGuildIDsWithout := make([]string, 0)
		for _, alreadyIgnoredGuildID := range entryBucket.IgnoredGuildIDs {
			if alreadyIgnoredGuildID != guildID {
				ignoredGuildIDsWithout = append(ignoredGuildIDsWithout, alreadyIgnoredGuildID)
			}
		}

		if len(ignoredGuildIDsWithout) != len(entryBucket.IgnoredGuildIDs) {
			entryBucket.IgnoredGuildIDs = ignoredGuildIDsWithout
			added = false
		} else {
			entryBucket.IgnoredGuildIDs = append(entryBucket.IgnoredGuildIDs, guildID)
			added = true
		}
	} else {
		// ignore Channel
		ignoredChannelIDsWithout := make([]string, 0)
		for _, alreadyIgnoredChannelID := range entryBucket.IgnoredChannelIDs {
			if alreadyIgnoredChannelID != channelID {
				ignoredChannelIDsWithout = append(ignoredChannelIDsWithout, alreadyIgnoredChannelID)
			}
		}

		if len(ignoredChannelIDsWithout) != len(entryBucket.IgnoredChannelIDs) {
			entryBucket.IgnoredChannelIDs = ignoredChannelIDsWithout
			added = false
		} else {
			entryBucket.IgnoredChannelIDs = append(entryBucket.IgnoredChannelIDs, channelID)
			added = true
		}
	}

	err = helpers.MDbUpdate(models.NotificationsTable, entryBucket.ID, entryBucket)
	if err != nil {
		return added, err
	}

	asyncRefresh()

	return added, nil
}
<|endoftext|>


<commit_before>/* Go (cgo) interface to libgeoip */
package geoip

/*
#cgo CFLAGS: -I/opt/local/include -I/usr/local/include -I/usr/include
#cgo LDFLAGS: -lGeoIP -L/opt/local/lib -L/usr/local/lib -L/usr/lib
#include <stdio.h>
#include <errno.h>
#include <GeoIP.h>
#include <GeoIPCity.h>

//typedef GeoIP* GeoIP_pnt
*/
import "C"

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"sync"
	"unsafe"
)

type GeoIP struct {
	db *C.GeoIP
	mu sync.Mutex
}

func (gi *GeoIP) free() {
	if gi == nil {
		return
	}
	if gi.db == nil {
		gi = nil
		return
	}
	C.GeoIP_delete(gi.db)
	gi = nil
	return
}

// Opens a GeoIP database by filename, all formats supported by libgeoip are
// supported though there are only functions to access some of the databases in this API.
// The database is opened in MEMORY_CACHE mode, if you need to optimize for memory
// instead of performance you should change this.
// If you don't pass a filename, it will try opening the database from
// a list of common paths.
func Open(files ...string) (*GeoIP, error) {
	if len(files) == 0 {
		files = []string{
			"/usr/share/GeoIP/GeoIP.dat",       // Linux default
			"/usr/share/local/GeoIP/GeoIP.dat", // source install?
			"/usr/local/share/GeoIP/GeoIP.dat", // FreeBSD
			"/opt/local/share/GeoIP/GeoIP.dat", // MacPorts
			"/usr/share/GeoIP/GeoIP.dat",       // ArchLinux
		}
	}

	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	for _, file := range files {

		// libgeoip prints errors if it can't open the file, so check first
		if _, err := os.Stat(file); err != nil {
			if os.IsExist(err) {
				log.Println(err)
			}
			continue
		}

		cbase := C.CString(file)
		defer C.free(unsafe.Pointer(cbase))

		g.db, err = C.GeoIP_open(cbase, C.GEOIP_MEMORY_CACHE)
		if g.db != nil && err != nil {
			break
		}
	}
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%s): %s", files, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%s)", files)
	}

	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)
	return g, nil
}

// OpenType opens a specified GeoIP database type in the default location. Constants
// are defined for each database type (for example GEOIP_COUNTRY_EDITION).
func OpenType(dbType int) (*GeoIP, error) {
	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	g.db, err = C.GeoIP_open_type(C.int(dbType), C.GEOIP_MEMORY_CACHE)
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%d): %s", dbType, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%d)", dbType)
	}

	return g, nil
}

// Takes an IPv4 address string and returns the organization name for that IP.
// Requires the GeoIP organization database.
func (gi *GeoIP) GetOrg(ip string) string {
	name, _ := gi.GetName(ip)
	return name
}

// Works on the ASN, Netspeed, Organization and probably other
// databases, takes and IP string and returns a "name" and the
// netmask.
func (gi *GeoIP) GetName(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

type GeoIPRecord struct {
	CountryCode  string
	CountryCode3 string
	CountryName  string
	Region       string
	City         string
	PostalCode   string
	Latitude     float32
	Longitude    float32
	// DMACode       int
	AreaCode      int
	CharSet       int
	ContinentCode string
}

// Returns the "City Record" for an IP address. Requires the GeoCity(Lite)
// database - http://www.maxmind.com/en/city
func (gi *GeoIP) GetRecord(ip string) *GeoIPRecord {
	if gi.db == nil {
		return nil
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	record := C.GeoIP_record_by_addr(gi.db, cip)
	if record == nil {
		return nil
	}
	// defer C.free(unsafe.Pointer(record))
	defer C.GeoIPRecord_delete(record)
	rec := new(GeoIPRecord)
	rec.CountryCode = C.GoString(record.country_code)
	rec.CountryCode3 = C.GoString(record.country_code3)
	rec.CountryName = C.GoString(record.country_name)
	rec.Region = C.GoString(record.region)
	rec.City = C.GoString(record.city)
	rec.PostalCode = C.GoString(record.postal_code)
	rec.Latitude = float32(record.latitude)
	rec.Longitude = float32(record.longitude)
	rec.AreaCode = int(record.area_code)
	rec.CharSet = int(record.charset)
	rec.ContinentCode = C.GoString(record.continent_code)

	return rec
}

// Returns the country code and region code for an IP address. Requires
// the GeoIP Region database.
func (gi *GeoIP) GetRegion(ip string) (string, string) {
	if gi.db == nil {
		return "", ""
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	region := C.GeoIP_region_by_addr(gi.db, cip)
	if region == nil {
		return "", ""
	}

	countryCode := C.GoString(&region.country_code[0])
	regionCode := C.GoString(&region.region[0])
	defer C.free(unsafe.Pointer(region))

	return countryCode, regionCode
}

// Same as GetName() but for IPv6 addresses.
func (gi *GeoIP) GetNameV6(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr_v6(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// Takes an IPv4 address string and returns the country code for that IP
// and the netmask for that IP range.
func (gi *GeoIP) GetCountry(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock() // Lock to make sure we get the right result from GeoIP_last_netmask
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr(gi.db, cip)

	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// GetCountry_v6 works the same as GetCountry except for IPv6 addresses, be sure to
// load a database with IPv6 data to get any results.
func (gi *GeoIP) GetCountry_v6(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr_v6(gi.db, cip)
	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}
<commit_msg>Set charset when using OpenType()<commit_after>/* Go (cgo) interface to libgeoip */
package geoip

/*
#cgo CFLAGS: -I/opt/local/include -I/usr/local/include -I/usr/include
#cgo LDFLAGS: -lGeoIP -L/opt/local/lib -L/usr/local/lib -L/usr/lib
#include <stdio.h>
#include <errno.h>
#include <GeoIP.h>
#include <GeoIPCity.h>

//typedef GeoIP* GeoIP_pnt
*/
import "C"

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"sync"
	"unsafe"
)

type GeoIP struct {
	db *C.GeoIP
	mu sync.Mutex
}

func (gi *GeoIP) free() {
	if gi == nil {
		return
	}
	if gi.db == nil {
		gi = nil
		return
	}
	C.GeoIP_delete(gi.db)
	gi = nil
	return
}

// Opens a GeoIP database by filename, all formats supported by libgeoip are
// supported though there are only functions to access some of the databases in this API.
// The database is opened in MEMORY_CACHE mode, if you need to optimize for memory
// instead of performance you should change this.
// If you don't pass a filename, it will try opening the database from
// a list of common paths.
func Open(files ...string) (*GeoIP, error) {
	if len(files) == 0 {
		files = []string{
			"/usr/share/GeoIP/GeoIP.dat",       // Linux default
			"/usr/share/local/GeoIP/GeoIP.dat", // source install?
			"/usr/local/share/GeoIP/GeoIP.dat", // FreeBSD
			"/opt/local/share/GeoIP/GeoIP.dat", // MacPorts
			"/usr/share/GeoIP/GeoIP.dat",       // ArchLinux
		}
	}

	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	for _, file := range files {

		// libgeoip prints errors if it can't open the file, so check first
		if _, err := os.Stat(file); err != nil {
			if os.IsExist(err) {
				log.Println(err)
			}
			continue
		}

		cbase := C.CString(file)
		defer C.free(unsafe.Pointer(cbase))

		g.db, err = C.GeoIP_open(cbase, C.GEOIP_MEMORY_CACHE)
		if g.db != nil && err != nil {
			break
		}
	}
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%s): %s", files, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%s)", files)
	}

	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)
	return g, nil
}

// OpenType opens a specified GeoIP database type in the default location. Constants
// are defined for each database type (for example GEOIP_COUNTRY_EDITION).
func OpenType(dbType int) (*GeoIP, error) {
	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	g.db, err = C.GeoIP_open_type(C.int(dbType), C.GEOIP_MEMORY_CACHE)
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%d): %s", dbType, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%d)", dbType)
	}

	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)

	return g, nil
}

// Takes an IPv4 address string and returns the organization name for that IP.
// Requires the GeoIP organization database.
func (gi *GeoIP) GetOrg(ip string) string {
	name, _ := gi.GetName(ip)
	return name
}

// Works on the ASN, Netspeed, Organization and probably other
// databases, takes and IP string and returns a "name" and the
// netmask.
func (gi *GeoIP) GetName(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

type GeoIPRecord struct {
	CountryCode  string
	CountryCode3 string
	CountryName  string
	Region       string
	City         string
	PostalCode   string
	Latitude     float32
	Longitude    float32
	// DMACode       int
	AreaCode      int
	CharSet       int
	ContinentCode string
}

// Returns the "City Record" for an IP address. Requires the GeoCity(Lite)
// database - http://www.maxmind.com/en/city
func (gi *GeoIP) GetRecord(ip string) *GeoIPRecord {
	if gi.db == nil {
		return nil
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	record := C.GeoIP_record_by_addr(gi.db, cip)
	if record == nil {
		return nil
	}
	// defer C.free(unsafe.Pointer(record))
	defer C.GeoIPRecord_delete(record)
	rec := new(GeoIPRecord)
	rec.CountryCode = C.GoString(record.country_code)
	rec.CountryCode3 = C.GoString(record.country_code3)
	rec.CountryName = C.GoString(record.country_name)
	rec.Region = C.GoString(record.region)
	rec.City = C.GoString(record.city)
	rec.PostalCode = C.GoString(record.postal_code)
	rec.Latitude = float32(record.latitude)
	rec.Longitude = float32(record.longitude)
	rec.AreaCode = int(record.area_code)
	rec.CharSet = int(record.charset)
	rec.ContinentCode = C.GoString(record.continent_code)

	return rec
}

// Returns the country code and region code for an IP address. Requires
// the GeoIP Region database.
func (gi *GeoIP) GetRegion(ip string) (string, string) {
	if gi.db == nil {
		return "", ""
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	region := C.GeoIP_region_by_addr(gi.db, cip)
	if region == nil {
		return "", ""
	}

	countryCode := C.GoString(&region.country_code[0])
	regionCode := C.GoString(&region.region[0])
	defer C.free(unsafe.Pointer(region))

	return countryCode, regionCode
}

// Same as GetName() but for IPv6 addresses.
func (gi *GeoIP) GetNameV6(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr_v6(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// Takes an IPv4 address string and returns the country code for that IP
// and the netmask for that IP range.
func (gi *GeoIP) GetCountry(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock() // Lock to make sure we get the right result from GeoIP_last_netmask
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr(gi.db, cip)

	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// GetCountry_v6 works the same as GetCountry except for IPv6 addresses, be sure to
// load a database with IPv6 data to get any results.
func (gi *GeoIP) GetCountry_v6(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr_v6(gi.db, cip)
	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}
<|endoftext|>


<commit_before>package context

import (
	"time"
	"database/sql"
	"fmt"
	"log"
	"sync"

	"github.com/bluedevel/mosel/api"
	"github.com/bluedevel/mosel/commons"
)

type table struct {
	name        string
	createQuery string
}

type dbState struct {
	nodes map[string]*dbNodeState
}

type dbNodeState struct {
	id       int
	name     string
	diagrams map[string]*dbDiagramState
}

type dbDiagramState struct {
	id     int
	name   string
	graphs map[string]*dbGraphState
}

type dbGraphState struct {
	id   int
	name string
}

type dbResult struct {
	value     string
	timestamp int64
	graphId   int
	graph     string
	diagramId int
	diagram   string
	nodeId    int
	node      string
	url       string
}

type sqlDataPersistence struct {
	db            *sql.DB
	q             commons.SqlQueries
	serverContext *MoseldServerContext

	dbLock        sync.RWMutex
	dbState       *dbState
}

func NewSqlDataPersistence(db *sql.DB, queries commons.SqlQueries) *sqlDataPersistence {
	return &sqlDataPersistence{
		db:db,
		q:queries,
	}
}

func (pers *sqlDataPersistence) query(name string, args ...interface{}) (*sql.Rows, error) {
	query, exists := pers.q[name]

	if !exists {
		return nil, fmt.Errorf("Quers %s is not registered", name)
	}

	rows, err := pers.db.Query(query, args...)
	if err != nil {
		return nil, err
	}

	cols, err := rows.Columns()
	if err != nil {
		return nil, err
	}

	if len(cols) == 0 {
		// nothing to scan here
		rows.Close()
	}

	return rows, err
}

func (pers *sqlDataPersistence) queryResultNotEmpty(name string, args ...interface{}) (bool, error) {
	rows, err := pers.query(name, args...)
	defer rows.Close()

	if err != nil {
		return false, err
	}

	return !rows.Next(), nil
}

func (pers *sqlDataPersistence) tableExists(name string) (bool, error) {
	//todo be clever bout this
	rows, err := pers.db.Query(pers.q["tableExists"] + " '" + name + "'")
	defer rows.Close()
	if err != nil {
		return false, err
	}

	return rows.Next(), nil
}

func (pers *sqlDataPersistence) lastInsertedId() (int, error) {
	rows, err := pers.query("lastInsertedId")
	defer rows.Close()
	if err != nil {
		return -1, err
	}

	if !rows.Next() {
		return -1, fmt.Errorf("Last instered id couldn't be found")
	}

	var id int
	rows.Scan(&id)
	return id, nil
}

func (pers *sqlDataPersistence) Init() error {
	tables := make([]table, 4)
	tables[0] = table{name:"Nodes", createQuery:"createNodes", }
	tables[1] = table{name:"Diagrams", createQuery:"createDiagrams", }
	tables[2] = table{name:"Graphs", createQuery:"createGraphs", }
	tables[3] = table{name:"DataPoints", createQuery:"createDataPoints", }

	for _, table := range tables {
		if exists, err := pers.tableExists(table.name); err != nil {
			return err
		} else if !exists {
			log.Printf("Create table %s ", table)
			_, err := pers.query(table.createQuery)

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (pers *sqlDataPersistence) Add(node string, t time.Time, info api.NodeInfo) {
	pers.dbLock.Lock()
	defer pers.dbLock.Unlock()

	if pers.dbState == nil {
		log.Println("No database state initialized")
		return
	}

	nodeState, ok := pers.dbState.nodes[node]

	if !ok {
		if _, err := pers.query("insertNode", node, ""); err != nil {
			log.Println(err)
			return
		}

		// update the state. dirty but what the heck
		pers.getAll()
		nodeState = pers.dbState.nodes[node]
	}

	for diagram, graphs := range info {
		diagramState, ok := nodeState.diagrams[diagram]

		if !ok {
			if _, err := pers.query("insertDiagram", diagram, nodeState.id); err != nil {
				log.Println(err)
				return
			}

			// update the state. dirty but what the heck
			pers.getAll()
			diagramState = pers.dbState.nodes[node].diagrams[diagram]
		}

		for graph, value := range graphs {
			graphState, ok := diagramState.graphs[graph]

			if !ok {
				if _, err := pers.query("insertGraph", graph, diagramState.id); err != nil {
					log.Println(err)
					return
				}

				// update the state. dirty but what the heck
				pers.getAll()
				graphState = pers.dbState.nodes[node].diagrams[diagram].graphs[graph]
			}

			if _, err := pers.query("insertDataPoint", value, t.Round(time.Second).Unix(), graphState.id); err != nil {
				log.Println(err)
				return
			}
		}
	}
}

func (pers *sqlDataPersistence) GetAll() (DataCacheStorage, error) {
	pers.dbLock.RLock()
	defer pers.dbLock.RUnlock()
	return pers.getAll()
}

func (pers *sqlDataPersistence) getAll() (DataCacheStorage, error) {
	res := make(DataCacheStorage)

	rows, err := pers.query("all")

	if err != nil {
		return res, err
	}

	defer rows.Close()
	return pers.get(rows)
}

func (pers *sqlDataPersistence) GetAllSince(since time.Duration) (DataCacheStorage, error) {
	res := make(DataCacheStorage)

	pers.dbLock.RLock()
	defer pers.dbLock.RUnlock()

	t := time.Now().Add(-since)
	rows, err := pers.query("allSince", t)

	if err != nil {
		return res, err
	}

	defer rows.Close()
	return pers.get(rows)
}

func (pers *sqlDataPersistence) get(rows *sql.Rows) (DataCacheStorage, error) {
	res := make(DataCacheStorage)

	// dirty force reset
	pers.dbState = &dbState{
		nodes:make(map[string]*dbNodeState),
	}

	for rows.Next() {
		var dbRes dbResult
		err := rows.Scan(
			&dbRes.value,
			&dbRes.timestamp,
			&dbRes.graphId,
			&dbRes.graph,
			&dbRes.diagramId,
			&dbRes.diagram,
			&dbRes.nodeId,
			&dbRes.node,
			&dbRes.url)

		if err != nil {
			return nil, err
		}
		pers.updateDbState(dbRes)

		//parse db result
		node, ok := res[dbRes.node]
		if !ok {
			node = make(map[time.Time]DataPoint)
			res[dbRes.node] = node
		}

		t := time.Unix(dbRes.timestamp, 0)
		point, ok := node[t]
		if !ok {
			point = DataPoint{
				Time:t,
				Info:make(api.NodeInfo),
			}
			node[t] = point
		}

		diagram, ok := point.Info[dbRes.diagram]
		if !ok {
			diagram = make(map[string]string)
			point.Info[dbRes.diagram] = diagram
		}

		diagram[dbRes.graph] = dbRes.value
	}

	return res, nil
}

func (pers *sqlDataPersistence) updateDbState(dbRes dbResult) {
	if dbRes.nodeId == -1 {
		return
	}

	node, ok := pers.dbState.nodes[dbRes.node]
	if !ok {
		node = &dbNodeState{
			id:dbRes.nodeId,
			name:dbRes.node,
			diagrams:make(map[string]*dbDiagramState),
		}
		pers.dbState.nodes[dbRes.node] = node
	}

	if dbRes.diagramId == -1 {
		return
	}

	diagram, ok := node.diagrams[dbRes.diagram]
	if !ok {
		diagram = &dbDiagramState{
			id:dbRes.diagramId,
			name:dbRes.diagram,
			graphs:make(map[string]*dbGraphState),
		}
		node.diagrams[dbRes.diagram] = diagram
	}

	if dbRes.graphId == -1 {
		return
	}

	graph, ok := diagram.graphs[dbRes.graph]
	if !ok {
		graph = &dbGraphState{
			id:dbRes.graphId,
			name:dbRes.graph,
		}
		diagram.graphs[dbRes.graph] = graph
	}
}<commit_msg>remove unused method in mysql persistence<commit_after>package context

import (
	"time"
	"database/sql"
	"fmt"
	"log"
	"sync"

	"github.com/bluedevel/mosel/api"
	"github.com/bluedevel/mosel/commons"
)

type table struct {
	name        string
	createQuery string
}

type dbState struct {
	nodes map[string]*dbNodeState
}

type dbNodeState struct {
	id       int
	name     string
	diagrams map[string]*dbDiagramState
}

type dbDiagramState struct {
	id     int
	name   string
	graphs map[string]*dbGraphState
}

type dbGraphState struct {
	id   int
	name string
}

type dbResult struct {
	value     string
	timestamp int64
	graphId   int
	graph     string
	diagramId int
	diagram   string
	nodeId    int
	node      string
	url       string
}

type sqlDataPersistence struct {
	db            *sql.DB
	q             commons.SqlQueries
	serverContext *MoseldServerContext

	dbLock        sync.RWMutex
	dbState       *dbState
}

func NewSqlDataPersistence(db *sql.DB, queries commons.SqlQueries) *sqlDataPersistence {
	return &sqlDataPersistence{
		db:db,
		q:queries,
	}
}

func (pers *sqlDataPersistence) query(name string, args ...interface{}) (*sql.Rows, error) {
	query, exists := pers.q[name]

	if !exists {
		return nil, fmt.Errorf("Quers %s is not registered", name)
	}

	rows, err := pers.db.Query(query, args...)
	if err != nil {
		return nil, err
	}

	cols, err := rows.Columns()
	if err != nil {
		return nil, err
	}

	if len(cols) == 0 {
		// nothing to scan here
		rows.Close()
	}

	return rows, err
}

func (pers *sqlDataPersistence) queryResultNotEmpty(name string, args ...interface{}) (bool, error) {
	rows, err := pers.query(name, args...)
	defer rows.Close()

	if err != nil {
		return false, err
	}

	return !rows.Next(), nil
}

func (pers *sqlDataPersistence) tableExists(name string) (bool, error) {
	//todo be clever bout this
	rows, err := pers.db.Query(pers.q["tableExists"] + " '" + name + "'")
	defer rows.Close()
	if err != nil {
		return false, err
	}

	return rows.Next(), nil
}

func (pers *sqlDataPersistence) Init() error {
	tables := make([]table, 4)
	tables[0] = table{name:"Nodes", createQuery:"createNodes", }
	tables[1] = table{name:"Diagrams", createQuery:"createDiagrams", }
	tables[2] = table{name:"Graphs", createQuery:"createGraphs", }
	tables[3] = table{name:"DataPoints", createQuery:"createDataPoints", }

	for _, table := range tables {
		if exists, err := pers.tableExists(table.name); err != nil {
			return err
		} else if !exists {
			log.Printf("Create table %s ", table.name)
			_, err := pers.query(table.createQuery)

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (pers *sqlDataPersistence) Add(node string, t time.Time, info api.NodeInfo) {
	pers.dbLock.Lock()
	defer pers.dbLock.Unlock()

	if pers.dbState == nil {
		log.Println("No database state initialized")
		return
	}

	nodeState, ok := pers.dbState.nodes[node]

	if !ok {
		if _, err := pers.query("insertNode", node, ""); err != nil {
			log.Println(err)
			return
		}

		// update the state. dirty but what the heck
		pers.getAll()
		nodeState = pers.dbState.nodes[node]
	}

	for diagram, graphs := range info {
		diagramState, ok := nodeState.diagrams[diagram]

		if !ok {
			if _, err := pers.query("insertDiagram", diagram, nodeState.id); err != nil {
				log.Println(err)
				return
			}

			// update the state. dirty but what the heck
			pers.getAll()
			diagramState = pers.dbState.nodes[node].diagrams[diagram]
		}

		for graph, value := range graphs {
			graphState, ok := diagramState.graphs[graph]

			if !ok {
				if _, err := pers.query("insertGraph", graph, diagramState.id); err != nil {
					log.Println(err)
					return
				}

				// update the state. dirty but what the heck
				pers.getAll()
				graphState = pers.dbState.nodes[node].diagrams[diagram].graphs[graph]
			}

			if _, err := pers.query("insertDataPoint", value, t.Round(time.Second).Unix(), graphState.id); err != nil {
				log.Println(err)
				return
			}
		}
	}
}

func (pers *sqlDataPersistence) GetAll() (DataCacheStorage, error) {
	pers.dbLock.RLock()
	defer pers.dbLock.RUnlock()
	return pers.getAll()
}

func (pers *sqlDataPersistence) getAll() (DataCacheStorage, error) {
	res := make(DataCacheStorage)

	rows, err := pers.query("all")

	if err != nil {
		return res, err
	}

	defer rows.Close()
	return pers.get(rows)
}

func (pers *sqlDataPersistence) GetAllSince(since time.Duration) (DataCacheStorage, error) {
	res := make(DataCacheStorage)

	pers.dbLock.RLock()
	defer pers.dbLock.RUnlock()

	t := time.Now().Add(-since)
	rows, err := pers.query("allSince", t)

	if err != nil {
		return res, err
	}

	defer rows.Close()
	return pers.get(rows)
}

func (pers *sqlDataPersistence) get(rows *sql.Rows) (DataCacheStorage, error) {
	res := make(DataCacheStorage)

	// dirty force reset
	pers.dbState = &dbState{
		nodes:make(map[string]*dbNodeState),
	}

	for rows.Next() {
		var dbRes dbResult
		err := rows.Scan(
			&dbRes.value,
			&dbRes.timestamp,
			&dbRes.graphId,
			&dbRes.graph,
			&dbRes.diagramId,
			&dbRes.diagram,
			&dbRes.nodeId,
			&dbRes.node,
			&dbRes.url)

		if err != nil {
			return nil, err
		}
		pers.updateDbState(dbRes)

		//parse db result
		node, ok := res[dbRes.node]
		if !ok {
			node = make(map[time.Time]DataPoint)
			res[dbRes.node] = node
		}

		t := time.Unix(dbRes.timestamp, 0)
		point, ok := node[t]
		if !ok {
			point = DataPoint{
				Time:t,
				Info:make(api.NodeInfo),
			}
			node[t] = point
		}

		diagram, ok := point.Info[dbRes.diagram]
		if !ok {
			diagram = make(map[string]string)
			point.Info[dbRes.diagram] = diagram
		}

		diagram[dbRes.graph] = dbRes.value
	}

	return res, nil
}

func (pers *sqlDataPersistence) updateDbState(dbRes dbResult) {
	if dbRes.nodeId == -1 {
		return
	}

	node, ok := pers.dbState.nodes[dbRes.node]
	if !ok {
		node = &dbNodeState{
			id:dbRes.nodeId,
			name:dbRes.node,
			diagrams:make(map[string]*dbDiagramState),
		}
		pers.dbState.nodes[dbRes.node] = node
	}

	if dbRes.diagramId == -1 {
		return
	}

	diagram, ok := node.diagrams[dbRes.diagram]
	if !ok {
		diagram = &dbDiagramState{
			id:dbRes.diagramId,
			name:dbRes.diagram,
			graphs:make(map[string]*dbGraphState),
		}
		node.diagrams[dbRes.diagram] = diagram
	}

	if dbRes.graphId == -1 {
		return
	}

	graph, ok := diagram.graphs[dbRes.graph]
	if !ok {
		graph = &dbGraphState{
			id:dbRes.graphId,
			name:dbRes.graph,
		}
		diagram.graphs[dbRes.graph] = graph
	}
}<|endoftext|>


<commit_before>package brewerydb

import (
	"fmt"
	"net/http"
)

// GlassService provides access to the BreweryDB Glassware API.
// Use Client.Glass.
type GlassService struct {
	c *Client
}

// Glass represents a Glass assigned to a UPC code.
type Glass struct {
	ID          int
	Name        string
	Description string
	CreateDate  string
	UpdateDate  string
}

// List returns a list of Glasses.
func (gs *GlassService) List() (gl []Glass, err error) {
	// GET: /glassware
	var req *http.Request
	req, err = gs.c.NewRequest("GET", "/glassware", nil)
	if err != nil {
		return
	}

	resp := struct {
		Status  string
		Data    []Glass
		Message string
	}{}
	err = gs.c.Do(req, &resp)
	return resp.Data, err
}

// Get returns the Glass with the given Glass ID.
func (gs *GlassService) Get(id int) (g Glass, err error) {
	// GET: /glass/:glassId
	var req *http.Request
	req, err = gs.c.NewRequest("GET", fmt.Sprintf("/glass/%d", id), nil)
	if err != nil {
		return
	}

	resp := struct {
		Status  string
		Data    Glass
		Message string
	}{}
	err = gs.c.Do(req, &resp)
	return resp.Data, err
}
<commit_msg>api links for glass.go<commit_after>package brewerydb

import (
	"fmt"
	"net/http"
)

// GlassService provides access to the BreweryDB Glassware API.
// Use Client.Glass.
//
// See: http://www.brewerydb.com/developers/docs-endpoint/glass_index
type GlassService struct {
	c *Client
}

// Glass represents a Glass assigned to a UPC code.
type Glass struct {
	ID          int
	Name        string
	Description string
	CreateDate  string
	UpdateDate  string
}

// List returns a list of Glasses.
//
// See: http://www.brewerydb.com/developers/docs-endpoint/glass_index#1
func (gs *GlassService) List() (gl []Glass, err error) {
	// GET: /glassware
	var req *http.Request
	req, err = gs.c.NewRequest("GET", "/glassware", nil)
	if err != nil {
		return
	}

	resp := struct {
		Status  string
		Data    []Glass
		Message string
	}{}
	err = gs.c.Do(req, &resp)
	return resp.Data, err
}

// Get returns the Glass with the given Glass ID.
//
// See: http://www.brewerydb.com/developers/docs-endpoint/glass_index#2
func (gs *GlassService) Get(id int) (g Glass, err error) {
	// GET: /glass/:glassId
	var req *http.Request
	req, err = gs.c.NewRequest("GET", fmt.Sprintf("/glass/%d", id), nil)
	if err != nil {
		return
	}

	resp := struct {
		Status  string
		Data    Glass
		Message string
	}{}
	err = gs.c.Do(req, &resp)
	return resp.Data, err
}
<|endoftext|>


<commit_before>package wdclient

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"math/rand"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

type MasterClient struct {
	FilerGroup     string
	clientType     string
	clientHost     pb.ServerAddress
	currentMaster  pb.ServerAddress
	masters        map[string]pb.ServerAddress
	grpcDialOption grpc.DialOption

	vidMap

	OnPeerUpdate func(update *master_pb.ClusterNodeUpdate, startFrom time.Time)
}

func NewMasterClient(grpcDialOption grpc.DialOption, filerGroup string, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters map[string]pb.ServerAddress) *MasterClient {
	return &MasterClient{
		FilerGroup:     filerGroup,
		clientType:     clientType,
		clientHost:     clientHost,
		masters:        masters,
		grpcDialOption: grpcDialOption,
		vidMap:         newVidMap(clientDataCenter),
	}
}

func (mc *MasterClient) GetMaster() pb.ServerAddress {
	mc.WaitUntilConnected()
	return mc.currentMaster
}

func (mc *MasterClient) GetMasters() map[string]pb.ServerAddress {
	mc.WaitUntilConnected()
	return mc.masters
}

func (mc *MasterClient) WaitUntilConnected() {
	for mc.currentMaster == "" {
		time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond)
	}
}

func (mc *MasterClient) KeepConnectedToMaster() {
	glog.V(1).Infof("%s.%s masterClient bootstraps with masters %v", mc.FilerGroup, mc.clientType, mc.masters)
	for {
		mc.tryAllMasters()
		time.Sleep(time.Second)
	}
}

func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddress) (leader string) {
	for _, master := range mc.masters {
		if master == myMasterAddress {
			continue
		}
		if grpcErr := pb.WithMasterClient(false, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
			defer cancel()
			resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
			if err != nil {
				return err
			}
			leader = resp.Leader
			return nil
		}); grpcErr != nil {
			glog.V(0).Infof("connect to %s: %v", master, grpcErr)
		}
		if leader != "" {
			glog.V(0).Infof("existing leader is %s", leader)
			return
		}
	}
	glog.V(0).Infof("No existing leader found!")
	return
}

func (mc *MasterClient) tryAllMasters() {
	var nextHintedLeader pb.ServerAddress
	for _, master := range mc.masters {

		nextHintedLeader = mc.tryConnectToMaster(master)
		for nextHintedLeader != "" {
			nextHintedLeader = mc.tryConnectToMaster(nextHintedLeader)
		}

		mc.currentMaster = ""
	}
}

func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedLeader pb.ServerAddress) {
	glog.V(1).Infof("%s.%s masterClient Connecting to master %v", mc.FilerGroup, mc.clientType, master)
	stats.MasterClientConnectCounter.WithLabelValues("total").Inc()
	gprcErr := pb.WithMasterClient(true, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		stream, err := client.KeepConnected(ctx)
		if err != nil {
			glog.V(1).Infof("%s.%s masterClient failed to keep connected to %s: %v", mc.FilerGroup, mc.clientType, master, err)
			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToKeepConnected).Inc()
			return err
		}

		if err = stream.Send(&master_pb.KeepConnectedRequest{
			FilerGroup:    mc.FilerGroup,
			ClientType:    mc.clientType,
			ClientAddress: string(mc.clientHost),
			Version:       util.Version(),
		}); err != nil {
			glog.V(0).Infof("%s.%s masterClient failed to send to %s: %v", mc.FilerGroup, mc.clientType, master, err)
			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc()
			return err
		}
		glog.V(1).Infof("%s.%s masterClient Connected to %v", mc.FilerGroup, mc.clientType, master)

		//reset vidMap only when masterClient successfully connected to new leader
		resp, err := stream.Recv()
		if err != nil {
			glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
			return err
		}
		if resp.VolumeLocation != nil {
			//immediately check whether it is the leader
			if resp.VolumeLocation.Leader != "" {
				glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader)
				nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
				stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc()
				return nil
			}
		}
		mc.currentMaster = master
		mc.vidMap = newVidMap("")

		for {
			resp, err := stream.Recv()
			if err != nil {
				glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
				stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
				return err
			}

			if resp.VolumeLocation != nil {
				// maybe the leader is changed
				if resp.VolumeLocation.Leader != "" {
					glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader)
					nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
					stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc()
					return nil
				}

				// process new volume location
				loc := Location{
					Url:        resp.VolumeLocation.Url,
					PublicUrl:  resp.VolumeLocation.PublicUrl,
					DataCenter: resp.VolumeLocation.DataCenter,
					GrpcPort:   int(resp.VolumeLocation.GrpcPort),
				}
				for _, newVid := range resp.VolumeLocation.NewVids {
					glog.V(1).Infof("%s.%s: %s masterClient adds volume %d", mc.FilerGroup, mc.clientType, loc.Url, newVid)
					mc.addLocation(newVid, loc)
				}
				for _, deletedVid := range resp.VolumeLocation.DeletedVids {
					glog.V(1).Infof("%s.%s: %s masterClient removes volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedVid)
					mc.deleteLocation(deletedVid, loc)
				}
				for _, newEcVid := range resp.VolumeLocation.NewEcVids {
					glog.V(1).Infof("%s.%s: %s masterClient adds ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, newEcVid)
					mc.addEcLocation(newEcVid, loc)
				}
				for _, deletedEcVid := range resp.VolumeLocation.DeletedEcVids {
					glog.V(1).Infof("%s.%s: %s masterClient removes ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedEcVid)
					mc.deleteEcLocation(deletedEcVid, loc)
				}
			}

			if resp.ClusterNodeUpdate != nil {
				update := resp.ClusterNodeUpdate
				if mc.OnPeerUpdate != nil {
					if update.FilerGroup == mc.FilerGroup {
						if update.IsAdd {
							glog.V(0).Infof("+ %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader)
						} else {
							glog.V(0).Infof("- %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader)
						}
						stats.MasterClientConnectCounter.WithLabelValues(stats.OnPeerUpdate).Inc()
						mc.OnPeerUpdate(update, time.Now())
					}
				}
			}

		}

	})
	if gprcErr != nil {
		stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc()
		glog.V(1).Infof("%s.%s masterClient failed to connect with master %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
	}
	return
}

func (mc *MasterClient) WithClient(streamingMode bool, fn func(client master_pb.SeaweedClient) error) error {
	return util.Retry("master grpc", func() error {
		for mc.currentMaster == "" {
			time.Sleep(3 * time.Second)
		}
		return pb.WithMasterClient(streamingMode, mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			return fn(client)
		})
	})
}
<commit_msg>Code comment optimization<commit_after>package wdclient

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"math/rand"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

type MasterClient struct {
	FilerGroup     string
	clientType     string
	clientHost     pb.ServerAddress
	currentMaster  pb.ServerAddress
	masters        map[string]pb.ServerAddress
	grpcDialOption grpc.DialOption

	vidMap

	OnPeerUpdate func(update *master_pb.ClusterNodeUpdate, startFrom time.Time)
}

func NewMasterClient(grpcDialOption grpc.DialOption, filerGroup string, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters map[string]pb.ServerAddress) *MasterClient {
	return &MasterClient{
		FilerGroup:     filerGroup,
		clientType:     clientType,
		clientHost:     clientHost,
		masters:        masters,
		grpcDialOption: grpcDialOption,
		vidMap:         newVidMap(clientDataCenter),
	}
}

func (mc *MasterClient) GetMaster() pb.ServerAddress {
	mc.WaitUntilConnected()
	return mc.currentMaster
}

func (mc *MasterClient) GetMasters() map[string]pb.ServerAddress {
	mc.WaitUntilConnected()
	return mc.masters
}

func (mc *MasterClient) WaitUntilConnected() {
	for mc.currentMaster == "" {
		time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond)
	}
}

func (mc *MasterClient) KeepConnectedToMaster() {
	glog.V(1).Infof("%s.%s masterClient bootstraps with masters %v", mc.FilerGroup, mc.clientType, mc.masters)
	for {
		mc.tryAllMasters()
		time.Sleep(time.Second)
	}
}

func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddress) (leader string) {
	for _, master := range mc.masters {
		if master == myMasterAddress {
			continue
		}
		if grpcErr := pb.WithMasterClient(false, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
			defer cancel()
			resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
			if err != nil {
				return err
			}
			leader = resp.Leader
			return nil
		}); grpcErr != nil {
			glog.V(0).Infof("connect to %s: %v", master, grpcErr)
		}
		if leader != "" {
			glog.V(0).Infof("existing leader is %s", leader)
			return
		}
	}
	glog.V(0).Infof("No existing leader found!")
	return
}

func (mc *MasterClient) tryAllMasters() {
	var nextHintedLeader pb.ServerAddress
	for _, master := range mc.masters {

		nextHintedLeader = mc.tryConnectToMaster(master)
		for nextHintedLeader != "" {
			nextHintedLeader = mc.tryConnectToMaster(nextHintedLeader)
		}

		mc.currentMaster = ""
	}
}

func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedLeader pb.ServerAddress) {
	glog.V(1).Infof("%s.%s masterClient Connecting to master %v", mc.FilerGroup, mc.clientType, master)
	stats.MasterClientConnectCounter.WithLabelValues("total").Inc()
	gprcErr := pb.WithMasterClient(true, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		stream, err := client.KeepConnected(ctx)
		if err != nil {
			glog.V(1).Infof("%s.%s masterClient failed to keep connected to %s: %v", mc.FilerGroup, mc.clientType, master, err)
			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToKeepConnected).Inc()
			return err
		}

		if err = stream.Send(&master_pb.KeepConnectedRequest{
			FilerGroup:    mc.FilerGroup,
			ClientType:    mc.clientType,
			ClientAddress: string(mc.clientHost),
			Version:       util.Version(),
		}); err != nil {
			glog.V(0).Infof("%s.%s masterClient failed to send to %s: %v", mc.FilerGroup, mc.clientType, master, err)
			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc()
			return err
		}
		glog.V(1).Infof("%s.%s masterClient Connected to %v", mc.FilerGroup, mc.clientType, master)

		resp, err := stream.Recv()
		if err != nil {
			glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
			return err
		}

		// check if it is the leader to determine whether to reset the vidMap
		if resp.VolumeLocation != nil && resp.VolumeLocation.Leader != "" {
			glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader)
			nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
			stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc()
			return nil
		}

		mc.currentMaster = master
		mc.vidMap = newVidMap("")

		for {
			resp, err := stream.Recv()
			if err != nil {
				glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
				stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
				return err
			}

			if resp.VolumeLocation != nil {
				// maybe the leader is changed
				if resp.VolumeLocation.Leader != "" {
					glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader)
					nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
					stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc()
					return nil
				}

				// process new volume location
				loc := Location{
					Url:        resp.VolumeLocation.Url,
					PublicUrl:  resp.VolumeLocation.PublicUrl,
					DataCenter: resp.VolumeLocation.DataCenter,
					GrpcPort:   int(resp.VolumeLocation.GrpcPort),
				}
				for _, newVid := range resp.VolumeLocation.NewVids {
					glog.V(1).Infof("%s.%s: %s masterClient adds volume %d", mc.FilerGroup, mc.clientType, loc.Url, newVid)
					mc.addLocation(newVid, loc)
				}
				for _, deletedVid := range resp.VolumeLocation.DeletedVids {
					glog.V(1).Infof("%s.%s: %s masterClient removes volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedVid)
					mc.deleteLocation(deletedVid, loc)
				}
				for _, newEcVid := range resp.VolumeLocation.NewEcVids {
					glog.V(1).Infof("%s.%s: %s masterClient adds ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, newEcVid)
					mc.addEcLocation(newEcVid, loc)
				}
				for _, deletedEcVid := range resp.VolumeLocation.DeletedEcVids {
					glog.V(1).Infof("%s.%s: %s masterClient removes ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedEcVid)
					mc.deleteEcLocation(deletedEcVid, loc)
				}
			}

			if resp.ClusterNodeUpdate != nil {
				update := resp.ClusterNodeUpdate
				if mc.OnPeerUpdate != nil {
					if update.FilerGroup == mc.FilerGroup {
						if update.IsAdd {
							glog.V(0).Infof("+ %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader)
						} else {
							glog.V(0).Infof("- %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader)
						}
						stats.MasterClientConnectCounter.WithLabelValues(stats.OnPeerUpdate).Inc()
						mc.OnPeerUpdate(update, time.Now())
					}
				}
			}

		}

	})
	if gprcErr != nil {
		stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc()
		glog.V(1).Infof("%s.%s masterClient failed to connect with master %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
	}
	return
}

func (mc *MasterClient) WithClient(streamingMode bool, fn func(client master_pb.SeaweedClient) error) error {
	return util.Retry("master grpc", func() error {
		for mc.currentMaster == "" {
			time.Sleep(3 * time.Second)
		}
		return pb.WithMasterClient(streamingMode, mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			return fn(client)
		})
	})
}
<|endoftext|>


<commit_before>package zlistutil

import (
	"encoding/json"
	// "fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const (
	// V2ex
	V2EX_BASE_URL = "https://www.v2ex.com/api/topics/"
	V2EX_HOT      = "https://www.v2ex.com/api/topics/hot.json"
	V2EX_LATEST   = "https://www.v2ex.com/api/topics/latest.json"
	// Zhihu Daily
	DAILY_FETCH_NOW    = "http://news.at.zhihu.com/api/1.2/news/latest"
	DAILY_FETCH_BEFORE = "http://news.at.zhihu.com/api/1.2/news/before/"
	//Hacker News
	HACKER_NEWS_BASE_URL     = "https://news.ycombinator.com"
	HACKER_NEWS_BASE_API_URL = "https://hacker-news.firebaseio.com"
	HACKER_NEWS_TOP          = "https://hacker-news.firebaseio.com/v0/topstories.json"
	HACKER_NEWS_ITEM         = "https://hacker-news.firebaseio.com/v0/item/"
	// Jianshu
	JIANSHU_BASE_URL = "http://www.jianshu.com"
	JIANSHU_TOP_DAY  = "http://www.jianshu.com/trending/now"
	// Product Hunt
	PRODUCTHUNT_TODAY     = "https://api.producthunt.com/v1/posts"
	PRODUCTHUNT_YESTERDAY = "https://api.producthunt.com/v1/posts?days_ago=1"
	//36kr NEXT
	NEXT_BASE_URL = "http://next.36kr.com"
	NEXT          = "http://next.36kr.com/posts"
	//湾区日报
	WANQU = "http://wanqu.co"
	//PingWest快讯
	PINGWEST_NEWS = "http://news.pingwest.com/"
	//Solidot
	SOLIDOT = "http://www.solidot.org"
	//GitHub
	GITHUB_BASE_URL = "https://github.com"
	GITHUB          = "https://github.com/trending"
	//豆瓣 一刻
	DOUBAN_MOMENT = "http://moment.douban.com/api/stream/date/"
	//ifanr 观察
	IFANR = "http://www.ifanr.com"
	//mindstore
	MINDSTORE = "http://mindstore.io"
	//Kickstarter technology newest
	KICKSTARTER_BASE_URL = "https://www.kickstarter.com"
	KICKSTARTER          = "https://www.kickstarter.com/discover/categories/technology?sort=newest"
	//gadgethunt daily
	GADGETHUNT = "http://www.gadgethunt.club/daily"
)

type Item struct {
	Title string `json:"title"`
	Url   string `json:"url"`
}

type DailyItems struct {
	News []struct {
		Title string `json:"title"`
		Url   string `json:"share_url"`
	}
}
type MomentItems struct {
	Posts []struct {
		Title string `json:"title"`
		Url   string `json:"url"`
	}
}
type ProductHuntsItems struct {
	Posts []struct {
		Name    string `json:"name"`
		Tagline string `json:"tagline"`
		Url     string `json:"redirect_url"`
	}
}

func http_helper(method, url string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest(method, url, nil)
	req.Close = true
	// req.Header.Set("Content-Type", "application/json")
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func min(a, b int) int {
	if a < b {
		return a
	} else {
		return b
	}
}
func getDate() string {
	const layout = "2006-01-02"
	t := time.Now()
	return t.Format(layout)
}

func FetchProductHunt(url string, num int) []Item {
	var item_list []Item
	item_list = fetchProductHuntHelper(url, num)
	if item_list == nil && url == PRODUCTHUNT_TODAY {
		item_list = fetchProductHuntHelper(PRODUCTHUNT_YESTERDAY, num)
	}
	return item_list
}

func fetchProductHuntHelper(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.Close = true
	req.Header.Set("Authorization", "Bearer 2dd283a9b3643bc72211c5c2b4aa085b7c9906d68194ea4805c00c46e7be01f4")
	res, err := client.Do(req)
	if err != nil {
		return nil
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var productItems ProductHuntsItems
	err = json.Unmarshal(body, &productItems)
	perror(err)
	var items []Item
	num = min(num, len(productItems.Posts))
	for i := 0; i < num; i++ {
		var item Item
		item.Title = productItems.Posts[i].Name + " : " + productItems.Posts[i].Tagline
		item.Url = productItems.Posts[i].Url
		items = append(items, item)
	}
	return items
}

func FetchJianshu(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".top-notes li").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("h4").Text()
		item.Url, _ = s.Find("h4 a").Attr("href")
		item.Url = JIANSHU_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchNEXT(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".post").First().Find(".product-item").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find(".product-url a").Text() + " : " + s.Find(".post-tagline").Text()
		item.Url, _ = s.Find(".product-url a").Attr("href")
		item.Url = NEXT_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchHackerNews(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	body, err := http_helper("GET", url)
	perror(err)
	var ids []int
	err = json.Unmarshal(body, &ids)
	perror(err)
	var items []Item
	num = min(num, len(ids))
	for i := 0; i < num; i++ {
		item_url := HACKER_NEWS_ITEM + strconv.Itoa(ids[i]) + ".json"
		body, err := http_helper("GET", item_url)
		perror(err)
		var item Item
		err = json.Unmarshal(body, &item)
		perror(err)
		if item.Url == "" { //Ask HN
			item.Url = HACKER_NEWS_BASE_URL + "/item?id=" + strconv.Itoa(ids[i])
		}
		items = append(items, item)
	}
	return items
}
func FetchV2ex(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	res, err := http.Get(url)
	perror(err)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var items []Item
	err = json.Unmarshal(body, &items)
	perror(err)
	num = min(num, len(items))
	return items[:num]
}

func FetchZhihuDaily(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	res, err := http.Get(url)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var daily DailyItems
	err = json.Unmarshal(body, &daily)
	perror(err)

	var items []Item
	num = min(num, len(daily.News))
	for i := 0; i < num; i++ {
		var item Item
		item.Title = daily.News[i].Title
		item.Url = daily.News[i].Url
		items = append(items, item)
	}
	return items

}
func FetchWanqu(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".list-group-item").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("a").Text()
		item.Url, _ = s.Find("a").Attr("href")
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]

}
func FetchPingWestNews(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".item_title").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find(".topic").Text()
		item.Url, _ = s.Find(".topic").Attr("href")
		items = append(items, item)
		// fmt.Print(i)
	})
	num = min(num, len(items))
	return items[:num]

}

func FetchSolidot(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".bg_htit").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("a").Last().Text()
		item.Url, _ = s.Find("a").Last().Attr("href")
		item.Url = SOLIDOT + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}
func FetchGitHub(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".repo-list-item").Each(func(i int, s *goquery.Selection) {
		var item Item
		// item.Title = s.Find(".repo-list-name a").Text()
		item.Url, _ = s.Find(".repo-list-name a").Attr("href")
		item.Title = item.Url[1:] + " : " + strings.TrimSpace(s.Find(".repo-list-description").Text())
		item.Url = GITHUB_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}
func FetchDoubanMoment(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	url += getDate()
	res, err := http.Get(url)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var moment MomentItems
	err = json.Unmarshal(body, &moment)
	perror(err)

	var items []Item
	num = min(num, len(moment.Posts))
	for i := 0; i < num; i++ {
		var item Item
		item.Title = moment.Posts[i].Title
		item.Url = moment.Posts[i].Url
		items = append(items, item)
	}
	return items

}
func FetchIfanr(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".live-list li").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("a").Text()
		item.Url, _ = s.Find("a").Attr("href")
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchMindStore(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".mind-list-ul li").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = strings.TrimSpace(s.Find(".mind-title a").Text()) + " : " + strings.TrimSpace(s.Find(".mind-des").Text())
		item.Url, _ = s.Find(".mind-title a").Attr("href")
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchKickstarter(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".project").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find(".project-card-content a").Text() + " : " + strings.TrimSpace(s.Find(".project-blurb").Text())
		item.Url, _ = s.Find(".project-thumbnail a").Attr("href")
		item.Url = KICKSTARTER_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}
<commit_msg>Fix ifanr #1<commit_after>package zlistutil

import (
	"encoding/json"
	// "fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const (
	// V2ex
	V2EX_BASE_URL = "https://www.v2ex.com/api/topics/"
	V2EX_HOT      = "https://www.v2ex.com/api/topics/hot.json"
	V2EX_LATEST   = "https://www.v2ex.com/api/topics/latest.json"
	// Zhihu Daily
	DAILY_FETCH_NOW    = "http://news.at.zhihu.com/api/1.2/news/latest"
	DAILY_FETCH_BEFORE = "http://news.at.zhihu.com/api/1.2/news/before/"
	//Hacker News
	HACKER_NEWS_BASE_URL     = "https://news.ycombinator.com"
	HACKER_NEWS_BASE_API_URL = "https://hacker-news.firebaseio.com"
	HACKER_NEWS_TOP          = "https://hacker-news.firebaseio.com/v0/topstories.json"
	HACKER_NEWS_ITEM         = "https://hacker-news.firebaseio.com/v0/item/"
	// Jianshu
	JIANSHU_BASE_URL = "http://www.jianshu.com"
	JIANSHU_TOP_DAY  = "http://www.jianshu.com/trending/now"
	// Product Hunt
	PRODUCTHUNT_TODAY     = "https://api.producthunt.com/v1/posts"
	PRODUCTHUNT_YESTERDAY = "https://api.producthunt.com/v1/posts?days_ago=1"
	//36kr NEXT
	NEXT_BASE_URL = "http://next.36kr.com"
	NEXT          = "http://next.36kr.com/posts"
	//湾区日报
	WANQU = "http://wanqu.co"
	//PingWest快讯
	PINGWEST_NEWS = "http://news.pingwest.com/"
	//Solidot
	SOLIDOT = "http://www.solidot.org"
	//GitHub
	GITHUB_BASE_URL = "https://github.com"
	GITHUB          = "https://github.com/trending"
	//豆瓣 一刻
	DOUBAN_MOMENT = "http://moment.douban.com/api/stream/date/"
	//ifanr 观察
	IFANR = "http://www.ifanr.com"
	//mindstore
	MINDSTORE = "http://mindstore.io"
	//Kickstarter technology newest
	KICKSTARTER_BASE_URL = "https://www.kickstarter.com"
	KICKSTARTER          = "https://www.kickstarter.com/discover/categories/technology?sort=newest"
	//gadgethunt daily
	GADGETHUNT = "http://www.gadgethunt.club/daily"
)

type Item struct {
	Title string `json:"title"`
	Url   string `json:"url"`
}

type DailyItems struct {
	News []struct {
		Title string `json:"title"`
		Url   string `json:"share_url"`
	}
}
type MomentItems struct {
	Posts []struct {
		Title string `json:"title"`
		Url   string `json:"url"`
	}
}
type ProductHuntsItems struct {
	Posts []struct {
		Name    string `json:"name"`
		Tagline string `json:"tagline"`
		Url     string `json:"redirect_url"`
	}
}

func http_helper(method, url string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest(method, url, nil)
	req.Close = true
	// req.Header.Set("Content-Type", "application/json")
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func min(a, b int) int {
	if a < b {
		return a
	} else {
		return b
	}
}
func getDate() string {
	const layout = "2006-01-02"
	t := time.Now()
	return t.Format(layout)
}

func FetchProductHunt(url string, num int) []Item {
	var item_list []Item
	item_list = fetchProductHuntHelper(url, num)
	if item_list == nil && url == PRODUCTHUNT_TODAY {
		item_list = fetchProductHuntHelper(PRODUCTHUNT_YESTERDAY, num)
	}
	return item_list
}

func fetchProductHuntHelper(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.Close = true
	req.Header.Set("Authorization", "Bearer 2dd283a9b3643bc72211c5c2b4aa085b7c9906d68194ea4805c00c46e7be01f4")
	res, err := client.Do(req)
	if err != nil {
		return nil
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var productItems ProductHuntsItems
	err = json.Unmarshal(body, &productItems)
	perror(err)
	var items []Item
	num = min(num, len(productItems.Posts))
	for i := 0; i < num; i++ {
		var item Item
		item.Title = productItems.Posts[i].Name + " : " + productItems.Posts[i].Tagline
		item.Url = productItems.Posts[i].Url
		items = append(items, item)
	}
	return items
}

func FetchJianshu(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".top-notes li").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("h4").Text()
		item.Url, _ = s.Find("h4 a").Attr("href")
		item.Url = JIANSHU_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchNEXT(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".post").First().Find(".product-item").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find(".product-url a").Text() + " : " + s.Find(".post-tagline").Text()
		item.Url, _ = s.Find(".product-url a").Attr("href")
		item.Url = NEXT_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchHackerNews(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	body, err := http_helper("GET", url)
	perror(err)
	var ids []int
	err = json.Unmarshal(body, &ids)
	perror(err)
	var items []Item
	num = min(num, len(ids))
	for i := 0; i < num; i++ {
		item_url := HACKER_NEWS_ITEM + strconv.Itoa(ids[i]) + ".json"
		body, err := http_helper("GET", item_url)
		perror(err)
		var item Item
		err = json.Unmarshal(body, &item)
		perror(err)
		if item.Url == "" { //Ask HN
			item.Url = HACKER_NEWS_BASE_URL + "/item?id=" + strconv.Itoa(ids[i])
		}
		items = append(items, item)
	}
	return items
}
func FetchV2ex(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	res, err := http.Get(url)
	perror(err)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var items []Item
	err = json.Unmarshal(body, &items)
	perror(err)
	num = min(num, len(items))
	return items[:num]
}

func FetchZhihuDaily(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	res, err := http.Get(url)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var daily DailyItems
	err = json.Unmarshal(body, &daily)
	perror(err)

	var items []Item
	num = min(num, len(daily.News))
	for i := 0; i < num; i++ {
		var item Item
		item.Title = daily.News[i].Title
		item.Url = daily.News[i].Url
		items = append(items, item)
	}
	return items

}
func FetchWanqu(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".list-group-item").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("a").Text()
		item.Url, _ = s.Find("a").Attr("href")
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]

}
func FetchPingWestNews(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".item_title").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find(".topic").Text()
		item.Url, _ = s.Find(".topic").Attr("href")
		items = append(items, item)
		// fmt.Print(i)
	})
	num = min(num, len(items))
	return items[:num]

}

func FetchSolidot(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".bg_htit").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("a").Last().Text()
		item.Url, _ = s.Find("a").Last().Attr("href")
		item.Url = SOLIDOT + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}
func FetchGitHub(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".repo-list-item").Each(func(i int, s *goquery.Selection) {
		var item Item
		// item.Title = s.Find(".repo-list-name a").Text()
		item.Url, _ = s.Find(".repo-list-name a").Attr("href")
		item.Title = item.Url[1:] + " : " + strings.TrimSpace(s.Find(".repo-list-description").Text())
		item.Url = GITHUB_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}
func FetchDoubanMoment(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	url += getDate()
	res, err := http.Get(url)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var moment MomentItems
	err = json.Unmarshal(body, &moment)
	perror(err)

	var items []Item
	num = min(num, len(moment.Posts))
	for i := 0; i < num; i++ {
		var item Item
		item.Title = moment.Posts[i].Title
		item.Url = moment.Posts[i].Url
		items = append(items, item)
	}
	return items

}
func FetchIfanr(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".post-item-content h2").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find("a").Text()
		item.Url, _ = s.Find("a").Attr("href")
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchMindStore(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".mind-list-ul li").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = strings.TrimSpace(s.Find(".mind-title a").Text()) + " : " + strings.TrimSpace(s.Find(".mind-des").Text())
		item.Url, _ = s.Find(".mind-title a").Attr("href")
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}

func FetchKickstarter(url string, num int) []Item {
	if num < 0 {
		return nil
	}
	doc, err := goquery.NewDocument(url)
	perror(err)
	var items []Item
	doc.Find(".project").Each(func(i int, s *goquery.Selection) {
		var item Item
		item.Title = s.Find(".project-card-content a").Text() + " : " + strings.TrimSpace(s.Find(".project-blurb").Text())
		item.Url, _ = s.Find(".project-thumbnail a").Attr("href")
		item.Url = KICKSTARTER_BASE_URL + item.Url
		items = append(items, item)
	})
	num = min(num, len(items))
	return items[:num]
}
<|endoftext|>


<commit_before>package myconfig

import (
	"context"
	"errors"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/clientv3"
)

const (
	dialTimeout    = 5 * time.Second
	requestTimeout = 1 * time.Second
)

// Config holds configuration data.
type Config struct {
	Endpoints []string
	Env       string
	Service   string
}

// Client is configuration client.
type Client struct {
	etcdClient    *clientv3.Client
	globalPrefix  string
	servicePrefix string
	storage       *data
}

// New creates configuration client.
func New(cfg Config) (*Client, error) {

	ec, err := clientv3.New(clientv3.Config{
		Endpoints:   cfg.Endpoints,
		DialTimeout: dialTimeout,
	})

	if err != nil {
		return nil, err
	}

	c := &Client{
		etcdClient: ec,
		storage:    newData(),
	}

	if err := c.setPrefixes(cfg.Env, cfg.Service); err != nil {
		return nil, fmt.Errorf("prefixes: %v", err)
	}

	var wg sync.WaitGroup
	wg.Add(2)

	go c.updateStorage(&wg, c.globalPrefix)

	go c.updateStorage(&wg, c.servicePrefix)

	wg.Wait()

	return c, nil

}

func (c *Client) String(key string) (string, bool) {
	v, ok := c.storage.get(key)
	return v, ok
}

func (c *Client) updateStorage(wg *sync.WaitGroup, prefix string) {
	defer wg.Done()
	dm, err := c.get(prefix)
	if err != nil {
		log.Printf("update client: %s: %v", prefix, err)
	}
	c.storage.update(dm)
}

const companyKey = "com"

func (c *Client) setPrefixes(env, service string) error {
	if env == "" {
		return errors.New("empty env")
	}
	if service == "" {
		return errors.New("empty service")
	}
	root := fmt.Sprintf("/%s/%s", companyKey, env)
	c.globalPrefix = fmt.Sprintf("%s/global", root)
	c.servicePrefix = fmt.Sprintf("%s/%s", root, service)
	return nil
}

func (c *Client) get(prefix string) (dataMap, error) {

	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
	resp, err := c.etcdClient.Get(
		ctx,
		prefix,
		clientv3.WithPrefix(),
		clientv3.WithSerializable(),
	)

	defer cancel()

	dm := dataMap{}

	if err != nil {
		return dm, err
	}

	if len(resp.Kvs) < 1 {
		return dm, errors.New("not exists")
	}

	var k, v string

	for _, ev := range resp.Kvs {
		k = string(ev.Key)
		if strings.HasPrefix(k, c.globalPrefix) {
			k = "/global" + strings.TrimPrefix(k, c.globalPrefix)
		}
		k = strings.TrimPrefix(k, c.servicePrefix)

		v = string(ev.Value)

		dm[k] = v
	}
	return dm, nil
}

// Close closes client.
func Close(c *Client) {

	err := c.etcdClient.Close()
	if err != nil {
		// TODO(dvrkps): add better logging
		log.Print(err)
	}
}
<commit_msg>etcdconfig/myconfig: add checkKey<commit_after>package myconfig

import (
	"context"
	"errors"
	"fmt"
	"log"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/clientv3"
)

const (
	dialTimeout    = 5 * time.Second
	requestTimeout = 1 * time.Second
)

// Config holds configuration data.
type Config struct {
	Endpoints []string
	Env       string
	Service   string
}

// Client is configuration client.
type Client struct {
	etcdClient    *clientv3.Client
	globalPrefix  string
	servicePrefix string
	storage       *data
}

// New creates configuration client.
func New(cfg Config) (*Client, error) {

	ec, err := clientv3.New(clientv3.Config{
		Endpoints:   cfg.Endpoints,
		DialTimeout: dialTimeout,
	})

	if err != nil {
		return nil, err
	}

	c := &Client{
		etcdClient: ec,
		storage:    newData(),
	}

	if err := c.setPrefixes(cfg.Env, cfg.Service); err != nil {
		return nil, fmt.Errorf("prefixes: %v", err)
	}

	var wg sync.WaitGroup
	wg.Add(2)

	go c.updateStorage(&wg, c.globalPrefix)

	go c.updateStorage(&wg, c.servicePrefix)

	wg.Wait()

	return c, nil

}

func (c *Client) String(key string) (string, bool) {
	v, ok := c.storage.get(key)
	return v, ok
}

func (c *Client) updateStorage(wg *sync.WaitGroup, prefix string) {
	defer wg.Done()
	dm, err := c.get(prefix)
	if err != nil {
		log.Printf("update client: %s: %v", prefix, err)
	}
	c.storage.update(dm)
}

const companyKey = "com"

func (c *Client) setPrefixes(env, service string) error {
	if env == "" {
		return errors.New("empty env")
	}
	if service == "" {
		return errors.New("empty service")
	}
	root := fmt.Sprintf("/%s/%s", companyKey, env)
	c.globalPrefix = fmt.Sprintf("%s/global", root)
	c.servicePrefix = fmt.Sprintf("%s/%s", root, service)
	return nil
}

func (c *Client) get(prefix string) (dataMap, error) {

	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
	resp, err := c.etcdClient.Get(
		ctx,
		prefix,
		clientv3.WithPrefix(),
		clientv3.WithSerializable(),
	)

	defer cancel()

	dm := dataMap{}

	if err != nil {
		return dm, err
	}

	if len(resp.Kvs) < 1 {
		return dm, errors.New("not exists")
	}

	var k, v string

	for _, ev := range resp.Kvs {
		k = string(ev.Key)
		if strings.HasPrefix(k, c.globalPrefix) {
			k = "/global" + strings.TrimPrefix(k, c.globalPrefix)
		}
		k = strings.TrimPrefix(k, c.servicePrefix)

		v = string(ev.Value)

		dm[k] = v
	}
	return dm, nil
}

// Close closes client.
func Close(c *Client) {

	err := c.etcdClient.Close()
	if err != nil {
		// TODO(dvrkps): add better logging
		log.Print(err)
	}
}

func checkKey(key string) error {
	if key == "" || key == "/" {
		return errors.New("empty key")
	}
	if path.IsAbs(key) {
		return errors.New("relative key")
	}
	return nil
}
<|endoftext|>


<commit_before>package caaWorker

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/miekg/dns"
	"github.com/mozilla/tls-observatory/worker"
)

var (
	workerName = "caaWorker"
	workerDesc = "Checks domains DNS records for a CAA record and reports it."
)

func init() {
	worker.RegisterWorker(workerName, worker.Info{Runner: new(eval), Description: workerDesc})
}

// Result describes the result produced by CAAWorker
type Result struct {
	HasCAA       bool     `json:"has_caa"`
	Host         string   `json:"host"`
	IssueCAs     []string `json:"issue"`
	IssueWildCAs []string `json:"issuewild"`
}

type eval struct{}

// Run implements the worker interface.It is called to get the worker results.
func (e eval) Run(in worker.Input, resChan chan worker.Result) {
	result := worker.Result{WorkerName: workerName, Success: true}
	caaRes := Result{}

	hostPieces := strings.Split(in.Target, ".")
	for i := 0; i < len(hostPieces); i++ {
		host := strings.Join(hostPieces[i:], ".")

		msg := new(dns.Msg)
		msg.SetQuestion(dns.Fqdn(host), dns.TypeCAA)

		client := dns.Client{}
		res, _, err := client.Exchange(msg, "8.8.8.8:53")
		if err != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("CAA lookup failed for %s: %v", host, err))
			continue
		}

		if res.Rcode != dns.RcodeSuccess {
			result.Errors = append(result.Errors, fmt.Sprintf("CAA lookup failed for %s with %s", host, dns.RcodeToString[res.Rcode]))
			continue
		}

		for _, rr := range res.Answer {
			if caa, ok := rr.(*dns.CAA); ok {
				caaRes.HasCAA = true
				if caa.Tag == "issue" {
					caaRes.IssueCAs = append(caaRes.IssueCAs, caa.Value)
				} else if caa.Tag == "issuewild" {
					caaRes.IssueWildCAs = append(caaRes.IssueWildCAs, caa.Value)
				}
			}
		}

		if caaRes.HasCAA {
			caaRes.Host = host
			break
		}
	}

	res, err := json.Marshal(caaRes)
	if err != nil {
		result.Errors = append(result.Errors, err.Error())
		result.Success = false
	} else {
		result.Success = true
		result.Result = res
	}

	resChan <- result
}

// Assertor compares 2 caaResults and reports differences.
func (e eval) Assertor(caaResult, assertresults []byte) (pass bool, body []byte, err error) {
	var result, assertres Result
	pass = false
	err = json.Unmarshal(caaResult, &result)
	if err != nil {
		return
	}
	err = json.Unmarshal(assertresults, &assertres)
	if err != nil {
		return
	}

	if result.HasCAA != assertres.HasCAA {
		body = []byte("CAA mismatch")
		return
	}
	if result.Host != assertres.Host {
		body = []byte(fmt.Sprintf(`Assertion failed MatchedHost= %s`,
			result.Host))
		return
	}

	if len(result.IssueCAs) != len(assertres.IssueCAs) {
		body = []byte("Issue CAs count mismatch")
		return
	}

	for i := range result.IssueCAs {
		if result.IssueCAs[i] != assertres.IssueCAs[i] {
			body = []byte(fmt.Sprintf(`Issue CAs mismatch %s != %s`,
				result.IssueCAs[i], assertres.IssueCAs[i]))
			return
		}
	}

	if len(result.IssueWildCAs) != len(assertres.IssueWildCAs) {
		body = []byte("Issue CAs count mismatch")
		return
	}

	for i := range result.IssueWildCAs {
		if result.IssueWildCAs[i] != assertres.IssueWildCAs[i] {
			body = []byte(fmt.Sprintf(`Issue CAs mismatch %s != %s`,
				result.IssueWildCAs[i], assertres.IssueWildCAs[i]))
			return
		}
	}

	pass = true
	return
}

func (e eval) AnalysisPrinter(input []byte, printAll interface{}) (results []string, err error) {
	var r Result
	err = json.Unmarshal(input, &r)
	if err != nil {
		err = fmt.Errorf("CAA worker: failed to parse results: %v", err)
		return
	}

	results = append(results, "* CAA Analyzer:")

	if !r.HasCAA {
		results = append(results, "  * CAA records: Not found")
	} else {
		results = append(results, "  * CAA records: Found")
		for _, issue := range r.IssueCAs {
			results = append(results, fmt.Sprintf("    - CA '%s' permitted to issue certs for '%s'", issue, r.Host))
		}
		for _, wild := range r.IssueWildCAs {
			results = append(results, fmt.Sprintf("    - CA '%s' permitted to issue wildcard certs for '%s'", wild, r.Host))
		}
	}

	return results, nil
}
<commit_msg>Use configured nameserver if available, google's as fallback<commit_after>package caaWorker

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/miekg/dns"
	"github.com/mozilla/tls-observatory/worker"
)

var (
	workerName = "caaWorker"
	workerDesc = "Checks domains DNS records for a CAA record and reports it."
)

//Use the google DNS servers as fallback
var DNSServer = "8.8.8.8:53"

func init() {
	cfg, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err == nil && len(cfg.Servers) > 0 {
		//if there are configured nameservers use them
		DNSServer = strings.Join([]string{cfg.Servers[0], cfg.Port}, ":")
	}
	worker.RegisterWorker(workerName, worker.Info{Runner: new(eval), Description: workerDesc})
}

// Result describes the result produced by CAAWorker
type Result struct {
	HasCAA       bool     `json:"has_caa"`
	Host         string   `json:"host"`
	IssueCAs     []string `json:"issue"`
	IssueWildCAs []string `json:"issuewild"`
}

type eval struct{}

// Run implements the worker interface.It is called to get the worker results.
func (e eval) Run(in worker.Input, resChan chan worker.Result) {
	result := worker.Result{WorkerName: workerName, Success: true}
	caaRes := Result{}

	hostPieces := strings.Split(in.Target, ".")
	for i := 0; i < len(hostPieces); i++ {
		host := strings.Join(hostPieces[i:], ".")

		msg := new(dns.Msg)
		msg.SetQuestion(dns.Fqdn(host), dns.TypeCAA)

		client := dns.Client{}
		res, _, err := client.Exchange(msg, DNSServer)
		if err != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("CAA lookup failed for %s: %v", host, err))
			continue
		}

		if res.Rcode != dns.RcodeSuccess {
			result.Errors = append(result.Errors, fmt.Sprintf("CAA lookup failed for %s with %s", host, dns.RcodeToString[res.Rcode]))
			continue
		}

		for _, rr := range res.Answer {
			if caa, ok := rr.(*dns.CAA); ok {
				caaRes.HasCAA = true
				if caa.Tag == "issue" {
					caaRes.IssueCAs = append(caaRes.IssueCAs, caa.Value)
				} else if caa.Tag == "issuewild" {
					caaRes.IssueWildCAs = append(caaRes.IssueWildCAs, caa.Value)
				}
			}
		}

		if caaRes.HasCAA {
			caaRes.Host = host
			break
		}
	}

	res, err := json.Marshal(caaRes)
	if err != nil {
		result.Errors = append(result.Errors, err.Error())
		result.Success = false
	} else {
		result.Success = true
		result.Result = res
	}

	resChan <- result
}

// Assertor compares 2 caaResults and reports differences.
func (e eval) Assertor(caaResult, assertresults []byte) (pass bool, body []byte, err error) {
	var result, assertres Result
	pass = false
	err = json.Unmarshal(caaResult, &result)
	if err != nil {
		return
	}
	err = json.Unmarshal(assertresults, &assertres)
	if err != nil {
		return
	}

	if result.HasCAA != assertres.HasCAA {
		body = []byte("CAA mismatch")
		return
	}
	if result.Host != assertres.Host {
		body = []byte(fmt.Sprintf(`Assertion failed MatchedHost= %s`,
			result.Host))
		return
	}

	if len(result.IssueCAs) != len(assertres.IssueCAs) {
		body = []byte("Issue CAs count mismatch")
		return
	}

	for i := range result.IssueCAs {
		if result.IssueCAs[i] != assertres.IssueCAs[i] {
			body = []byte(fmt.Sprintf(`Issue CAs mismatch %s != %s`,
				result.IssueCAs[i], assertres.IssueCAs[i]))
			return
		}
	}

	if len(result.IssueWildCAs) != len(assertres.IssueWildCAs) {
		body = []byte("Issue CAs count mismatch")
		return
	}

	for i := range result.IssueWildCAs {
		if result.IssueWildCAs[i] != assertres.IssueWildCAs[i] {
			body = []byte(fmt.Sprintf(`Issue CAs mismatch %s != %s`,
				result.IssueWildCAs[i], assertres.IssueWildCAs[i]))
			return
		}
	}

	pass = true
	return
}

func (e eval) AnalysisPrinter(input []byte, printAll interface{}) (results []string, err error) {
	var r Result
	err = json.Unmarshal(input, &r)
	if err != nil {
		err = fmt.Errorf("CAA worker: failed to parse results: %v", err)
		return
	}

	results = append(results, "* CAA Analyzer:")

	if !r.HasCAA {
		results = append(results, "  * CAA records: Not found")
	} else {
		results = append(results, "  * CAA records: Found")
		for _, issue := range r.IssueCAs {
			results = append(results, fmt.Sprintf("    - CA '%s' permitted to issue certs for '%s'", issue, r.Host))
		}
		for _, wild := range r.IssueWildCAs {
			results = append(results, fmt.Sprintf("    - CA '%s' permitted to issue wildcard certs for '%s'", wild, r.Host))
		}
	}

	return results, nil
}
<|endoftext|>