<commit_before>// Copyright 2014 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

package http

import (
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"

	"github.com/chihaya/bencode"
	"github.com/chihaya/chihaya/config"
	"github.com/chihaya/chihaya/models"

	_ "github.com/chihaya/chihaya/drivers/backend/noop"
	_ "github.com/chihaya/chihaya/drivers/tracker/memory"
)

func loadTestData(tkr *Tracker) (err error) {
	conn, err := tkr.tp.Get()
	if err != nil {
		return
	}

	users := []string{
		"vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv1",
		"vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv2",
		"vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv3",
	}

	for i, passkey := range users {
		err = conn.PutUser(&models.User{
			ID:      uint64(i + 1),
			Passkey: passkey,
		})

		if err != nil {
			return
		}
	}

	err = conn.PutClient("TR2820")
	if err != nil {
		return
	}

	torrent := &models.Torrent{
		ID:       1,
		Infohash: string([]byte{0x89, 0xd4, 0xbc, 0x52, 0x11, 0x16, 0xca, 0x1d, 0x42, 0xa2, 0xf3, 0x0d, 0x1f, 0x27, 0x4d, 0x94, 0xe4, 0x68, 0x1d, 0xaf}),
		Seeders:  make(map[string]models.Peer),
		Leechers: make(map[string]models.Peer),
	}

	err = conn.PutTorrent(torrent)
	if err != nil {
		return
	}

	err = conn.PutLeecher(torrent.Infohash, &models.Peer{
		ID:        "-TR2820-vvvvvvvvvvv1",
		UserID:    1,
		TorrentID: torrent.ID,
		IP:        net.ParseIP("127.0.0.1"),
		Port:      34000,
		Left:      0,
	})
	if err != nil {
		return
	}

	err = conn.PutLeecher(torrent.Infohash, &models.Peer{
		ID:        "-TR2820-vvvvvvvvvvv3",
		UserID:    3,
		TorrentID: torrent.ID,
		IP:        net.ParseIP("::1"),
		Port:      34000,
		Left:      0,
	})
	return
}

func testRoute(cfg *config.Config, url string) ([]byte, error) {
	tkr, err := NewTracker(cfg)
	if err != nil {
		return nil, err
	}

	err = loadTestData(tkr)
	if err != nil {
		return nil, err
	}

	srv := httptest.NewServer(setupRoutes(tkr, cfg))
	defer srv.Close()

	resp, err := http.Get(srv.URL + url)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func TestPrivateAnnounce(t *testing.T) {
	cfg := config.DefaultConfig
	cfg.Private = true

	url := "/users/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv2/announce?info_hash=%89%d4%bcR%11%16%ca%1dB%a2%f3%0d%1f%27M%94%e4h%1d%af&peer_id=-TR2820-vvvvvvvvvvv2&port=51413&uploaded=0&downloaded=0&left=0&numwant=1&key=3c8e3319&compact=0"

	expected := bencode.Dict{
		"complete":     int64(1),
		"incomplete":   int64(2),
		"interval":     int64(1800),
		"min interval": int64(900),
		"peers": bencode.List{
			bencode.Dict{
				"ip":      "127.0.0.1",
				"peer id": "-TR2820-vvvvvvvvvvv1",
				"port":    int64(34000),
			},
		},
	}

	response, err := testRoute(&cfg, url)
	if err != nil {
		t.Error(err)
	}
	got, err := bencode.Unmarshal(response)

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("\ngot:    %#v\nwanted: %#v", got, expected)
	}

	url = "/users/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv2/announce?info_hash=%89%d4%bcR%11%16%ca%1dB%a2%f3%0d%1f%27M%94%e4h%1d%af&peer_id=-TR2820-vvvvvvvvvvv2&port=51413&uploaded=0&downloaded=0&left=0&numwant=2&key=3c8e3319&compact=0"

	expected = bencode.Dict{
		"complete":     int64(1),
		"incomplete":   int64(2),
		"interval":     int64(1800),
		"min interval": int64(900),
		"peers": bencode.List{
			bencode.Dict{
				"ip":      "127.0.0.1",
				"peer id": "-TR2820-vvvvvvvvvvv1",
				"port":    int64(34000),
			},
			bencode.Dict{
				"ip":      "::1",
				"peer id": "-TR2820-vvvvvvvvvvv3",
				"port":    int64(34000),
			},
		},
	}

	response, err = testRoute(&cfg, url)
	if err != nil {
		t.Error(err)
	}
	got, err = bencode.Unmarshal(response)

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("\ngot:    %#v\nwanted: %#v", got, expected)
	}
}
<commit_msg>Less duplication<commit_after>// Copyright 2014 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

package http

import (
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"testing"

	"github.com/chihaya/bencode"
	"github.com/chihaya/chihaya/config"
	"github.com/chihaya/chihaya/models"

	_ "github.com/chihaya/chihaya/drivers/backend/noop"
	_ "github.com/chihaya/chihaya/drivers/tracker/memory"
)

func loadTestData(tkr *Tracker) (err error) {
	conn, err := tkr.tp.Get()
	if err != nil {
		return
	}

	users := []string{
		"vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv1",
		"vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv2",
		"vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv3",
	}

	for i, passkey := range users {
		err = conn.PutUser(&models.User{
			ID:      uint64(i + 1),
			Passkey: passkey,
		})

		if err != nil {
			return
		}
	}

	err = conn.PutClient("TR2820")
	if err != nil {
		return
	}

	hash, err := url.QueryUnescape(infoHash)
	if err != nil {
		return
	}

	torrent := &models.Torrent{
		ID:       1,
		Infohash: hash,
		Seeders:  make(map[string]models.Peer),
		Leechers: make(map[string]models.Peer),
	}

	err = conn.PutTorrent(torrent)
	if err != nil {
		return
	}

	err = conn.PutLeecher(torrent.Infohash, &models.Peer{
		ID:        "-TR2820-vvvvvvvvvvv1",
		UserID:    1,
		TorrentID: torrent.ID,
		IP:        net.ParseIP("127.0.0.1"),
		Port:      34000,
		Left:      0,
	})
	if err != nil {
		return
	}

	err = conn.PutLeecher(torrent.Infohash, &models.Peer{
		ID:        "-TR2820-vvvvvvvvvvv3",
		UserID:    3,
		TorrentID: torrent.ID,
		IP:        net.ParseIP("::1"),
		Port:      34000,
		Left:      0,
	})
	return
}

func testRoute(cfg *config.Config, path string) ([]byte, error) {
	tkr, err := NewTracker(cfg)
	if err != nil {
		return nil, err
	}

	err = loadTestData(tkr)
	if err != nil {
		return nil, err
	}

	srv := httptest.NewServer(setupRoutes(tkr, cfg))
	defer srv.Close()

	resp, err := http.Get(srv.URL + path)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func TestPrivateAnnounce(t *testing.T) {
	cfg := config.DefaultConfig
	cfg.Private = true

	path := "/users/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv2/announce?info_hash=%89%d4%bcR%11%16%ca%1dB%a2%f3%0d%1f%27M%94%e4h%1d%af&peer_id=-TR2820-vvvvvvvvvvv2&port=51413&uploaded=0&downloaded=0&left=0&numwant=1&key=3c8e3319&compact=0"

	expected := bencode.Dict{
		"complete":     int64(1),
		"incomplete":   int64(2),
		"interval":     int64(1800),
		"min interval": int64(900),
		"peers": bencode.List{
			bencode.Dict{
				"ip":      "127.0.0.1",
				"peer id": "-TR2820-vvvvvvvvvvv1",
				"port":    int64(34000),
			},
		},
	}

	response, err := testRoute(&cfg, path)
	if err != nil {
		t.Error(err)
	}
	got, err := bencode.Unmarshal(response)

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("\ngot:    %#v\nwanted: %#v", got, expected)
	}

	path = "/users/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv2/announce?info_hash=%89%d4%bcR%11%16%ca%1dB%a2%f3%0d%1f%27M%94%e4h%1d%af&peer_id=-TR2820-vvvvvvvvvvv2&port=51413&uploaded=0&downloaded=0&left=0&numwant=2&key=3c8e3319&compact=0"

	expected = bencode.Dict{
		"complete":     int64(1),
		"incomplete":   int64(2),
		"interval":     int64(1800),
		"min interval": int64(900),
		"peers": bencode.List{
			bencode.Dict{
				"ip":      "127.0.0.1",
				"peer id": "-TR2820-vvvvvvvvvvv1",
				"port":    int64(34000),
			},
			bencode.Dict{
				"ip":      "::1",
				"peer id": "-TR2820-vvvvvvvvvvv3",
				"port":    int64(34000),
			},
		},
	}

	response, err = testRoute(&cfg, path)
	if err != nil {
		t.Error(err)
	}
	got, err = bencode.Unmarshal(response)

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("\ngot:    %#v\nwanted: %#v", got, expected)
	}
}
<|endoftext|>


<commit_before>package main

type TotalIOPerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewTotalIOPerCommand(collector *IOTopCollector, path string) *TotalIOPerCommand {
	return &TotalIOPerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *TotalIOPerCommand) GetUnits() string {
	return "%"
}

func (t *TotalIOPerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *TotalIOPerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *TotalIOPerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeIOPercent()
	return t.collector.GetUniqKeys(t.samples)
}

type ReadRatePerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewReadRatePerCommand(collector *IOTopCollector, path string) *ReadRatePerCommand {
	return &ReadRatePerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *ReadRatePerCommand) GetUnits() string {
	return "kbps"
}

func (t *ReadRatePerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *ReadRatePerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *ReadRatePerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeDiskReadRate()
	return t.collector.GetUniqKeys(t.samples)
}

type WriteRatePerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewWriteRatePerCommand(collector *IOTopCollector, path string) *WriteRatePerCommand {
	return &WriteRatePerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *WriteRatePerCommand) GetUnits() string {
	return "kbps"
}

func (t *WriteRatePerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *WriteRatePerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *WriteRatePerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeDiskWriteRate()
	return t.collector.GetUniqKeys(t.samples)
}

type SwapinPerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewSwapinPerCommand(collector *IOTopCollector, path string) *SwapinPerCommand {
	return &SwapinPerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *SwapinPerCommand) GetUnits() string {
	return "%"
}

func (t *SwapinPerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *SwapinPerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *SwapinPerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeSwapinPercent()
	return t.collector.GetUniqKeys(t.samples)
}

type RrqmpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRrqmpsPerDevice(processor *DeviceStatsProcessor, path string) *RrqmpsPerDevice {
	return &RrqmpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RrqmpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RrqmpsPerDevice) GetUnits() string {
	return "rrqm/s"
}

func (r *RrqmpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RrqmpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRrqmps()
	return r.processor.GetUniqKeys(r.samples)
}

type WrqmpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWrqmpsPerDevice(processor *DeviceStatsProcessor, path string) *WrqmpsPerDevice {
	return &WrqmpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WrqmpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WrqmpsPerDevice) GetUnits() string {
	return "wrqm/s"
}

func (r *WrqmpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WrqmpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWrqmps()
	return r.processor.GetUniqKeys(r.samples)
}

type RpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRpsPerDevice(processor *DeviceStatsProcessor, path string) *RpsPerDevice {
	return &RpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RpsPerDevice) GetUnits() string {
	return "r/s"
}

func (r *RpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRps()
	return r.processor.GetUniqKeys(r.samples)
}

type WpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWpsPerDevice(processor *DeviceStatsProcessor, path string) *WpsPerDevice {
	return &WpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WpsPerDevice) GetUnits() string {
	return "w/s"
}

func (r *WpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWps()
	return r.processor.GetUniqKeys(r.samples)
}

type RkbpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRkbpsPerDevice(processor *DeviceStatsProcessor, path string) *RkbpsPerDevice {
	return &RkbpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RkbpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RkbpsPerDevice) GetUnits() string {
	return "kb/s"
}

func (r *RkbpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RkbpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRkbps()
	return r.processor.GetUniqKeys(r.samples)
}

type WkbpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWkbpsPerDevice(processor *DeviceStatsProcessor, path string) *WkbpsPerDevice {
	return &WkbpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WkbpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WkbpsPerDevice) GetUnits() string {
	return "kb/s"
}

func (r *WkbpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WkbpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWkbps()
	return r.processor.GetUniqKeys(r.samples)
}

type AvgrqszPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewAvgrqszPerDevice(processor *DeviceStatsProcessor, path string) *AvgrqszPerDevice {
	return &AvgrqszPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *AvgrqszPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *AvgrqszPerDevice) GetUnits() string {
	return "avgrq_sz"
}

func (r *AvgrqszPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *AvgrqszPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeAvgrq_sz()
	return r.processor.GetUniqKeys(r.samples)
}

type AvgquszPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewAvgquszPerDevice(processor *DeviceStatsProcessor, path string) *AvgquszPerDevice {
	return &AvgquszPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *AvgquszPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *AvgquszPerDevice) GetUnits() string {
	return "avgqu_sz"
}

func (r *AvgquszPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *AvgquszPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeAvgqu_sz()
	return r.processor.GetUniqKeys(r.samples)
}

type AwaitPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewAwaitPerDevice(processor *DeviceStatsProcessor, path string) *AwaitPerDevice {
	return &AwaitPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *AwaitPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *AwaitPerDevice) GetUnits() string {
	return "await"
}

func (r *AwaitPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *AwaitPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeAwait()
	return r.processor.GetUniqKeys(r.samples)
}

type RawaitPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRawaitPerDevice(processor *DeviceStatsProcessor, path string) *RawaitPerDevice {
	return &RawaitPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RawaitPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RawaitPerDevice) GetUnits() string {
	return "r_await"
}

func (r *RawaitPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RawaitPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRawait()
	return r.processor.GetUniqKeys(r.samples)
}

type WawaitPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWawaitPerDevice(processor *DeviceStatsProcessor, path string) *WawaitPerDevice {
	return &WawaitPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WawaitPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WawaitPerDevice) GetUnits() string {
	return "w_await"
}

func (r *WawaitPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WawaitPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWawait()
	return r.processor.GetUniqKeys(r.samples)
}

type SvctmPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewSvctmPerDevice(processor *DeviceStatsProcessor, path string) *SvctmPerDevice {
	return &SvctmPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *SvctmPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *SvctmPerDevice) GetUnits() string {
	return "svctm"
}

func (r *SvctmPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *SvctmPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeSvctm()
	return r.processor.GetUniqKeys(r.samples)
}

type UtilPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewUtilPerDevice(processor *DeviceStatsProcessor, path string) *UtilPerDevice {
	return &UtilPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *UtilPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *UtilPerDevice) GetUnits() string {
	return "%"
}

func (r *UtilPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *UtilPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeUtil()
	return r.processor.GetUniqKeys(r.samples)
}
<commit_msg>Fix proper unit naming for metricas<commit_after>package main

type TotalIOPerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewTotalIOPerCommand(collector *IOTopCollector, path string) *TotalIOPerCommand {
	return &TotalIOPerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *TotalIOPerCommand) GetUnits() string {
	return "%"
}

func (t *TotalIOPerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *TotalIOPerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *TotalIOPerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeIOPercent()
	return t.collector.GetUniqKeys(t.samples)
}

type ReadRatePerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewReadRatePerCommand(collector *IOTopCollector, path string) *ReadRatePerCommand {
	return &ReadRatePerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *ReadRatePerCommand) GetUnits() string {
	return "kb/s"
}

func (t *ReadRatePerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *ReadRatePerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *ReadRatePerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeDiskReadRate()
	return t.collector.GetUniqKeys(t.samples)
}

type WriteRatePerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewWriteRatePerCommand(collector *IOTopCollector, path string) *WriteRatePerCommand {
	return &WriteRatePerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *WriteRatePerCommand) GetUnits() string {
	return "kb/s"
}

func (t *WriteRatePerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *WriteRatePerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *WriteRatePerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeDiskWriteRate()
	return t.collector.GetUniqKeys(t.samples)
}

type SwapinPerCommand struct {
	collector *IOTopCollector
	path      string
	samples   map[string]*StatSample
}

func NewSwapinPerCommand(collector *IOTopCollector, path string) *SwapinPerCommand {
	return &SwapinPerCommand{
		collector: collector,
		path:      path,
	}
}

func (t *SwapinPerCommand) GetUnits() string {
	return "%"
}

func (t *SwapinPerCommand) GetName(id string) string {
	return t.path + "/" + id
}

func (t *SwapinPerCommand) GetValue(id string) (float64, error) {
	return t.samples[id].GetAverage(), nil
}

func (t *SwapinPerCommand) GetIdList() []string {
	t.samples = t.collector.GetAndPurgeSwapinPercent()
	return t.collector.GetUniqKeys(t.samples)
}

type RrqmpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRrqmpsPerDevice(processor *DeviceStatsProcessor, path string) *RrqmpsPerDevice {
	return &RrqmpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RrqmpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RrqmpsPerDevice) GetUnits() string {
	return "request/s"
}

func (r *RrqmpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RrqmpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRrqmps()
	return r.processor.GetUniqKeys(r.samples)
}

type WrqmpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWrqmpsPerDevice(processor *DeviceStatsProcessor, path string) *WrqmpsPerDevice {
	return &WrqmpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WrqmpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WrqmpsPerDevice) GetUnits() string {
	return "request/s"
}

func (r *WrqmpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WrqmpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWrqmps()
	return r.processor.GetUniqKeys(r.samples)
}

type RpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRpsPerDevice(processor *DeviceStatsProcessor, path string) *RpsPerDevice {
	return &RpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RpsPerDevice) GetUnits() string {
	return "r/s"
}

func (r *RpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRps()
	return r.processor.GetUniqKeys(r.samples)
}

type WpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWpsPerDevice(processor *DeviceStatsProcessor, path string) *WpsPerDevice {
	return &WpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WpsPerDevice) GetUnits() string {
	return "w/s"
}

func (r *WpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWps()
	return r.processor.GetUniqKeys(r.samples)
}

type RkbpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRkbpsPerDevice(processor *DeviceStatsProcessor, path string) *RkbpsPerDevice {
	return &RkbpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RkbpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RkbpsPerDevice) GetUnits() string {
	return "kb/s"
}

func (r *RkbpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RkbpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRkbps()
	return r.processor.GetUniqKeys(r.samples)
}

type WkbpsPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWkbpsPerDevice(processor *DeviceStatsProcessor, path string) *WkbpsPerDevice {
	return &WkbpsPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WkbpsPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WkbpsPerDevice) GetUnits() string {
	return "kb/s"
}

func (r *WkbpsPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WkbpsPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWkbps()
	return r.processor.GetUniqKeys(r.samples)
}

type AvgrqszPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewAvgrqszPerDevice(processor *DeviceStatsProcessor, path string) *AvgrqszPerDevice {
	return &AvgrqszPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *AvgrqszPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *AvgrqszPerDevice) GetUnits() string {
	return "sectors"
}

func (r *AvgrqszPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *AvgrqszPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeAvgrq_sz()
	return r.processor.GetUniqKeys(r.samples)
}

type AvgquszPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewAvgquszPerDevice(processor *DeviceStatsProcessor, path string) *AvgquszPerDevice {
	return &AvgquszPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *AvgquszPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *AvgquszPerDevice) GetUnits() string {
	return "count"
}

func (r *AvgquszPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *AvgquszPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeAvgqu_sz()
	return r.processor.GetUniqKeys(r.samples)
}

type AwaitPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewAwaitPerDevice(processor *DeviceStatsProcessor, path string) *AwaitPerDevice {
	return &AwaitPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *AwaitPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *AwaitPerDevice) GetUnits() string {
	return "ms"
}

func (r *AwaitPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *AwaitPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeAwait()
	return r.processor.GetUniqKeys(r.samples)
}

type RawaitPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewRawaitPerDevice(processor *DeviceStatsProcessor, path string) *RawaitPerDevice {
	return &RawaitPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *RawaitPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *RawaitPerDevice) GetUnits() string {
	return "ms"
}

func (r *RawaitPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *RawaitPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeRawait()
	return r.processor.GetUniqKeys(r.samples)
}

type WawaitPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewWawaitPerDevice(processor *DeviceStatsProcessor, path string) *WawaitPerDevice {
	return &WawaitPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *WawaitPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *WawaitPerDevice) GetUnits() string {
	return "ms"
}

func (r *WawaitPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *WawaitPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeWawait()
	return r.processor.GetUniqKeys(r.samples)
}

type SvctmPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewSvctmPerDevice(processor *DeviceStatsProcessor, path string) *SvctmPerDevice {
	return &SvctmPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *SvctmPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *SvctmPerDevice) GetUnits() string {
	return "ms"
}

func (r *SvctmPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *SvctmPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeSvctm()
	return r.processor.GetUniqKeys(r.samples)
}

type UtilPerDevice struct {
	processor *DeviceStatsProcessor
	path      string
	samples   map[string]ISampleStats
}

func NewUtilPerDevice(processor *DeviceStatsProcessor, path string) *UtilPerDevice {
	return &UtilPerDevice{
		processor: processor,
		path:      path,
	}
}

func (r *UtilPerDevice) GetName(id string) string {
	return r.path + "/" + id
}

func (r *UtilPerDevice) GetUnits() string {
	return "%"
}

func (r *UtilPerDevice) GetValue(id string) (float64, error) {
	return r.samples[id].GetAverage(), nil
}

func (r *UtilPerDevice) GetIdList() []string {
	r.samples = r.processor.GetAndPurgeUtil()
	return r.processor.GetUniqKeys(r.samples)
}
<|endoftext|>


<commit_before>package socket

import (
	"github.com/exitcodezero/picloud/hub"
	"github.com/exitcodezero/picloud/message"
	"github.com/gorilla/context"
	"github.com/gorilla/websocket"
	"net/http"
	"time"
)

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

func writeSocket(socket *websocket.Conn, c *hub.Connection) {
	for {
		m := <-c.Out
		socket.WriteJSON(&m)
	}
}

// Handler handles websocket connections at /connect
func Handler(w http.ResponseWriter, r *http.Request) {

	clientName, _ := context.Get(r, "ClientName").(string)

	// Upgrade the request
	socket, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		panic(err)
	}
	defer socket.Close()

	// Create a Connection instance
	c := hub.NewConnection(socket.RemoteAddr().String(), clientName)
	hub.Manager.RegisterConnection(&c)
	defer hub.Manager.Cleanup(&c)

	// Start pushing outbound messages from a goroutine
	go writeSocket(socket, &c)

	// Handle inbound messages
	for {
		m := message.SocketMessage{}
		m.CreatedAt = time.Now().UTC()

		err = socket.ReadJSON(&m)
		if err != nil {
			break
		}

		switch m.Action {
		case "publish":
			hub.Manager.Publish(m)
		case "subscribe":
			hub.Manager.Subscribe(m.Event, &c)
		case "unsubscribe":
			hub.Manager.Unsubscribe(m.Event, &c)
		case "unsubscribe:all":
			hub.Manager.UnsubscribeAll(&c)
		}
	}
}
<commit_msg>Don't forget to clear<commit_after>package socket

import (
	"github.com/exitcodezero/picloud/hub"
	"github.com/exitcodezero/picloud/message"
	"github.com/gorilla/context"
	"github.com/gorilla/websocket"
	"net/http"
	"time"
)

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

func writeSocket(socket *websocket.Conn, c *hub.Connection) {
	for {
		m := <-c.Out
		socket.WriteJSON(&m)
	}
}

// Handler handles websocket connections at /connect
func Handler(w http.ResponseWriter, r *http.Request) {

	clientName, _ := context.Get(r, "ClientName").(string)
	context.Clear(r)

	// Upgrade the request
	socket, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		panic(err)
	}
	defer socket.Close()

	// Create a Connection instance
	c := hub.NewConnection(socket.RemoteAddr().String(), clientName)
	hub.Manager.RegisterConnection(&c)
	defer hub.Manager.Cleanup(&c)

	// Start pushing outbound messages from a goroutine
	go writeSocket(socket, &c)

	// Handle inbound messages
	for {
		m := message.SocketMessage{}
		m.CreatedAt = time.Now().UTC()

		err = socket.ReadJSON(&m)
		if err != nil {
			break
		}

		switch m.Action {
		case "publish":
			hub.Manager.Publish(m)
		case "subscribe":
			hub.Manager.Subscribe(m.Event, &c)
		case "unsubscribe":
			hub.Manager.Unsubscribe(m.Event, &c)
		case "unsubscribe:all":
			hub.Manager.UnsubscribeAll(&c)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package weighted_balancing_test

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/gribigo/chk"
	"github.com/openconfig/gribigo/fluent"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/ondatra/telemetry"
)

// nextHopsEvenly generates []nexthop that distributes weights evenly
// across the atePorts.
func nextHopsEvenly(atePorts []*ondatra.Port) []nextHop {
	var nexthops []nextHop
	for _, ap := range atePorts {
		ateid := "ate:" + ap.ID()
		if ateid == ateSrcPort {
			continue
		}
		nexthops = append(nexthops, nextHop{ateid, 1, 0})
	}
	return nexthops
}

// portWantsEvenly generates wanted weights assuming that the traffic
// should be evenly distributed across the ports that are still up.
func portWantsEvenly(atePorts []*ondatra.Port, numUps int) []float64 {
	weights := make([]float64, len(atePorts))
	x := 1.0 / float64(numUps)
	for i := 1; i <= numUps; i++ {
		weights[i] = x
	}
	return weights
}

func testNextHopRemaining(
	t *testing.T,
	numUps int,
	dut *ondatra.DUTDevice,
	ate *ondatra.ATEDevice,
	top *ondatra.ATETopology,
) {
	// Generate and analyze traffic.
	atePorts, inPkts, outPkts := generateTraffic(t, ate, top)
	t.Logf("atePorts = %v", atePorts)
	t.Logf("inPkts = %v", inPkts)
	t.Logf("outPkts = %v", outPkts)

	got, inSum := normalize(inPkts)
	want := portWantsEvenly(atePorts, numUps)
	t.Logf("inPkts normalized got: %v", got)
	t.Logf("weights want: %v", want)

	// Report diagnosis.
	t.Run("Ratio", func(t *testing.T) {
		if diff := cmp.Diff(want, got, approxOpt); diff != "" {
			t.Errorf("Packet distribution ratios -want,+got:\n%s", diff)
		}
	})
	t.Run("Loss", func(t *testing.T) {
		if outPkts[0] > inSum {
			t.Errorf("Traffic flow sent %d packets, received only %d",
				outPkts[0], inSum)
		}
	})
}

const portFlapDesc = "With NHG 10 containing 8 next-hops, with a weight of 1 assigned to each, sequentially remove each next-hop by turning down the port at the ATE (invalidates nexthop), ensure that traffic is rebalanced across remaining NHs until only one NH remains"

func TestPortFlap(t *testing.T) {
	// Dial gRIBI
	ctx := context.Background()
	dut := ondatra.DUT(t, "dut")
	gribic := dut.RawAPIs().GRIBI().Default(t)

	// Configure the DUT
	configureDUT(t, dut)

	// Configure the ATE
	ate := ondatra.ATE(t, "ate")
	top := configureATE(t, ate)
	top.Push(t).StartProtocols(t)

	// Create nexthops across the dst atePorts.
	atePorts := sortPorts(ate.Ports())
	nexthops := nextHopsEvenly(atePorts)
	t.Logf("Description: %s", portFlapDesc)
	t.Logf("NextHops: %+v", nexthops)

	// Configure the gRIBI client.
	c := fluent.NewClient()
	c.Connection().
		WithStub(gribic).
		WithRedundancyMode(fluent.ElectedPrimaryClient).
		WithInitialElectionID(1 /* low */, 0 /* hi */) // ID must be > 0.
	c.Start(ctx, t)
	defer c.Stop(t)
	c.StartSending(ctx, t)
	if err := awaitTimeout(ctx, c, t, time.Minute); err != nil {
		t.Fatalf("Await got error during session negotiation: %v", err)
	}

	ents, wants := buildNextHops(t, nexthops, 1)

	c.Modify().AddEntry(t, ents...)
	if err := awaitTimeout(ctx, c, t, time.Minute); err != nil {
		t.Fatalf("Await got error for entries: %v", err)
	}

	res := c.Results(t)

	for _, want := range wants {
		t.Logf("Checking for result: %+v", want)
		chk.HasResult(t, res, want)
	}

	// Turn down ports one by one.
	dt := dut.Telemetry()

	for i := len(atePorts); i >= 2; i-- {
		numUps := i - 1
		numDowns := len(atePorts) - i
		testName := fmt.Sprintf("%d Up, %d Down", numUps, numDowns)

		t.Run(testName, func(t *testing.T) {
			if i < len(atePorts) {
				t.Logf("Bringing down ate port: %v", atePorts[i])
				ate.Operations().NewSetInterfaceState().
					WithPhysicalInterface(atePorts[i]).
					WithStateEnabled(false).
					Operate(t)

				// ATE and DUT ports in the linked pair have the same ID(), but
				// they are mapped to different Name().
				dp := dut.Port(t, atePorts[i].ID())
				dip := dt.Interface(dp.Name())
				t.Logf("Awaiting DUT port down: %v", dp)
				dip.OperStatus().Await(t, time.Minute, telemetry.Interface_OperStatus_DOWN)
				t.Log("Port is down.")
			}
			testNextHopRemaining(t, numUps, dut, ate, top)
			debugGRIBI(t, dut)
		})
	}
}
<commit_msg>Migrate uses of ate.Operations().SetInterfaceState to ate.Actions().SetPortState (#250)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package weighted_balancing_test

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/gribigo/chk"
	"github.com/openconfig/gribigo/fluent"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/ondatra/telemetry"
)

// nextHopsEvenly generates []nexthop that distributes weights evenly
// across the atePorts.
func nextHopsEvenly(atePorts []*ondatra.Port) []nextHop {
	var nexthops []nextHop
	for _, ap := range atePorts {
		ateid := "ate:" + ap.ID()
		if ateid == ateSrcPort {
			continue
		}
		nexthops = append(nexthops, nextHop{ateid, 1, 0})
	}
	return nexthops
}

// portWantsEvenly generates wanted weights assuming that the traffic
// should be evenly distributed across the ports that are still up.
func portWantsEvenly(atePorts []*ondatra.Port, numUps int) []float64 {
	weights := make([]float64, len(atePorts))
	x := 1.0 / float64(numUps)
	for i := 1; i <= numUps; i++ {
		weights[i] = x
	}
	return weights
}

func testNextHopRemaining(
	t *testing.T,
	numUps int,
	dut *ondatra.DUTDevice,
	ate *ondatra.ATEDevice,
	top *ondatra.ATETopology,
) {
	// Generate and analyze traffic.
	atePorts, inPkts, outPkts := generateTraffic(t, ate, top)
	t.Logf("atePorts = %v", atePorts)
	t.Logf("inPkts = %v", inPkts)
	t.Logf("outPkts = %v", outPkts)

	got, inSum := normalize(inPkts)
	want := portWantsEvenly(atePorts, numUps)
	t.Logf("inPkts normalized got: %v", got)
	t.Logf("weights want: %v", want)

	// Report diagnosis.
	t.Run("Ratio", func(t *testing.T) {
		if diff := cmp.Diff(want, got, approxOpt); diff != "" {
			t.Errorf("Packet distribution ratios -want,+got:\n%s", diff)
		}
	})
	t.Run("Loss", func(t *testing.T) {
		if outPkts[0] > inSum {
			t.Errorf("Traffic flow sent %d packets, received only %d",
				outPkts[0], inSum)
		}
	})
}

const portFlapDesc = "With NHG 10 containing 8 next-hops, with a weight of 1 assigned to each, sequentially remove each next-hop by turning down the port at the ATE (invalidates nexthop), ensure that traffic is rebalanced across remaining NHs until only one NH remains"

func TestPortFlap(t *testing.T) {
	// Dial gRIBI
	ctx := context.Background()
	dut := ondatra.DUT(t, "dut")
	gribic := dut.RawAPIs().GRIBI().Default(t)

	// Configure the DUT
	configureDUT(t, dut)

	// Configure the ATE
	ate := ondatra.ATE(t, "ate")
	top := configureATE(t, ate)
	top.Push(t).StartProtocols(t)

	// Create nexthops across the dst atePorts.
	atePorts := sortPorts(ate.Ports())
	nexthops := nextHopsEvenly(atePorts)
	t.Logf("Description: %s", portFlapDesc)
	t.Logf("NextHops: %+v", nexthops)

	// Configure the gRIBI client.
	c := fluent.NewClient()
	c.Connection().
		WithStub(gribic).
		WithRedundancyMode(fluent.ElectedPrimaryClient).
		WithInitialElectionID(1 /* low */, 0 /* hi */) // ID must be > 0.
	c.Start(ctx, t)
	defer c.Stop(t)
	c.StartSending(ctx, t)
	if err := awaitTimeout(ctx, c, t, time.Minute); err != nil {
		t.Fatalf("Await got error during session negotiation: %v", err)
	}

	ents, wants := buildNextHops(t, nexthops, 1)

	c.Modify().AddEntry(t, ents...)
	if err := awaitTimeout(ctx, c, t, time.Minute); err != nil {
		t.Fatalf("Await got error for entries: %v", err)
	}

	res := c.Results(t)

	for _, want := range wants {
		t.Logf("Checking for result: %+v", want)
		chk.HasResult(t, res, want)
	}

	// Turn down ports one by one.
	dt := dut.Telemetry()

	for i := len(atePorts); i >= 2; i-- {
		numUps := i - 1
		numDowns := len(atePorts) - i
		testName := fmt.Sprintf("%d Up, %d Down", numUps, numDowns)

		t.Run(testName, func(t *testing.T) {
			if i < len(atePorts) {
				t.Logf("Bringing down ate port: %v", atePorts[i])
				ate.Actions().
					NewSetPortState().
					WithPort(atePorts[i]).
					WithEnabled(false).
					Send(t)

				// ATE and DUT ports in the linked pair have the same ID(), but
				// they are mapped to different Name().
				dp := dut.Port(t, atePorts[i].ID())
				dip := dt.Interface(dp.Name())
				t.Logf("Awaiting DUT port down: %v", dp)
				dip.OperStatus().Await(t, time.Minute, telemetry.Interface_OperStatus_DOWN)
				t.Log("Port is down.")
			}
			testNextHopRemaining(t, numUps, dut, ate, top)
			debugGRIBI(t, dut)
		})
	}
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "STREAM" {
          log.Println("send", nntp.MessageID(), "to", feed.name)
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "READER" {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<commit_msg>prefer stream<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "STREAM" {
          log.Println("send", nntp.MessageID(), "to", feed.name)
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "READER" {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, c, "stream")
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<|endoftext|>


<commit_before>//
// markup.go
// memeposting markup parser
//
package srnd

import (
  "html"
  "regexp"
  "strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_external_link = regexp.MustCompile(`((https?|ftp):\/\/[\w\d\.\-_]*\/[\w\d\.\/\-_\?]*)`);
var re_backlink = regexp.MustCompile(`>> ?([0-9a-f]{10})`)

func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if len(line) > 0 {
    if strings.HasPrefix(line, ">") && ! strings.HasPrefix(line, ">>") {
      // le ebin meme arrows
      markup += "
"
      markup += html.EscapeString(line)
      markup += "
"
    } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
      // redtext
      markup += "
"
      markup += html.EscapeString(line[2:len(line)-2])
      markup += "
"
    } else {
      // regular line
      markup += "
"
      // for each word
      for _, word := range strings.Split(line, " ") {
        // check for backlink
        link := re_backlink.FindString(word)
        if len(link) > 0 {
          link = strings.Trim(link[2:], " ")
          url := template.findLink(link)
          if len(url) == 0 {
            url = "#"
          }
          // backlink exists
          markup += `<a href="`+url+`">>>` + link + ""

        } else {
          // linkify it as needed
          line = html.EscapeString(word)
          markup += re_external_link.ReplaceAllString(line, `<a href="$1">$1</a>`)
        }
        markup += " "
      }
      markup += "
"
    }
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += "
"
  markup += html.EscapeString(line)
  markup += "
"
  return
}

func memeposting(src string) (markup string) {
  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"
    } else if strings.Count(line, "[spoiler]") > 0 {
      // spoiler tag
      found_tag = true
      tag = "spoiler"
    } else if strings.Count(line, "[psy]") > 0 {
      // psy tag
      found_tag = true
      tag = "psy"
    }
    if found_tag {
      // collect content of tag
      tag_content += line + "\n"
      // end of our tag ?
      if strings.Count(line, "[/"+tag+"]") == 1 {
        // yah
        found_tag = false
        var tag_open, tag_close string
        if tag == "code" {
          tag_open = "
"
          tag_close = "
"
        } else if tag == "spoiler" {
          tag_open = ""
          tag_close = ""
        } else if tag == "psy" {
          tag_open = "
"
          tag_close = "
"
        }
        markup += tag_open
        // remove open tag, only once so we can have a code tag verbatum inside
        tag_content = strings.Replace(tag_content, "["+tag+"]", "", 1)
        // remove all close tags, should only have 1
        tag_content = strings.Replace(tag_content, "[/"+tag+"]", "", -1)
        // make into lines
        for _, tag_line := range strings.Split(tag_content, "\n") {
          if tag == "code" {
            markup += formatcodeline(tag_line)
          } else {
            markup += formatline(tag_line)
          }
        }
        // close pre tag
        markup += tag_close
        // reset content buffer
        tag_content = ""
      }
      // next line
      continue
    }
    // format line regularlly
    markup += formatline(line)
  }
  // flush the rest of an incomplete code tag
  for _, line := range strings.Split(tag_content, "\n") {
    markup += formatline(line)
  }
  return
}
<commit_msg>try different backlink detections<commit_after>//
// markup.go
// memeposting markup parser
//
package srnd

import (
  "html"
  "regexp"
  "strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_external_link = regexp.MustCompile(`((https?|ftp):\/\/[\w\d\.\-_]*\/[\w\d\.\/\-_\?]*)`);
var re_backlink = regexp.MustCompile(`>> ?([0-9a-f]{10})`)

// parse backlink
func backlink(word string) (markup string) {
  link := re_backlink.FindString(word)
  if len(link) > 0 {
    link = strings.Trim(link[2:], " ")
    url := template.findLink(link)
    if len(url) == 0 {
      return ">>" + link[2:]
    }
    // backlink exists
    return`<a href="`+url+`">>>` + link + ""
  }
  return word
}

func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if len(line) > 0 {
    if strings.HasPrefix(line, ">") || ( strings.HasPrefix(line, ">>") && ! re_backlink.MatchString(strings.Split(line, " ")[0])) {
      // le ebin meme arrows
      markup += "
"
      markup += html.EscapeString(line)
      markup += "
"
    } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
      // redtext
      markup += "
"
      markup += html.EscapeString(line[2:len(line)-2])
      markup += "
"
    } else {
      // regular line
      markup += "
"
      // for each word
      for _, word := range strings.Split(line, " ") {
        // check for backlink
        if re_backlink.MatchString(word) {
          markup += backlink(word)
        } else {
          // linkify it as needed
          line = html.EscapeString(word)
          markup += re_external_link.ReplaceAllString(line, `<a href="$1">$1</a>`)
        }
        markup += " "
      }
      markup += "
"
    }
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += "
"
  markup += html.EscapeString(line)
  markup += "
"
  return
}

func memeposting(src string) (markup string) {
  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"
    } else if strings.Count(line, "[spoiler]") > 0 {
      // spoiler tag
      found_tag = true
      tag = "spoiler"
    } else if strings.Count(line, "[psy]") > 0 {
      // psy tag
      found_tag = true
      tag = "psy"
    }
    if found_tag {
      // collect content of tag
      tag_content += line + "\n"
      // end of our tag ?
      if strings.Count(line, "[/"+tag+"]") == 1 {
        // yah
        found_tag = false
        var tag_open, tag_close string
        if tag == "code" {
          tag_open = "
"
          tag_close = "
"
        } else if tag == "spoiler" {
          tag_open = ""
          tag_close = ""
        } else if tag == "psy" {
          tag_open = "
"
          tag_close = "
"
        }
        markup += tag_open
        // remove open tag, only once so we can have a code tag verbatum inside
        tag_content = strings.Replace(tag_content, "["+tag+"]", "", 1)
        // remove all close tags, should only have 1
        tag_content = strings.Replace(tag_content, "[/"+tag+"]", "", -1)
        // make into lines
        for _, tag_line := range strings.Split(tag_content, "\n") {
          if tag == "code" {
            markup += formatcodeline(tag_line)
          } else {
            markup += formatline(tag_line)
          }
        }
        // close pre tag
        markup += tag_close
        // reset content buffer
        tag_content = ""
      }
      // next line
      continue
    }
    // format line regularlly
    markup += formatline(line)
  }
  // flush the rest of an incomplete code tag
  for _, line := range strings.Split(tag_content, "\n") {
    markup += formatline(line)
  }
  return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"
)

func main() {
	service := ":9000"
	tcpAddr, error := net.ResolveTCPAddr("ipv4", service)
	check_error(error)

	listener, error := net.ListenTCP("tcp", tcpAddr)
	check_error(error)

	for {
		conn, error := listener.Accept()
		if error != nil {
			continue
		}

		go handle_connection(conn)
	}
}

func handle_connection(conn net.Conn) {
	// close connection on exit
	defer conn.Close()
	buffer := make([]byte, 1024)
	length, error := conn.Read(buffer)
	if error != nil {
		// do something good to clean up?
	} else {
		// strip away newline and null termination
		command := string(buffer[:length-2])
		switch {

		case command == "gerrit strem-events":
			conn.Write([]byte("gerrit subcommand\n"))
		default:
			conn.Write([]byte(command))
		}
	}
}

func check_error(error error) {
	if error != nil {
		fmt.Fprintf(os.Stderr, "Fatal error %s", error.Error())
		os.Exit(1)
	}
}
<commit_msg>Fix spelling mistake<commit_after>package main

import (
	"fmt"
	"net"
	"os"
)

func main() {
	service := ":9000"
	tcpAddr, error := net.ResolveTCPAddr("ipv4", service)
	check_error(error)

	listener, error := net.ListenTCP("tcp", tcpAddr)
	check_error(error)

	for {
		conn, error := listener.Accept()
		if error != nil {
			continue
		}

		go handle_connection(conn)
	}
}

func handle_connection(conn net.Conn) {
	// close connection on exit
	defer conn.Close()
	buffer := make([]byte, 1024)
	length, error := conn.Read(buffer)
	if error != nil {
		// do something good to clean up?
	} else {
		// strip away newline and null termination
		command := string(buffer[:length-2])
		switch {

		case command == "gerrit stream-events":
			conn.Write([]byte("gerrit subcommand\n"))
		default:
			conn.Write([]byte(command))
		}
	}
}

func check_error(error error) {
	if error != nil {
		fmt.Fprintf(os.Stderr, "Fatal error %s", error.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package converterControllers

import (
	. "eaciit/wfdemo-git/library/helper"
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/controllers"
	"os"
	"strings"
	"time"

	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"github.com/tealeg/xlsx"
)

// ConvPermanentMetTowerNewFormat
type ConvPermanentMetTowerNewFormat struct {
	*BaseController
}

/*func (d *ConvPermanentMetTowerNewFormat) Validate(base *BaseController) {
	list := []tk.M{}
	d.BaseController = base
	ctx := d.BaseController.Ctx
	_ = ctx

	csr, e := ctx.Connection.NewQuery().
		From(new(MetTower).TableName()).
		Order("line").
		Cursor(nil)

	if e != nil {
		log.Printf("err: %v \n", e.Error())
	}

	e = csr.Fetch(&list, 0, false)
	csr.Close()

	mp := map[int]tk.M{}

	for _, val := range list {
		mp[val.GetInt("line")] = val
	}

	for i := 1; i < 4322; i++ {
		x := mp[i]

		if x == nil {
			log.Printf("%#v \n", i)
		} else {
		}
	}

}*/

// Generate
func (d *ConvPermanentMetTowerNewFormat) Generate(base *BaseController) {
	folderName := "met"
	funcName := "Converting Met Tower Data"
	count := 0
	total := 0
	errorLine := tk.M{}
	if base != nil {
		d.BaseController = base
		ctx := d.BaseController.Ctx
		_ = ctx
		dataSources, path := base.GetDataSource(folderName)
		tk.Println("Converting Met Tower Data from Excel File..")
		for _, source := range dataSources {
			if !strings.Contains(source.Name(), "~") {
				tk.Println(path + "/" + source.Name())
				file, e := xlsx.OpenFile(path + "/" + source.Name())
				if e != nil {
					ErrorHandler(e, funcName)
					os.Exit(0)
				}

				for _, sheet := range file.Sheet {
					for idx, row := range sheet.Rows {
						errorList := []error{}
						if idx > 0 {
							if len(row.Cells) > 0 {
								data := new(MetTower).New()
								data.Line = (idx + 1)

								tmp, e := row.Cells[0].Float()
								ErrorLog(e, funcName, errorList)
								row.Cells[0].SetDateTimeWithFormat(tmp, time.UnixDate)
								strDate, e := row.Cells[0].FormattedValue()
								ErrorLog(e, funcName, errorList)

								dtSplit := strings.Split(strings.Replace(strDate, "  ", " ", 1), " ")

								date, e := time.Parse("Jan 2 15:04:05 2006", dtSplit[1]+" "+dtSplit[2]+" "+dtSplit[3]+" "+dtSplit[5])

								// tk.Printf("%v | %v | %v \n", strDate, date.String(), e.Error())
								if e != nil {
									tk.Printf("%v | %v | %v \n", idx+1, e.Error(), strDate)
								}

								data.TimeStamp = date.UTC()
								data.DateInfo = GetDateInfo(date.UTC())

								/*for _, valx := range row.Cells {
									y, _ := valx.Float()
									log.Printf("cell: %#v \n", y)
								}*/

								if len(row.Cells) > 1 {
									VHubWS90mAvg, err := row.Cells[1].Float()
									if err != nil {
										data.VHubWS90mAvg = VHubWS90mAvg
									}

									ErrorLog(e, funcName, errorList)

									VHubWS90mStdDev, err := row.Cells[2].Float()
									if err != nil {
										data.VHubWS90mStdDev = VHubWS90mStdDev
									}
									ErrorLog(e, funcName, errorList)

									VRefWS88mAvg, err := row.Cells[3].Float()
									if err != nil {
										data.VRefWS88mAvg = VRefWS88mAvg
									}
									ErrorLog(e, funcName, errorList)

									VRefWS88mStdDev, err := row.Cells[4].Float()
									if err != nil {
										data.VRefWS88mStdDev = VRefWS88mStdDev
									}
									ErrorLog(e, funcName, errorList)

									VTipWS42mAvg, err := row.Cells[5].Float()
									if err != nil {
										data.VTipWS42mAvg = VTipWS42mAvg
									}
									ErrorLog(e, funcName, errorList)

									VTipWS42mStdDev, err := row.Cells[6].Float()
									if err != nil {
										data.VTipWS42mStdDev = VTipWS42mStdDev
									}
									ErrorLog(e, funcName, errorList)

									DHubWD88mAvg, err := row.Cells[7].Float()
									if err != nil {
										data.DHubWD88mAvg = DHubWD88mAvg
									}
									ErrorLog(e, funcName, errorList)

									DRefWD86mAvg, err := row.Cells[8].Float()
									if err != nil {
										data.DRefWD86mAvg = DRefWD86mAvg
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mAvg, err := row.Cells[9].Float()
									if err != nil {
										data.TRefHRefHumid855mAvg = TRefHRefHumid855mAvg
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mStdDev, err := row.Cells[10].Float()
									if err != nil {
										data.TRefHRefHumid855mStdDev = TRefHRefHumid855mStdDev
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mMax, err := row.Cells[11].Float()
									if err != nil {
										data.TRefHRefHumid855mMax = TRefHRefHumid855mMax
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mMin, err := row.Cells[12].Float()
									if err != nil {
										data.TRefHRefHumid855mMin = TRefHRefHumid855mMin
									}
									ErrorLog(e, funcName, errorList)

									BaroAirPress855mAvg, err := row.Cells[13].Float()
									if err != nil {
										data.BaroAirPress855mAvg = BaroAirPress855mAvg
									}
									ErrorLog(e, funcName, errorList)
								}

								if e != nil {
									errorLine.Set(tk.ToString(idx+1), errorList)
								} else {
									e = ctx.Insert(data)
									ErrorLog(e, funcName, errorList)
								}
							}
						} else {
							if idx != 0 {
								errorLine.Set(tk.ToString(idx+1), errorList)
							}
						}
					}
				}
				total += count
				tk.Printf("count: %v \n", total)
				tk.Printf("count line error: %v \n", len(errorLine))
				if len(errorLine) > 0 {
					WriteErrors(errorLine, source.Name())
				}
			}
		}
	}
}
<commit_msg>bug fixing on new met tower converter<commit_after>package converterControllers

import (
	. "eaciit/wfdemo-git/library/helper"
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/controllers"
	"os"
	"strings"
	"time"

	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"github.com/tealeg/xlsx"
)

// ConvPermanentMetTowerNewFormat
type ConvPermanentMetTowerNewFormat struct {
	*BaseController
}

/*func (d *ConvPermanentMetTowerNewFormat) Validate(base *BaseController) {
	list := []tk.M{}
	d.BaseController = base
	ctx := d.BaseController.Ctx
	_ = ctx

	csr, e := ctx.Connection.NewQuery().
		From(new(MetTower).TableName()).
		Order("line").
		Cursor(nil)

	if e != nil {
		log.Printf("err: %v \n", e.Error())
	}

	e = csr.Fetch(&list, 0, false)
	csr.Close()

	mp := map[int]tk.M{}

	for _, val := range list {
		mp[val.GetInt("line")] = val
	}

	for i := 1; i < 4322; i++ {
		x := mp[i]

		if x == nil {
			log.Printf("%#v \n", i)
		} else {
		}
	}

}*/

// Generate
func (d *ConvPermanentMetTowerNewFormat) Generate(base *BaseController) {
	folderName := "met"
	funcName := "Converting Met Tower Data"
	count := 0
	total := 0
	errorLine := tk.M{}
	if base != nil {
		d.BaseController = base
		ctx := d.BaseController.Ctx
		_ = ctx
		dataSources, path := base.GetDataSource(folderName)
		tk.Println("Converting Met Tower Data from Excel File..")
		for _, source := range dataSources {
			if !strings.Contains(source.Name(), "~") {
				tk.Println(path + "/" + source.Name())
				file, e := xlsx.OpenFile(path + "/" + source.Name())
				if e != nil {
					ErrorHandler(e, funcName)
					os.Exit(0)
				}

				for _, sheet := range file.Sheet {
					for idx, row := range sheet.Rows {
						errorList := []error{}
						if idx > 0 {
							if len(row.Cells) > 0 {
								data := new(MetTower).New()
								data.Line = (idx + 1)

								tmp, e := row.Cells[0].Float()
								ErrorLog(e, funcName, errorList)
								row.Cells[0].SetDateTimeWithFormat(tmp, time.UnixDate)
								strDate, e := row.Cells[0].FormattedValue()
								ErrorLog(e, funcName, errorList)

								dtSplit := strings.Split(strings.Replace(strDate, "  ", " ", 1), " ")

								date, e := time.Parse("Jan 2 15:04:05 2006", dtSplit[1]+" "+dtSplit[2]+" "+dtSplit[3]+" "+dtSplit[5])

								// tk.Printf("%v | %v | %v \n", strDate, date.String(), e.Error())
								if e != nil {
									tk.Printf("%v | %v | %v \n", idx+1, e.Error(), strDate)
								}

								data.TimeStamp = date.UTC()
								data.DateInfo = GetDateInfo(date.UTC())

								/*for _, valx := range row.Cells {
									y, _ := valx.Float()
									log.Printf("cell: %#v \n", y)
								}*/

								if len(row.Cells) > 1 {
									VHubWS90mAvg, err := row.Cells[1].Float()
									if err == nil {
										data.VHubWS90mAvg = VHubWS90mAvg
									}

									ErrorLog(e, funcName, errorList)

									VHubWS90mStdDev, err := row.Cells[2].Float()
									if err == nil {
										data.VHubWS90mStdDev = VHubWS90mStdDev
									}
									ErrorLog(e, funcName, errorList)

									VRefWS88mAvg, err := row.Cells[3].Float()
									if err == nil {
										data.VRefWS88mAvg = VRefWS88mAvg
									}
									ErrorLog(e, funcName, errorList)

									VRefWS88mStdDev, err := row.Cells[4].Float()
									if err == nil {
										data.VRefWS88mStdDev = VRefWS88mStdDev
									}
									ErrorLog(e, funcName, errorList)

									VTipWS42mAvg, err := row.Cells[5].Float()
									if err == nil {
										data.VTipWS42mAvg = VTipWS42mAvg
									}
									ErrorLog(e, funcName, errorList)

									VTipWS42mStdDev, err := row.Cells[6].Float()
									if err == nil {
										data.VTipWS42mStdDev = VTipWS42mStdDev
									}
									ErrorLog(e, funcName, errorList)

									DHubWD88mAvg, err := row.Cells[7].Float()
									if err == nil {
										data.DHubWD88mAvg = DHubWD88mAvg
									}
									ErrorLog(e, funcName, errorList)

									DRefWD86mAvg, err := row.Cells[8].Float()
									if err == nil {
										data.DRefWD86mAvg = DRefWD86mAvg
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mAvg, err := row.Cells[9].Float()
									if err == nil {
										data.TRefHRefHumid855mAvg = TRefHRefHumid855mAvg
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mStdDev, err := row.Cells[10].Float()
									if err == nil {
										data.TRefHRefHumid855mStdDev = TRefHRefHumid855mStdDev
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mMax, err := row.Cells[11].Float()
									if err == nil {
										data.TRefHRefHumid855mMax = TRefHRefHumid855mMax
									}
									ErrorLog(e, funcName, errorList)

									TRefHRefHumid855mMin, err := row.Cells[12].Float()
									if err == nil {
										data.TRefHRefHumid855mMin = TRefHRefHumid855mMin
									}
									ErrorLog(e, funcName, errorList)

									BaroAirPress855mAvg, err := row.Cells[13].Float()
									if err == nil {
										data.BaroAirPress855mAvg = BaroAirPress855mAvg
									}
									ErrorLog(e, funcName, errorList)
								}

								if e != nil {
									errorLine.Set(tk.ToString(idx+1), errorList)
								} else {
									e = ctx.Insert(data)
									ErrorLog(e, funcName, errorList)
								}
							}
						} else {
							if idx != 0 {
								errorLine.Set(tk.ToString(idx+1), errorList)
							}
						}
					}
				}
				total += count
				tk.Printf("count: %v \n", total)
				tk.Printf("count line error: %v \n", len(errorLine))
				if len(errorLine) > 0 {
					WriteErrors(errorLine, source.Name())
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/generic"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/kubernetes/pkg/apis/batch"
	"k8s.io/kubernetes/pkg/printers"
	printersinternal "k8s.io/kubernetes/pkg/printers/internalversion"
	printerstorage "k8s.io/kubernetes/pkg/printers/storage"
	"k8s.io/kubernetes/pkg/registry/batch/cronjob"
)

// REST implements a RESTStorage for scheduled jobs against etcd
type REST struct {
	*genericregistry.Store
}

// NewREST returns a RESTStorage object that will work against CronJobs.
func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) {
	store := &genericregistry.Store{
		NewFunc:                  func() runtime.Object { return &batch.CronJob{} },
		NewListFunc:              func() runtime.Object { return &batch.CronJobList{} },
		DefaultQualifiedResource: batch.Resource("cronjobs"),

		CreateStrategy: cronjob.Strategy,
		UpdateStrategy: cronjob.Strategy,
		DeleteStrategy: cronjob.Strategy,

		TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)},
	}
	options := &generic.StoreOptions{RESTOptions: optsGetter}
	if err := store.CompleteWithOptions(options); err != nil {
		panic(err) // TODO: Propagate error up
	}

	statusStore := *store
	statusStore.UpdateStrategy = cronjob.StatusStrategy

	return &REST{store}, &StatusREST{store: &statusStore}
}

var _ rest.CategoriesProvider = &REST{}

// Categories implements the CategoriesProvider interface. Returns a list of categories a resource is part of.
func (r *REST) Categories() []string {
	return []string{"all"}
}

// StatusREST implements the REST endpoint for changing the status of a resourcequota.
type StatusREST struct {
	store *genericregistry.Store
}

func (r *StatusREST) New() runtime.Object {
	return &batch.CronJob{}
}

// Get retrieves the object from the storage. It is required to support Patch.
func (r *StatusREST) Get(ctx genericapirequest.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
	return r.store.Get(ctx, name, options)
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx genericapirequest.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc) (runtime.Object, bool, error) {
	return r.store.Update(ctx, name, objInfo, createValidation, updateValidation)
}
<commit_msg>Create short name for cronjob<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/generic"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/kubernetes/pkg/apis/batch"
	"k8s.io/kubernetes/pkg/printers"
	printersinternal "k8s.io/kubernetes/pkg/printers/internalversion"
	printerstorage "k8s.io/kubernetes/pkg/printers/storage"
	"k8s.io/kubernetes/pkg/registry/batch/cronjob"
)

// REST implements a RESTStorage for scheduled jobs against etcd
type REST struct {
	*genericregistry.Store
}

// NewREST returns a RESTStorage object that will work against CronJobs.
func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) {
	store := &genericregistry.Store{
		NewFunc:                  func() runtime.Object { return &batch.CronJob{} },
		NewListFunc:              func() runtime.Object { return &batch.CronJobList{} },
		DefaultQualifiedResource: batch.Resource("cronjobs"),

		CreateStrategy: cronjob.Strategy,
		UpdateStrategy: cronjob.Strategy,
		DeleteStrategy: cronjob.Strategy,

		TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)},
	}
	options := &generic.StoreOptions{RESTOptions: optsGetter}
	if err := store.CompleteWithOptions(options); err != nil {
		panic(err) // TODO: Propagate error up
	}

	statusStore := *store
	statusStore.UpdateStrategy = cronjob.StatusStrategy

	return &REST{store}, &StatusREST{store: &statusStore}
}

var _ rest.CategoriesProvider = &REST{}
var _ rest.ShortNamesProvider = &REST{}

// Categories implements the CategoriesProvider interface. Returns a list of categories a resource is part of.
func (r *REST) Categories() []string {
	return []string{"all"}
}

// ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource.
func (r *REST) ShortNames() []string {
	return []string{"cj"}
}

// StatusREST implements the REST endpoint for changing the status of a resourcequota.
type StatusREST struct {
	store *genericregistry.Store
}

func (r *StatusREST) New() runtime.Object {
	return &batch.CronJob{}
}

// Get retrieves the object from the storage. It is required to support Patch.
func (r *StatusREST) Get(ctx genericapirequest.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
	return r.store.Get(ctx, name, options)
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx genericapirequest.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc) (runtime.Object, bool, error) {
	return r.store.Update(ctx, name, objInfo, createValidation, updateValidation)
}
<|endoftext|>