<commit_before>// This file contains XML structures for communicating with UPnP devices.

package goupnp

import (
	"encoding/xml"
	"errors"
	"fmt"
	"net/url"

	"github.com/huin/goupnp/scpd"
	"github.com/huin/goupnp/soap"
)

const (
	DeviceXMLNamespace = "urn:schemas-upnp-org:device-1-0"
)

// RootDevice is the device description as described by section 2.3 "Device
// description" in
// http://upnp.org/specs/arch/UPnP-arch-DeviceArchitecture-v1.1.pdf
type RootDevice struct {
	XMLName     xml.Name    `xml:"root"`
	SpecVersion SpecVersion `xml:"specVersion"`
	URLBase     url.URL     `xml:"-"`
	URLBaseStr  string      `xml:"URLBase"`
	Device      Device      `xml:"device"`
}

// SetURLBase sets the URLBase for the RootDevice and its underlying components.
func (root *RootDevice) SetURLBase(urlBase *url.URL) {
	root.URLBase = *urlBase
	root.URLBaseStr = urlBase.String()
	root.Device.SetURLBase(urlBase)
}

// SpecVersion is part of a RootDevice, describes the version of the
// specification that the data adheres to.
type SpecVersion struct {
	Major int32 `xml:"major"`
	Minor int32 `xml:"minor"`
}

// Device is a UPnP device. It can have child devices.
type Device struct {
	DeviceType       string    `xml:"deviceType"`
	FriendlyName     string    `xml:"friendlyName"`
	Manufacturer     string    `xml:"manufacturer"`
	ManufacturerURL  URLField  `xml:"manufacturerURL"`
	ModelDescription string    `xml:"modelDescription"`
	ModelName        string    `xml:"modelName"`
	ModelNumber      string    `xml:"modelNumber"`
	ModelURL         URLField  `xml:"modelURL"`
	SerialNumber     string    `xml:"serialNumber"`
	UDN              string    `xml:"UDN"`
	UPC              string    `xml:"UPC,omitempty"`
	Icons            []Icon    `xml:"iconList>icon,omitempty"`
	Services         []Service `xml:"serviceList>service,omitempty"`
	Devices          []Device  `xml:"deviceList>device,omitempty"`

	// Extra observed elements:
	PresentationURL URLField `xml:"presentationURL"`
}

// VisitDevices calls visitor for the device, and all its descendent devices.
func (device *Device) VisitDevices(visitor func(*Device)) {
	visitor(device)
	for i := range device.Devices {
		device.Devices[i].VisitDevices(visitor)
	}
}

// VisitServices calls visitor for all Services under the device and all its
// descendent devices.
func (device *Device) VisitServices(visitor func(*Service)) {
	device.VisitDevices(func(d *Device) {
		for i := range device.Services {
			visitor(&d.Services[i])
		}
	})
}

// FindService finds all (if any) Services under the device and its descendents
// that have the given ServiceType.
func (device *Device) FindService(serviceType string) []*Service {
	var services []*Service
	device.VisitServices(func(s *Service) {
		fmt.Println(s.ServiceType)
		if s.ServiceType == serviceType {
			services = append(services, s)
		}
	})
	return services
}

// SetURLBase sets the URLBase for the Device and its underlying components.
func (device *Device) SetURLBase(urlBase *url.URL) {
	device.ManufacturerURL.SetURLBase(urlBase)
	device.ModelURL.SetURLBase(urlBase)
	device.PresentationURL.SetURLBase(urlBase)
	for i := range device.Icons {
		device.Icons[i].SetURLBase(urlBase)
	}
	for i := range device.Services {
		device.Services[i].SetURLBase(urlBase)
	}
	for i := range device.Devices {
		device.Devices[i].SetURLBase(urlBase)
	}
}

func (device *Device) String() string {
	return fmt.Sprintf("Device ID %s : %s (%s)", device.UDN, device.DeviceType, device.FriendlyName)
}

// Icon is a representative image that a device might include in its
// description.
type Icon struct {
	Mimetype string   `xml:"mimetype"`
	Width    int32    `xml:"width"`
	Height   int32    `xml:"height"`
	Depth    int32    `xml:"depth"`
	URL      URLField `xml:"url"`
}

// SetURLBase sets the URLBase for the Icon.
func (icon *Icon) SetURLBase(url *url.URL) {
	icon.URL.SetURLBase(url)
}

// Service is a service provided by a UPnP Device.
type Service struct {
	ServiceType string   `xml:"serviceType"`
	ServiceId   string   `xml:"serviceId"`
	SCPDURL     URLField `xml:"SCPDURL"`
	ControlURL  URLField `xml:"controlURL"`
	EventSubURL URLField `xml:"eventSubURL"`
}

// SetURLBase sets the URLBase for the Service.
func (srv *Service) SetURLBase(urlBase *url.URL) {
	srv.SCPDURL.SetURLBase(urlBase)
	srv.ControlURL.SetURLBase(urlBase)
	srv.EventSubURL.SetURLBase(urlBase)
}

func (srv *Service) String() string {
	return fmt.Sprintf("Service ID %s : %s", srv.ServiceId, srv.ServiceType)
}

// RequestSCDP requests the SCPD (soap actions and state variables description)
// for the service.
func (srv *Service) RequestSCDP() (*scpd.SCPD, error) {
	if !srv.SCPDURL.Ok {
		return nil, errors.New("bad/missing SCPD URL, or no URLBase has been set")
	}
	s := new(scpd.SCPD)
	if err := requestXml(srv.SCPDURL.URL.String(), scpd.SCPDXMLNamespace, s); err != nil {
		return nil, err
	}
	return s, nil
}

func (srv *Service) NewSOAPClient() *soap.SOAPClient {
	return soap.NewSOAPClient(srv.ControlURL.URL)
}

// URLField is a URL that is part of a device description.
type URLField struct {
	URL url.URL `xml:"-"`
	Ok  bool    `xml:"-"`
	Str string  `xml:",chardata"`
}

func (uf *URLField) SetURLBase(urlBase *url.URL) {
	refUrl, err := url.Parse(uf.Str)
	if err != nil {
		uf.URL = url.URL{}
		uf.Ok = false
		return
	}

	uf.URL = *urlBase.ResolveReference(refUrl)
	uf.Ok = true
}
<commit_msg>Fix bug in visiting services.<commit_after>// This file contains XML structures for communicating with UPnP devices.

package goupnp

import (
	"encoding/xml"
	"errors"
	"fmt"
	"net/url"

	"github.com/huin/goupnp/scpd"
	"github.com/huin/goupnp/soap"
)

const (
	DeviceXMLNamespace = "urn:schemas-upnp-org:device-1-0"
)

// RootDevice is the device description as described by section 2.3 "Device
// description" in
// http://upnp.org/specs/arch/UPnP-arch-DeviceArchitecture-v1.1.pdf
type RootDevice struct {
	XMLName     xml.Name    `xml:"root"`
	SpecVersion SpecVersion `xml:"specVersion"`
	URLBase     url.URL     `xml:"-"`
	URLBaseStr  string      `xml:"URLBase"`
	Device      Device      `xml:"device"`
}

// SetURLBase sets the URLBase for the RootDevice and its underlying components.
func (root *RootDevice) SetURLBase(urlBase *url.URL) {
	root.URLBase = *urlBase
	root.URLBaseStr = urlBase.String()
	root.Device.SetURLBase(urlBase)
}

// SpecVersion is part of a RootDevice, describes the version of the
// specification that the data adheres to.
type SpecVersion struct {
	Major int32 `xml:"major"`
	Minor int32 `xml:"minor"`
}

// Device is a UPnP device. It can have child devices.
type Device struct {
	DeviceType       string    `xml:"deviceType"`
	FriendlyName     string    `xml:"friendlyName"`
	Manufacturer     string    `xml:"manufacturer"`
	ManufacturerURL  URLField  `xml:"manufacturerURL"`
	ModelDescription string    `xml:"modelDescription"`
	ModelName        string    `xml:"modelName"`
	ModelNumber      string    `xml:"modelNumber"`
	ModelURL         URLField  `xml:"modelURL"`
	SerialNumber     string    `xml:"serialNumber"`
	UDN              string    `xml:"UDN"`
	UPC              string    `xml:"UPC,omitempty"`
	Icons            []Icon    `xml:"iconList>icon,omitempty"`
	Services         []Service `xml:"serviceList>service,omitempty"`
	Devices          []Device  `xml:"deviceList>device,omitempty"`

	// Extra observed elements:
	PresentationURL URLField `xml:"presentationURL"`
}

// VisitDevices calls visitor for the device, and all its descendent devices.
func (device *Device) VisitDevices(visitor func(*Device)) {
	visitor(device)
	for i := range device.Devices {
		device.Devices[i].VisitDevices(visitor)
	}
}

// VisitServices calls visitor for all Services under the device and all its
// descendent devices.
func (device *Device) VisitServices(visitor func(*Service)) {
	device.VisitDevices(func(d *Device) {
		for i := range d.Services {
			visitor(&d.Services[i])
		}
	})
}

// FindService finds all (if any) Services under the device and its descendents
// that have the given ServiceType.
func (device *Device) FindService(serviceType string) []*Service {
	var services []*Service
	device.VisitServices(func(s *Service) {
		fmt.Println(s.ServiceType)
		if s.ServiceType == serviceType {
			services = append(services, s)
		}
	})
	return services
}

// SetURLBase sets the URLBase for the Device and its underlying components.
func (device *Device) SetURLBase(urlBase *url.URL) {
	device.ManufacturerURL.SetURLBase(urlBase)
	device.ModelURL.SetURLBase(urlBase)
	device.PresentationURL.SetURLBase(urlBase)
	for i := range device.Icons {
		device.Icons[i].SetURLBase(urlBase)
	}
	for i := range device.Services {
		device.Services[i].SetURLBase(urlBase)
	}
	for i := range device.Devices {
		device.Devices[i].SetURLBase(urlBase)
	}
}

func (device *Device) String() string {
	return fmt.Sprintf("Device ID %s : %s (%s)", device.UDN, device.DeviceType, device.FriendlyName)
}

// Icon is a representative image that a device might include in its
// description.
type Icon struct {
	Mimetype string   `xml:"mimetype"`
	Width    int32    `xml:"width"`
	Height   int32    `xml:"height"`
	Depth    int32    `xml:"depth"`
	URL      URLField `xml:"url"`
}

// SetURLBase sets the URLBase for the Icon.
func (icon *Icon) SetURLBase(url *url.URL) {
	icon.URL.SetURLBase(url)
}

// Service is a service provided by a UPnP Device.
type Service struct {
	ServiceType string   `xml:"serviceType"`
	ServiceId   string   `xml:"serviceId"`
	SCPDURL     URLField `xml:"SCPDURL"`
	ControlURL  URLField `xml:"controlURL"`
	EventSubURL URLField `xml:"eventSubURL"`
}

// SetURLBase sets the URLBase for the Service.
func (srv *Service) SetURLBase(urlBase *url.URL) {
	srv.SCPDURL.SetURLBase(urlBase)
	srv.ControlURL.SetURLBase(urlBase)
	srv.EventSubURL.SetURLBase(urlBase)
}

func (srv *Service) String() string {
	return fmt.Sprintf("Service ID %s : %s", srv.ServiceId, srv.ServiceType)
}

// RequestSCDP requests the SCPD (soap actions and state variables description)
// for the service.
func (srv *Service) RequestSCDP() (*scpd.SCPD, error) {
	if !srv.SCPDURL.Ok {
		return nil, errors.New("bad/missing SCPD URL, or no URLBase has been set")
	}
	s := new(scpd.SCPD)
	if err := requestXml(srv.SCPDURL.URL.String(), scpd.SCPDXMLNamespace, s); err != nil {
		return nil, err
	}
	return s, nil
}

func (srv *Service) NewSOAPClient() *soap.SOAPClient {
	return soap.NewSOAPClient(srv.ControlURL.URL)
}

// URLField is a URL that is part of a device description.
type URLField struct {
	URL url.URL `xml:"-"`
	Ok  bool    `xml:"-"`
	Str string  `xml:",chardata"`
}

func (uf *URLField) SetURLBase(urlBase *url.URL) {
	refUrl, err := url.Parse(uf.Str)
	if err != nil {
		uf.URL = url.URL{}
		uf.Ok = false
		return
	}

	uf.URL = *urlBase.ResolveReference(refUrl)
	uf.Ok = true
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"github.com/cocotyty/summer"
)

func main() {
	summer.Put(&P{})
	summer.Put(&PrinterProvider{})
	summer.Start()
}
type Printer interface {
	Println(args...interface{})
}

type P struct {
	Printer Printer `sm:"@.*"`
	Pr      Printer `sm:"@.P"`
	P       Printer `sm:"@.*"`
}

func (p *P)Ready() {
	p.Printer.Println("Printer is ready")
	p.Pr.Println("Printer is ready")
	p.P.Println("P is ready")
}

type PrinterProvider struct {
	logger *log.Logger
}

func (pp *PrinterProvider)Init() {
	pp.logger = log.New(os.Stderr, "", log.LstdFlags)
}
func (pp *PrinterProvider)Provide() interface{} {
	return pp.logger
}<commit_msg>add provider support<commit_after>package main

import (
	"log"
	"os"
	"github.com/cocotyty/summer"
)

func main() {
	summer.Put(&P{})
	summer.Put(&PrinterProvider{})
	summer.Add("P",&PrinterProvider{prefix:"[p!]"})
	summer.Start()
}
type Printer interface {
	Println(args...interface{})
}

type P struct {
	Printer Printer `sm:"@.*"`
	Pr      Printer `sm:"@.P"`
	P       Printer `sm:"@.*"`
}

func (p *P)Ready() {
	p.Printer.Println("Printer is ready")
	p.Pr.Println("Printer is ready")
	p.P.Println("P is ready")
}

type PrinterProvider struct {
	prefix string
	logger *log.Logger
}

func (pp *PrinterProvider)Init() {
	pp.logger = log.New(os.Stderr, pp.prefix, log.LstdFlags)
}
func (pp *PrinterProvider)Provide() interface{} {
	return pp.logger
}<|endoftext|>


<commit_before>/* Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gnmi

import (
    "errors"
    "fmt"
    "time"

    "github.com/google/link022/agent/context"
    "github.com/google/link022/agent/util/ocutil"
    "github.com/google/link022/agent/service"
    "github.com/google/link022/agent/syscmd"
    "github.com/google/link022/generated/ocstruct"
    "github.com/openconfig/ygot/ygot"

    pb "github.com/openconfig/gnmi/proto/gnmi"
    log "github.com/golang/glog"
)

var (
    cmdRunner = syscmd.Runner()
)

// handleSet is the callback function of the GNMI SET call.
// It is triggered by the GNMI server.
func handleSet(op pb.UpdateResult_Operation, updatedPath *pb.Path, updatedConfig interface{},
               existingConfig ygot.ValidatedGoStruct) error {
    // TODO: Handle delta change. Currently the GNMI server only supports replacing root.
    officeConfig, ok := updatedConfig.(*ocstruct.Office)
    if !ok {
        return errors.New("new configuration has invalid type")
    }

    configString, err := ygot.EmitJSON(officeConfig, &ygot.EmitJSONConfig{
        Format: ygot.RFC7951,
        Indent: "  ",
        RFC7951Config: &ygot.RFC7951JSONConfig{
            AppendModuleName: false,
        },
    })
    if err != nil {
        return err
    }
    log.Infof("Received a new configuration:\n%v\n", configString)

    // TODO: Validate the OpenConfig module.
    deviceConfig := context.GetDeviceConfig()

    // Check and clean up the existing configuration.
    var changedVLANIDs []int
    existingVLANIDs, err := cmdRunner.VLANOnIntf(deviceConfig.ETHINTFName)
    if err != nil {
        return fmt.Errorf("unable to fetch the existing VLAN with error (%v), may need to reboot the device.", err)
    }

    resetIntf := false
    newVLANIDs := ocutil.VLANIDs(officeConfig)
    if ocutil.VLANChanged(existingVLANIDs, newVLANIDs) {
        log.Infof("VLAN changes (%v -> %v) on interface %s.", existingVLANIDs, newVLANIDs ,deviceConfig.ETHINTFName)
        changedVLANIDs = existingVLANIDs
        resetIntf = true
    } else {
        log.Infof("No VLAN change on interface %s.", deviceConfig.ETHINTFName)
    }

    // Clean up the existing configuration.
    service.CleanupConfig(deviceConfig.ETHINTFName, changedVLANIDs)

    // Wait for link to be available again.
    time.Sleep(5 * time.Second)

    // Process the incoming configuraiton.
    if err = service.ApplyConfig(officeConfig, resetIntf, deviceConfig.Hostname, deviceConfig.ETHINTFName,
                                 deviceConfig.WLANINTFName); err != nil {
        return err
    }
    log.Info("Device configuration succeeded.")

    // Save the succeeded config file.
    if err := syscmd.SaveToFile(runFolder, apConfigFileName, configString); err != nil {
        return err
    }
    log.Info("Saved the configuration to file.")
    return nil
}

<commit_msg>Implement the new gnmi set handler interface.<commit_after>/* Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gnmi

import (
    "errors"
    "fmt"
    "time"

    "github.com/google/link022/agent/context"
    "github.com/google/link022/agent/util/ocutil"
    "github.com/google/link022/agent/service"
    "github.com/google/link022/agent/syscmd"
    "github.com/google/link022/generated/ocstruct"
    "github.com/openconfig/ygot/ygot"

    log "github.com/golang/glog"
)

var (
    cmdRunner = syscmd.Runner()
)

// handleSet is the callback function of the GNMI SET call.
// It is triggered by the GNMI server.
func handleSet(updatedConfig ygot.ValidatedGoStruct, existingConfig ygot.ValidatedGoStruct) error {
    // TODO: Handle delta change. Currently the GNMI server only supports replacing root.
    officeConfig, ok := updatedConfig.(*ocstruct.Office)
    if !ok {
        return errors.New("new configuration has invalid type")
    }

    configString, err := ygot.EmitJSON(officeConfig, &ygot.EmitJSONConfig{
        Format: ygot.RFC7951,
        Indent: "  ",
        RFC7951Config: &ygot.RFC7951JSONConfig{
            AppendModuleName: false,
        },
    })
    if err != nil {
        return err
    }
    log.Infof("Received a new configuration:\n%v\n", configString)

    // TODO: Validate the OpenConfig module.
    deviceConfig := context.GetDeviceConfig()

    // Check and clean up the existing configuration.
    var changedVLANIDs []int
    existingVLANIDs, err := cmdRunner.VLANOnIntf(deviceConfig.ETHINTFName)
    if err != nil {
        return fmt.Errorf("unable to fetch the existing VLAN with error (%v), may need to reboot the device.", err)
    }

    resetIntf := false
    newVLANIDs := ocutil.VLANIDs(officeConfig)
    if ocutil.VLANChanged(existingVLANIDs, newVLANIDs) {
        log.Infof("VLAN changes (%v -> %v) on interface %s.", existingVLANIDs, newVLANIDs ,deviceConfig.ETHINTFName)
        changedVLANIDs = existingVLANIDs
        resetIntf = true
    } else {
        log.Infof("No VLAN change on interface %s.", deviceConfig.ETHINTFName)
    }

    // Clean up the existing configuration.
    service.CleanupConfig(deviceConfig.ETHINTFName, changedVLANIDs)

    // Wait for link to be available again.
    time.Sleep(5 * time.Second)

    // Process the incoming configuraiton.
    if err = service.ApplyConfig(officeConfig, resetIntf, deviceConfig.Hostname, deviceConfig.ETHINTFName,
                                 deviceConfig.WLANINTFName); err != nil {
        return err
    }
    log.Info("Device configuration succeeded.")

    // Save the succeeded config file.
    if err := syscmd.SaveToFile(runFolder, apConfigFileName, configString); err != nil {
        return err
    }
    log.Info("Saved the configuration to file.")
    return nil
}

<|endoftext|>


<commit_before>package models

import (
	"encoding/xml"
	"fmt"
	"regexp"
	"strings"
)

type Vendor struct {
	Name    string
	HomeUrl string
	DealUrl string
}

var removeChars = []string{"'", " "}

func (vendor *Vendor) GetProcessingMethodName() string {
	name := vendor.Name
	for _, r := range removeChars {
		name = strings.Replace(name, r, "", -1)
	}
	return name
}

func (vendor *Vendor) NotFound() Deal {
	return Deal{
		Vendor:   vendor,
		Title:    "No Results",
		ImageUrl: "",
		Url:      vendor.HomeUrl,
	}
}

func (vendor *Vendor) Apress(payload []byte) Deal {
	rss := struct {
		Channel struct {
			Items []struct {
				Title string `xml:"title"`
				Link  string `xml:"link"`
				Sku   string `xml:"sku"`
			} `xml:"item"`
		} `xml:"channel"`
	}{}
	xml.Unmarshal(payload, &rss)
	if len(rss.Channel.Items) > 0 {
		item := rss.Channel.Items[0]
		return Deal{
			Vendor:   vendor,
			Title:    item.Title,
			ImageUrl: item.Sku,
			Url:      item.Link,
		}
	}
	return vendor.NotFound()
}

func (vendor *Vendor) Springer(payload []byte) Deal {
	return vendor.Apress(payload)
}

func (vendor *Vendor) InformIT(payload []byte) Deal {
	rss := struct {
		Channel struct {
			Item struct {
				Title string `xml:"title"`
				Link  string `xml:"link"`
				Guid  string `xml:guid"`
			} `xml:"item"`
		} `xml:"channel"`
	}{}
	xml.Unmarshal(payload, &rss)
	if rss.Channel.Item.Title != "" {
		item := rss.Channel.Item
		return Deal{
			Vendor:   vendor,
			Title:    item.Title,
			ImageUrl: item.Guid,
			Url:      item.Link,
		}
	}
	return vendor.NotFound()
}

func (vendor *Vendor) InformITVideo(payload []byte) Deal {
	return vendor.InformIT(payload)
}

func (vendor *Vendor) Peachpit(payload []byte) Deal {
	return vendor.InformIT(payload)
}

func (vendor *Vendor) PeachpitVideo(payload []byte) Deal {
	return vendor.InformIT(payload)
}

func (vendor *Vendor) OReilly(payload []byte) Deal {
	rss := struct {
		Entries []struct {
			Title   string `xml:"title"`
			Link    string `xml:"id"`
			Content string `xml:"content"`
		} `xml:"entry"`
	}{}
	xml.Unmarshal(payload, &rss)
	if len(rss.Entries) > 0 {
		item := rss.Entries[0]
		return Deal{
			Vendor:   vendor,
			Title:    item.Title,
			ImageUrl: "",
			Url:      item.Link,
		}
	}
	return vendor.NotFound()
}

func (vendor *Vendor) OReillyBusiness(payload []byte) Deal {
	return vendor.OReilly(payload)
}

func (vendor *Vendor) OReillyVideo(payload []byte) Deal {
	return vendor.OReilly(payload)
}

func (vendor *Vendor) Manning(payload []byte) Deal {
	re := regexp.MustCompile("\\
Deal of the Day\\
(?s:.+?)\\<a href=\"(.+?)\"(?s:.+?)\\
\\s*(.+?)\\s*\\<(?s:.+?)\\<div style=\"background-image: url\\('(.+?)'\\)")
	matches := re.FindSubmatch(payload)
	if matches != nil {
		return Deal{
			Vendor:   vendor,
			Title:    string(matches[2]),
			ImageUrl: string(matches[3]),
			Url:      fmt.Sprintf("%s%s", vendor.HomeUrl, strings.TrimPrefix(string(matches[1]), "/")),
		}
	}
	return vendor.NotFound()
}
<commit_msg>Close #4. Close #5. Close #9. Fix InformIT and Peachpit images<commit_after>package models

import (
	"encoding/xml"
	"fmt"
	"regexp"
	"strings"
)

type Vendor struct {
	Name    string
	HomeUrl string
	DealUrl string
}

var removeChars = []string{"'", " "}

func (vendor *Vendor) GetProcessingMethodName() string {
	name := vendor.Name
	for _, r := range removeChars {
		name = strings.Replace(name, r, "", -1)
	}
	return name
}

func (vendor *Vendor) NotFound() Deal {
	return Deal{
		Vendor:   vendor,
		Title:    "No Results",
		ImageUrl: "",
		Url:      vendor.HomeUrl,
	}
}

func (vendor *Vendor) Apress(payload []byte) Deal {
	rss := struct {
		Channel struct {
			Items []struct {
				Title string `xml:"title"`
				Link  string `xml:"link"`
				Sku   string `xml:"sku"`
			} `xml:"item"`
		} `xml:"channel"`
	}{}
	xml.Unmarshal(payload, &rss)
	if len(rss.Channel.Items) > 0 {
		item := rss.Channel.Items[0]
		return Deal{
			Vendor:   vendor,
			Title:    item.Title,
			ImageUrl: item.Sku,
			Url:      item.Link,
		}
	}
	return vendor.NotFound()
}

func (vendor *Vendor) Springer(payload []byte) Deal {
	return vendor.Apress(payload)
}

func (vendor *Vendor) InformIT(payload []byte) Deal {
	rss := struct {
		Channel struct {
			Item struct {
				Title string `xml:"title"`
				Link  string `xml:"link"`
				Isbn  string `xml:"guid"`
			} `xml:"item"`
		} `xml:"channel"`
	}{}
	xml.Unmarshal(payload, &rss)
	if rss.Channel.Item.Title != "" {
		item := rss.Channel.Item
		return Deal{
			Vendor:   vendor,
			Title:    item.Title,
			ImageUrl: fmt.Sprintf("%sShowCover.aspx?isbn=%s&type=f", vendor.HomeUrl, item.Isbn),
			Url:      item.Link,
		}
	}
	return vendor.NotFound()
}

func (vendor *Vendor) InformITVideo(payload []byte) Deal {
	return vendor.InformIT(payload)
}

func (vendor *Vendor) Peachpit(payload []byte) Deal {
	return vendor.InformIT(payload)
}

func (vendor *Vendor) PeachpitVideo(payload []byte) Deal {
	return vendor.InformIT(payload)
}

func (vendor *Vendor) OReilly(payload []byte) Deal {
	rss := struct {
		Entries []struct {
			Title   string `xml:"title"`
			Link    string `xml:"id"`
			Content string `xml:"content"`
		} `xml:"entry"`
	}{}
	xml.Unmarshal(payload, &rss)
	if len(rss.Entries) > 0 {
		item := rss.Entries[0]
		return Deal{
			Vendor:   vendor,
			Title:    item.Title,
			ImageUrl: "",
			Url:      item.Link,
		}
	}
	return vendor.NotFound()
}

func (vendor *Vendor) OReillyBusiness(payload []byte) Deal {
	return vendor.OReilly(payload)
}

func (vendor *Vendor) OReillyVideo(payload []byte) Deal {
	return vendor.OReilly(payload)
}

func (vendor *Vendor) Manning(payload []byte) Deal {
	re := regexp.MustCompile("\\
Deal of the Day\\
(?s:.+?)\\<a href=\"(.+?)\"(?s:.+?)\\
\\s*(.+?)\\s*\\<(?s:.+?)\\<div style=\"background-image: url\\('(.+?)'\\)")
	matches := re.FindSubmatch(payload)
	if matches != nil {
		return Deal{
			Vendor:   vendor,
			Title:    string(matches[2]),
			ImageUrl: string(matches[3]),
			Url:      fmt.Sprintf("%s%s", vendor.HomeUrl, strings.TrimPrefix(string(matches[1]), "/")),
		}
	}
	return vendor.NotFound()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/auth"
	"github.com/jrupac/goliath/storage"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const apiVersion = 3

var (
	serveParsedArticles = flag.Bool("serveParsedArticles", false, "If true, serve parsed article content.")
)

type itemType struct {
	ID          int64  `json:"id"`
	FeedID      int64  `json:"feed_id"`
	Title       string `json:"title"`
	Author      string `json:"author"`
	HTML        string `json:"html"`
	URL         string `json:"url"`
	IsSaved     bool   `json:"is_saved"`
	IsRead      bool   `json:"is_read"`
	CreatedTime int64  `json:"created_on_time"`
}

type feedType struct {
	ID          int64  `json:"id"`
	FaviconID   int64  `json:"favicon_id"`
	Title       string `json:"title"`
	URL         string `json:"url"`
	SiteURL     string `json:"site_url"`
	IsSpark     bool   `json:"is_spark"`
	LastUpdated int64  `json:"last_updated_on_time"`
}

type groupType struct {
	ID    int64  `json:"id"`
	Title string `json:"title"`
}

type faviconType struct {
	ID   int64  `json:"id"`
	Data string `json:"data"`
}

type feedsGroupType struct {
	GroupID int64  `json:"group_id"`
	FeedIDs string `json:"feed_ids"`
}

// HandleFever returns a handler function that implements the Fever API.
func HandleFever(d *storage.Database) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		handleFever(d, w, r)
	}
}

func handleFever(d *storage.Database, w http.ResponseWriter, r *http.Request) {
	// These two fields must always be set on responses.
	resp := map[string]interface{}{
		"api_version":            apiVersion,
		"last_refreshed_on_time": time.Now().Unix(),
	}

	r.ParseForm()
	log.Infof("Fever request URL: %s", r.URL.String())
	log.Infof("Fever request body: %s", r.PostForm.Encode())

	switch r.Form.Get("api") {
	case "":
		w.Header().Set("Content-Type", "application/json")
	case "xml":
		// TODO: Implement XML response type.
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	resp["auth"] = handleAuth(d, r)
	if resp["auth"] == 0 {
		returnSuccess(w, resp)
		return
	}

	if _, ok := r.Form["groups"]; ok {
		folders, err := d.GetAllFolders()
		if err != nil {
			returnError(w, "Failed to fetch folders: %s", err)
			return
		}
		var groups []groupType
		for _, f := range folders {
			g := groupType{
				ID:    f.ID,
				Title: f.Name,
			}
			groups = append(groups, g)
		}
		resp["groups"] = groups

		resp["feeds_groups"], err = constructFeedsGroups(d)
		if err != nil {
			returnError(w, "Failed to fetch feeds per folder: %s", err)
			return
		}
	}
	if _, ok := r.Form["feeds"]; ok {
		fetchedFeeds, err := d.GetAllFeeds()
		if err != nil {
			returnError(w, "Failed to fetch feeds: %s", err)
			return
		}
		var feeds []feedType
		for _, ff := range fetchedFeeds {
			f := feedType{
				ID:          ff.ID,
				FaviconID:   ff.ID,
				Title:       ff.Title,
				URL:         ff.URL,
				SiteURL:     ff.URL,
				IsSpark:     false,
				LastUpdated: time.Now().Unix(),
			}
			feeds = append(feeds, f)
		}
		resp["feeds"] = feeds
		resp["feeds_groups"], err = constructFeedsGroups(d)
		if err != nil {
			returnError(w, "Failed to fetch feeds per folder: %s", err)
			return
		}
	}
	if _, ok := r.Form["favicons"]; ok {
		faviconMap, err := d.GetAllFavicons()
		if err != nil {
			returnError(w, "Failed to fetch favicons: %s", err)
			return
		}
		var favicons []faviconType
		for k, v := range faviconMap {
			f := faviconType{
				ID:   k,
				Data: v,
			}
			favicons = append(favicons, f)
		}
		resp["favicons"] = favicons
	}
	if _, ok := r.Form["items"]; ok {
		// TODO: support "max_id" and "with_ids".
		sinceID := int64(-1)
		var err error

		if _, ok2 := r.Form["since_id"]; ok2 {
			sinceID, err = strconv.ParseInt(r.FormValue("since_id"), 10, 64)
			if err != nil {
				w.WriteHeader(http.StatusBadRequest)
				return
			}
		}

		articles, err := d.GetUnreadArticles(50, sinceID)
		if err != nil {
			returnError(w, "Failed to fetch unread articles: %s", err)
			return
		}
		var items []itemType
		var content string
		for _, a := range articles {
			if *serveParsedArticles && a.Parsed != "" {
				log.Infof("Serving parsed content for title: %s", a.Title)
				content = a.Parsed
			} else if a.Content != "" {
				// The "content" field usually has more text, but is not always set.
				content = a.Content
			} else {
				content = a.Summary
			}

			i := itemType{
				ID:          a.ID,
				FeedID:      a.FeedID,
				Title:       a.Title,
				Author:      "",
				HTML:        content,
				URL:         a.Link,
				IsSaved:     false,
				IsRead:      false,
				CreatedTime: a.Date.Unix(),
			}
			items = append(items, i)
		}
		resp["items"] = items
	}
	if _, ok := r.Form["links"]; ok {
		// Perhaps add support for Hot Links in the future.
		resp["links"] = ""
	}
	if _, ok := r.Form["unread_item_ids"]; ok {
		articles, err := d.GetUnreadArticles(-1, -1)
		if err != nil {
			returnError(w, "Failed to fetch unread articles: %s", err)
			return
		}
		var unreadItemIds []string
		for _, a := range articles {
			unreadItemIds = append(unreadItemIds, strconv.FormatInt(a.ID, 10))
		}
		resp["unread_item_ids"] = strings.Join(unreadItemIds, ",")
	}
	if _, ok := r.Form["saved_item_ids"]; ok {
		// Perhaps add support for saving items in the future.
		resp["saved_item_ids"] = ""
	}
	if _, ok := r.Form["mark"]; ok {
		// TODO: Support "before" argument.
		var as string
		switch r.FormValue("as") {
		case "read", "unread", "saved", "unsaved":
			as = r.FormValue("as")
		default:
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		id, err := strconv.ParseInt(r.FormValue("id"), 10, 64)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			return
		}

		switch r.FormValue("mark") {
		case "item":
			if err2 := d.MarkArticle(id, as); err2 != nil {
				returnError(w, "Unable to mark article: %s", err2)
			}
		case "feed":
			if err2 := d.MarkFeed(id, as); err2 != nil {
				returnError(w, "Unable to mark feed: %s", err2)
			}
		case "group":
			if err2 := d.MarkFolder(id, as); err2 != nil {
				returnError(w, "Unable to mark folder: %s", err2)
			}
		default:
			w.WriteHeader(http.StatusBadRequest)
			return
		}
	}

	returnSuccess(w, resp)
}

func returnError(w http.ResponseWriter, msg string, err error) {
	log.Warningf(msg, err)
	w.WriteHeader(http.StatusInternalServerError)
}

func returnSuccess(w http.ResponseWriter, resp map[string]interface{}) {
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		returnError(w, "Failed to encode response JSON: %s", err)
	}
}

func handleAuth(d *storage.Database, r *http.Request) int {
	// A request can be authenticated by cookie or api key in request.
	if auth.VerifyCookie(d, r) {
		log.V(2).Infof("Verified cookie: %+v", r)
		return 1
	} else if _, err := d.GetUserByKey(r.FormValue("api_key")); err != nil {
		log.Warningf("Rejected request: %+v", r)
		log.Warningf("Failed because: %s", err)
		return 0
	} else {
		log.V(2).Infof("Successfully authenticated by key: %+v", r)
		return 1
	}
}

func constructFeedsGroups(d *storage.Database) ([]feedsGroupType, error) {
	var feedGroups []feedsGroupType
	feedsPerFolder, err := d.GetFeedsPerFolder()
	if err != nil {
		log.Warningf("Failed to fetch feeds per folder: %s", err)
		return feedGroups, err
	}
	for k, v := range feedsPerFolder {
		feedGroup := feedsGroupType{
			GroupID: k,
			FeedIDs: v,
		}
		feedGroups = append(feedGroups, feedGroup)
	}
	return feedGroups, err
}
<commit_msg>fever: Refactor each type of API request to separate methods.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/auth"
	"github.com/jrupac/goliath/storage"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const apiVersion = 3

var (
	serveParsedArticles = flag.Bool("serveParsedArticles", false, "If true, serve parsed article content.")
)

type itemType struct {
	ID          int64  `json:"id"`
	FeedID      int64  `json:"feed_id"`
	Title       string `json:"title"`
	Author      string `json:"author"`
	HTML        string `json:"html"`
	URL         string `json:"url"`
	IsSaved     bool   `json:"is_saved"`
	IsRead      bool   `json:"is_read"`
	CreatedTime int64  `json:"created_on_time"`
}

type feedType struct {
	ID          int64  `json:"id"`
	FaviconID   int64  `json:"favicon_id"`
	Title       string `json:"title"`
	URL         string `json:"url"`
	SiteURL     string `json:"site_url"`
	IsSpark     bool   `json:"is_spark"`
	LastUpdated int64  `json:"last_updated_on_time"`
}

type groupType struct {
	ID    int64  `json:"id"`
	Title string `json:"title"`
}

type faviconType struct {
	ID   int64  `json:"id"`
	Data string `json:"data"`
}

type feedsGroupType struct {
	GroupID int64  `json:"group_id"`
	FeedIDs string `json:"feed_ids"`
}

type responseType map[string]interface{}

type feverError struct {
	wrapped  error
	internal bool
}

func (e *feverError) Error() string {
	return e.wrapped.Error()
}

// HandleFever returns a handler function that implements the Fever API.
func HandleFever(d *storage.Database) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		handleFever(d, w, r)
	}
}

func handleFever(d *storage.Database, w http.ResponseWriter, r *http.Request) {
	// These two fields must always be set on responses.
	resp := responseType{
		"api_version":            apiVersion,
		"last_refreshed_on_time": time.Now().Unix(),
	}

	r.ParseForm()
	log.Infof("Fever request URL: %s", r.URL.String())
	log.Infof("Fever request body: %s", r.PostForm.Encode())

	switch r.Form.Get("api") {
	case "":
		w.Header().Set("Content-Type", "application/json")
	case "xml":
		// TODO: Implement XML response type.
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	resp["auth"] = handleAuth(d, r)
	if resp["auth"] == 0 {
		returnSuccess(w, resp)
		return
	}

	if _, ok := r.Form["groups"]; ok {
		err := handleGroups(d, &resp)
		if err != nil {
			returnError(w, "Failed request 'groups': %s", err)
			return
		}
	}
	if _, ok := r.Form["feeds"]; ok {
		err := handleFeeds(d, &resp)
		if err != nil {
			returnError(w, "Failed request 'feeds': %s", err)
			return
		}
	}
	if _, ok := r.Form["favicons"]; ok {
		err := handleFavicons(d, &resp)
		if err != nil {
			returnError(w, "Failed request 'favicons': %s", err)
			return
		}
	}
	if _, ok := r.Form["items"]; ok {
		err := handleItems(d, &resp, r)
		if err != nil {
			returnError(w, "Failed request 'items': %s", err)
			return
		}
	}
	if _, ok := r.Form["links"]; ok {
		err := handleLinks(d, &resp)
		if err != nil {
			returnError(w, "Failed request 'links': %s", err)
			return
		}
	}
	if _, ok := r.Form["unread_item_ids"]; ok {
		err := handleUnreadItemIDs(d, &resp)
		if err != nil {
			returnError(w, "Failed request 'unread_item_ids': %s", err)
			return
		}
	}
	if _, ok := r.Form["saved_item_ids"]; ok {
		err := handleSavedItemIDs(d, &resp)
		if err != nil {
			returnError(w, "Failed request 'saved_item_ids': %s", err)
			return
		}
	}
	if _, ok := r.Form["mark"]; ok {
		err := handleMark(d, &resp, r)
		if err != nil {
			returnError(w, "Failed request 'mark': %s", err)
			return
		}
	}

	returnSuccess(w, resp)
}

func returnError(w http.ResponseWriter, msg string, err error) {
	log.Warningf(msg, err)
	if fe, ok := err.(*feverError); ok {
		if fe.internal {
			w.WriteHeader(http.StatusInternalServerError)
		} else {
			w.WriteHeader(http.StatusBadRequest)
		}
		return
	}
	w.WriteHeader(http.StatusInternalServerError)
}

func returnSuccess(w http.ResponseWriter, resp map[string]interface{}) {
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		returnError(w, "Failed to encode response JSON: %s", err)
	}
}

func handleAuth(d *storage.Database, r *http.Request) int {
	// A request can be authenticated by cookie or api key in request.
	if auth.VerifyCookie(d, r) {
		log.V(2).Infof("Verified cookie: %+v", r)
		return 1
	} else if _, err := d.GetUserByKey(r.FormValue("api_key")); err != nil {
		log.Warningf("Rejected request: %+v", r)
		log.Warningf("Failed because: %s", err)
		return 0
	} else {
		log.V(2).Infof("Successfully authenticated by key: %+v", r)
		return 1
	}
}

func handleGroups(d *storage.Database, resp *responseType) error {
	folders, err := d.GetAllFolders()
	if err != nil {
		return &feverError{err, true}
	}
	var groups []groupType
	for _, f := range folders {
		g := groupType{
			ID:    f.ID,
			Title: f.Name,
		}
		groups = append(groups, g)
	}
	(*resp)["groups"] = groups
	(*resp)["feeds_groups"], err = constructFeedsGroups(d)
	if err != nil {
		return &feverError{err, true}
	}
	return nil
}

func handleFeeds(d *storage.Database, resp *responseType) error {
	fetchedFeeds, err := d.GetAllFeeds()
	if err != nil {
		return &feverError{err, true}
	}
	var feeds []feedType
	for _, ff := range fetchedFeeds {
		f := feedType{
			ID:          ff.ID,
			FaviconID:   ff.ID,
			Title:       ff.Title,
			URL:         ff.URL,
			SiteURL:     ff.URL,
			IsSpark:     false,
			LastUpdated: time.Now().Unix(),
		}
		feeds = append(feeds, f)
	}
	(*resp)["feeds"] = feeds
	(*resp)["feeds_groups"], err = constructFeedsGroups(d)
	if err != nil {
		return &feverError{err, true}
	}
	return nil
}

func handleFavicons(d *storage.Database, resp *responseType) error {
	faviconMap, err := d.GetAllFavicons()
	if err != nil {
		return &feverError{err, true}
	}
	var favicons []faviconType
	for k, v := range faviconMap {
		f := faviconType{
			ID:   k,
			Data: v,
		}
		favicons = append(favicons, f)
	}
	(*resp)["favicons"] = favicons
	return nil
}

func handleItems(d *storage.Database, resp *responseType, r *http.Request) error {
	// TODO: support "max_id" and "with_ids".
	sinceID := int64(-1)
	var err error

	if _, ok := r.Form["since_id"]; ok {
		sinceID, err = strconv.ParseInt(r.FormValue("since_id"), 10, 64)
		if err != nil {
			return &feverError{err, false}
		}
	}

	articles, err := d.GetUnreadArticles(50, sinceID)
	if err != nil {
		return &feverError{err, true}
	}
	var items []itemType
	var content string
	for _, a := range articles {
		if *serveParsedArticles && a.Parsed != "" {
			log.Infof("Serving parsed content for title: %s", a.Title)
			content = a.Parsed
		} else if a.Content != "" {
			// The "content" field usually has more text, but is not always set.
			content = a.Content
		} else {
			content = a.Summary
		}

		i := itemType{
			ID:          a.ID,
			FeedID:      a.FeedID,
			Title:       a.Title,
			Author:      "",
			HTML:        content,
			URL:         a.Link,
			IsSaved:     false,
			IsRead:      false,
			CreatedTime: a.Date.Unix(),
		}
		items = append(items, i)
	}
	(*resp)["items"] = items
	return nil
}

func handleLinks(_ *storage.Database, resp *responseType) error {
	// Perhaps add support for links in the future.
	(*resp)["links"] = ""
	return nil
}

func handleUnreadItemIDs(d *storage.Database, resp *responseType) error {
	articles, err := d.GetUnreadArticles(-1, -1)
	if err != nil {
		return &feverError{err, true}
	}
	var unreadItemIds []string
	for _, a := range articles {
		unreadItemIds = append(unreadItemIds, strconv.FormatInt(a.ID, 10))
	}
	(*resp)["unread_item_ids"] = strings.Join(unreadItemIds, ",")
	return nil
}

func handleSavedItemIDs(_ *storage.Database, resp *responseType) error {
	// Perhaps add support for saving items in the future.
	(*resp)["saved_item_ids"] = ""
	return nil
}

func handleMark(d *storage.Database, _ *responseType, r *http.Request) error {
	// TODO: Support "before" argument.
	var as string
	switch r.FormValue("as") {
	case "read", "unread", "saved", "unsaved":
		as = r.FormValue("as")
	default:
		return &feverError{fmt.Errorf("unknown 'as' value: %s", r.FormValue("as")), false}
	}
	id, err := strconv.ParseInt(r.FormValue("id"), 10, 64)
	if err != nil {
		return err
	}

	switch r.FormValue("mark") {
	case "item":
		if err = d.MarkArticle(id, as); err != nil {
			return &feverError{err, true}
		}
	case "feed":
		if err = d.MarkFeed(id, as); err != nil {
			return &feverError{err, true}
		}
	case "group":
		if err = d.MarkFolder(id, as); err != nil {
			return &feverError{err, true}
		}
	default:
		return &feverError{fmt.Errorf("malformed 'mark' value: %s", r.FormValue("mark")), false}
	}
	return nil
}

func constructFeedsGroups(d *storage.Database) ([]feedsGroupType, error) {
	var feedGroups []feedsGroupType
	feedsPerFolder, err := d.GetFeedsPerFolder()
	if err != nil {
		log.Warningf("Failed to fetch feeds per folder: %s", err)
		return feedGroups, err
	}
	for k, v := range feedsPerFolder {
		feedGroup := feedsGroupType{
			GroupID: k,
			FeedIDs: v,
		}
		feedGroups = append(feedGroups, feedGroup)
	}
	return feedGroups, nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"

	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/identity"

	"github.com/spf13/cobra"
)

// Fatalf prints the Printf formatted message to stderr and exits the program
// Note: os.Exit(1) is not recoverable
func Fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", fmt.Sprintf(msg, args...))
	os.Exit(1)
}

// Usagef prints the Printf formatted message to stderr, prints usage help and
// exits the program
// Note: os.Exit(1) is not recoverable
func Usagef(cmd *cobra.Command, msg string, args ...interface{}) {
	txt := fmt.Sprintf(msg, args...)
	fmt.Fprintf(os.Stderr, "Error: %s\n\n", txt)
	cmd.Help()
	os.Exit(1)
}

func requireEndpointID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id argument")
	}

	if id := identity.ReservedIdentities[args[0]]; id == identity.IdentityUnknown {
		_, _, err := endpoint.ValidateID(args[0])

		if err != nil {
			Fatalf("Cannot parse endpoint id \"%s\": %s", args[0], err)
		}
	}
}

func requireEndpointIDorGlobal(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id or 'global' argument")
	}

	if args[0] != "global" {
		requireEndpointID(cmd, args)
	}
}

func requirePath(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing path argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty path argument")
	}
}

func requireServiceID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing service id argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty service id argument")
	}
}

// Search 'result' for strings with escaped JSON inside, and expand the JSON.
func expandNestedJSON(result bytes.Buffer) (bytes.Buffer, error) {
	reStringWithJSON := regexp.MustCompile(`"[^"\\{]*{.*[^\\]"`)
	reJSON := regexp.MustCompile(`{.*}`)
	for {
		var (
			loc    []int
			indent string
		)

		// Search for nested JSON; if we don't find any, then break.
		resBytes := result.Bytes()
		if loc = reStringWithJSON.FindIndex(resBytes); loc == nil {
			break
		}

		// Determine the current indentation
		for i := 0; i < loc[0]-1; i++ {
			idx := loc[0] - i - 1
			if resBytes[idx] != ' ' {
				break
			}
			indent = fmt.Sprintf("%s ", indent)
		}

		stringStart := loc[0]
		stringEnd := loc[1]

		// Unquote the string with the nested json.
		quotedBytes := resBytes[stringStart:stringEnd]
		unquoted, err := strconv.Unquote(string(quotedBytes))
		if err != nil {
			return bytes.Buffer{}, fmt.Errorf("Failed to Unquote string: %s\n%s", err.Error(), string(quotedBytes))
		}

		// Find the JSON within the quoted string.
		nestedStart := 0
		nestedEnd := 0
		if locs := reJSON.FindAllStringIndex(unquoted, -1); locs != nil {
			// The last match is the longest one.
			last := len(locs) - 1
			nestedStart = locs[last][0]
			nestedEnd = locs[last][1]
		} else if reJSON.Match(quotedBytes) {
			// The entire string is JSON
			nestedEnd = len(unquoted)
		}

		// Decode the nested JSON
		decoded := ""
		if nestedEnd != 0 {
			m := make(map[string]interface{})
			nested := bytes.NewBufferString(unquoted[nestedStart:nestedEnd])
			if err := json.NewDecoder(nested).Decode(&m); err != nil {
				return bytes.Buffer{}, fmt.Errorf("Failed to decode nested JSON: %s", err.Error())
			}
			decodedBytes, err := json.MarshalIndent(m, indent, "  ")
			if err != nil {
				return bytes.Buffer{}, fmt.Errorf("Cannot marshal nested JSON: %s", err.Error())
			}
			decoded = string(decodedBytes)
		}

		// Serialize
		nextResult := bytes.Buffer{}
		nextResult.Write(resBytes[0:stringStart])
		nextResult.WriteString(string(unquoted[:nestedStart]))
		nextResult.WriteString(string(decoded))
		nextResult.WriteString(string(unquoted[nestedEnd:]))
		nextResult.Write(resBytes[stringEnd:])
		result = nextResult
	}

	return result, nil
}
<commit_msg>cilium/cmd: Add TablePrinter helper<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"text/tabwriter"

	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/identity"

	"github.com/spf13/cobra"
)

// Fatalf prints the Printf formatted message to stderr and exits the program
// Note: os.Exit(1) is not recoverable
func Fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", fmt.Sprintf(msg, args...))
	os.Exit(1)
}

// Usagef prints the Printf formatted message to stderr, prints usage help and
// exits the program
// Note: os.Exit(1) is not recoverable
func Usagef(cmd *cobra.Command, msg string, args ...interface{}) {
	txt := fmt.Sprintf(msg, args...)
	fmt.Fprintf(os.Stderr, "Error: %s\n\n", txt)
	cmd.Help()
	os.Exit(1)
}

func requireEndpointID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id argument")
	}

	if id := identity.ReservedIdentities[args[0]]; id == identity.IdentityUnknown {
		_, _, err := endpoint.ValidateID(args[0])

		if err != nil {
			Fatalf("Cannot parse endpoint id \"%s\": %s", args[0], err)
		}
	}
}

func requireEndpointIDorGlobal(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id or 'global' argument")
	}

	if args[0] != "global" {
		requireEndpointID(cmd, args)
	}
}

func requirePath(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing path argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty path argument")
	}
}

func requireServiceID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing service id argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty service id argument")
	}
}

// TablePrinter prints the map[string][]string, which is an usual representation
// of dumped BPF map, using tabwriter.
func TablePrinter(firstTitle, secondTitle string, data map[string][]string) {
	w := tabwriter.NewWriter(os.Stdout, 5, 0, 3, ' ', 0)

	fmt.Fprintf(w, "%s\t%s\t\n", firstTitle, secondTitle)

	for key, value := range data {
		for k, v := range value {
			if k == 0 {
				fmt.Fprintf(w, "%s\t%s\t\n", key, v)
			} else {
				fmt.Fprintf(w, "%s\t%s\t\n", "", v)
			}
		}
	}

	w.Flush()
}

// Search 'result' for strings with escaped JSON inside, and expand the JSON.
func expandNestedJSON(result bytes.Buffer) (bytes.Buffer, error) {
	reStringWithJSON := regexp.MustCompile(`"[^"\\{]*{.*[^\\]"`)
	reJSON := regexp.MustCompile(`{.*}`)
	for {
		var (
			loc    []int
			indent string
		)

		// Search for nested JSON; if we don't find any, then break.
		resBytes := result.Bytes()
		if loc = reStringWithJSON.FindIndex(resBytes); loc == nil {
			break
		}

		// Determine the current indentation
		for i := 0; i < loc[0]-1; i++ {
			idx := loc[0] - i - 1
			if resBytes[idx] != ' ' {
				break
			}
			indent = fmt.Sprintf("%s ", indent)
		}

		stringStart := loc[0]
		stringEnd := loc[1]

		// Unquote the string with the nested json.
		quotedBytes := resBytes[stringStart:stringEnd]
		unquoted, err := strconv.Unquote(string(quotedBytes))
		if err != nil {
			return bytes.Buffer{}, fmt.Errorf("Failed to Unquote string: %s\n%s", err.Error(), string(quotedBytes))
		}

		// Find the JSON within the quoted string.
		nestedStart := 0
		nestedEnd := 0
		if locs := reJSON.FindAllStringIndex(unquoted, -1); locs != nil {
			// The last match is the longest one.
			last := len(locs) - 1
			nestedStart = locs[last][0]
			nestedEnd = locs[last][1]
		} else if reJSON.Match(quotedBytes) {
			// The entire string is JSON
			nestedEnd = len(unquoted)
		}

		// Decode the nested JSON
		decoded := ""
		if nestedEnd != 0 {
			m := make(map[string]interface{})
			nested := bytes.NewBufferString(unquoted[nestedStart:nestedEnd])
			if err := json.NewDecoder(nested).Decode(&m); err != nil {
				return bytes.Buffer{}, fmt.Errorf("Failed to decode nested JSON: %s", err.Error())
			}
			decodedBytes, err := json.MarshalIndent(m, indent, "  ")
			if err != nil {
				return bytes.Buffer{}, fmt.Errorf("Cannot marshal nested JSON: %s", err.Error())
			}
			decoded = string(decodedBytes)
		}

		// Serialize
		nextResult := bytes.Buffer{}
		nextResult.Write(resBytes[0:stringStart])
		nextResult.WriteString(string(unquoted[:nestedStart]))
		nextResult.WriteString(string(decoded))
		nextResult.WriteString(string(unquoted[nestedEnd:]))
		nextResult.Write(resBytes[stringEnd:])
		result = nextResult
	}

	return result, nil
}
<|endoftext|>


<commit_before>package mirror

import (
	"reflect"
)

// VisitType
type VisitType func(t *reflect.StructField, typeIndex, depth int) error

type typeWalker struct {
	indices map[reflect.Type]int
	types   []reflect.Type
}

func (walker *typeWalker) index(t reflect.Type) (int, bool) {
	idx, known := walker.indices[t]
	if known {
		return idx, true
	}
	idx = len(walker.types)
	walker.indices[t] = idx
	walker.types = append(walker.types, t)
	return idx, false
}

func (walker *typeWalker) walk(t *reflect.StructField, visit VisitType) error {
	if t == nil || visit == nil {
		return nil
	}
	type stackNode struct {
		field      *reflect.StructField
		currentIdx int
		parentIdx  int
		depth      int
		known      bool
	}
	idx, known := walker.index(t.Type)
	stack := []stackNode{{t, idx, idx, 0, known}}
	var node stackNode
	for len(stack) > 0 {
		lastIdx := len(stack) - 1
		stack, node = stack[:lastIdx], stack[lastIdx]
		err := visit(node.field, node.currentIdx, node.depth)
		if err != nil {
			return err
		}
		if node.known {
			// to prevent endless loops, don't follow known types
			continue
		}
		// follow container types
		parentIdx := node.currentIdx
		depth := node.depth + 1
		switch t := node.field.Type; t.Kind() {
		case reflect.Struct:
			fields := t.NumField()
			for i := fields - 1; i >= 0; i-- {
				// add each field from struct to stack (hightest first, visiting order is reversed)
				field := t.Field(i)
				typeIdx, known := walker.index(field.Type)
				stack = append(stack, stackNode{&field, typeIdx, parentIdx, depth, known})
			}
		case reflect.Ptr, reflect.Array, reflect.Slice, reflect.Map, reflect.Chan:
			// add element to stack
			field := &reflect.StructField{Type: t.Elem()}
			typeIdx, known := walker.index(field.Type)
			stack = append(stack, stackNode{field, typeIdx, parentIdx, depth, known})
		}
	}
	return nil
}

// Walk will call visit on t and each type in t.
// It will follow struct fields, pointers, arrays, slices, maps and chans
// and call visit on each element.
// For struct fields, Walk will only follow into fields at the first occurence of their type.
// Function handling for reflect.Interface and reflect.Method has to be done by visit.
func Walk(t reflect.Type, visit VisitType) error {
	walker := &typeWalker{indices: make(map[reflect.Type]int)}
	return walker.walk(&reflect.StructField{Type: t}, visit)
}
<commit_msg>off by one<commit_after>package mirror

import (
	"reflect"
)

// VisitType
type VisitType func(t *reflect.StructField, typeIndex, depth int) error

type typeWalker struct {
	indices map[reflect.Type]int
	types   []reflect.Type
}

func (walker *typeWalker) index(t reflect.Type) (int, bool) {
	idx, known := walker.indices[t]
	if known {
		return idx, true
	}
	idx = len(walker.types)
	walker.indices[t] = idx
	walker.types = append(walker.types, t)
	return idx, false
}

func (walker *typeWalker) walk(t *reflect.StructField, visit VisitType) error {
	if t == nil || visit == nil {
		return nil
	}
	type stackNode struct {
		field      *reflect.StructField
		currentIdx int
		parentIdx  int
		depth      int
		known      bool
	}
	idx, known := walker.index(t.Type)
	stack := []stackNode{{t, idx, idx, 0, known}}
	var node stackNode
	for lastIdx := 0; lastIdx >= 0; lastIdx = len(stack) - 1 {
		stack, node = stack[:lastIdx], stack[lastIdx]
		err := visit(node.field, node.currentIdx, node.depth)
		if err != nil {
			return err
		}
		if node.known {
			// to prevent endless loops, don't follow known types
			continue
		}
		// follow container types
		parentIdx := node.currentIdx
		depth := node.depth
		switch t := node.field.Type; t.Kind() {
		case reflect.Struct:
			fields := t.NumField()
			for i := fields - 1; i >= 0; i-- {
				// add each field from struct to stack (hightest first, visiting order is reversed)
				field := t.Field(i)
				typeIdx, known := walker.index(field.Type)
				stack = append(stack, stackNode{&field, typeIdx, parentIdx, depth, known})
			}
		case reflect.Ptr, reflect.Array, reflect.Slice, reflect.Map, reflect.Chan:
			// add element to stack
			field := &reflect.StructField{Type: t.Elem()}
			typeIdx, known := walker.index(field.Type)
			stack = append(stack, stackNode{field, typeIdx, parentIdx, depth, known})
		}
	}
	return nil
}

// Walk will call visit on t and each type in t.
// It will follow struct fields, pointers, arrays, slices, maps and chans
// and call visit on each element.
// For struct fields, Walk will only follow into fields at the first occurence of their type.
// Function handling for reflect.Interface and reflect.Method has to be done by visit.
func Walk(t reflect.Type, visit VisitType) error {
	walker := &typeWalker{indices: make(map[reflect.Type]int)}
	return walker.walk(&reflect.StructField{Type: t}, visit)
}
<|endoftext|>


<commit_before>// Copyright 2012-2016 The GoSNMP Authors. All rights reserved.  Use of this
// source code is governed by a BSD-style license that can be found in the
// LICENSE file.

package gosnmp

import (
	"fmt"
	"strings"
)

func (x *GoSNMP) walk(getRequestType PDUType, rootOid string, walkFn WalkFunc) error {
	if rootOid == "" || rootOid == "." {
		rootOid = baseOid
	}

	if !strings.HasPrefix(rootOid, ".") {
		rootOid = string(".") + rootOid
	}

	oid := rootOid
	requests := 0
	maxReps := x.MaxRepetitions
	if maxReps == 0 {
		maxReps = defaultMaxRepetitions
	}

	getFn := func(oid string) (result *SnmpPacket, err error) {
		switch getRequestType {
		case GetBulkRequest:
			return x.GetBulk([]string{oid}, uint8(x.NonRepeaters), uint8(maxReps))
		case GetNextRequest:
			return x.GetNext([]string{oid})
		default:
			return nil, fmt.Errorf("Unsupported request type: %d", getRequestType)
		}
	}

RequestLoop:
	for {

		requests++
		response, err := getFn(oid)
		if err != nil {
			return err
		}
		if len(response.Variables) == 0 {
			break RequestLoop
		}

		if response.Error == NoSuchName {
			x.Logger.Print("Walk terminated with NoSuchName")
			break RequestLoop
		}

		for _, v := range response.Variables {
			if v.Type == EndOfMibView || v.Type == NoSuchObject || v.Type == NoSuchInstance {
				x.Logger.Printf("BulkWalk terminated with type 0x%x", v.Type)
				break RequestLoop
			}
			if !strings.HasPrefix(v.Name, rootOid) {
				// Not in the requested root range.
				break RequestLoop
			}
			if v.Name == oid {
				return fmt.Errorf("OID not increasing: %s", v.Name)
			}
			// Report our pdu
			if err := walkFn(v); err != nil {
				return err
			}
		}
		// Save last oid for next request
		oid = response.Variables[len(response.Variables)-1].Name
	}
	x.Logger.Printf("BulkWalk completed in %d requests", requests)
	return nil
}

func (x *GoSNMP) walkAll(getRequestType PDUType, rootOid string) (results []SnmpPDU, err error) {
	err = x.walk(getRequestType, rootOid, func(dataUnit SnmpPDU) error {
		results = append(results, dataUnit)
		return nil
	})
	return results, err
}
<commit_msg>A simple fix for one of the bugs brought up in issue 78<commit_after>// Copyright 2012-2016 The GoSNMP Authors. All rights reserved.  Use of this
// source code is governed by a BSD-style license that can be found in the
// LICENSE file.

package gosnmp

import (
	"fmt"
	"strings"
)

func (x *GoSNMP) walk(getRequestType PDUType, rootOid string, walkFn WalkFunc) error {
	if rootOid == "" || rootOid == "." {
		rootOid = baseOid
	}

	if !strings.HasPrefix(rootOid, ".") {
		rootOid = string(".") + rootOid
	}

	oid := rootOid
	requests := 0
	maxReps := x.MaxRepetitions
	if maxReps == 0 {
		maxReps = defaultMaxRepetitions
	}

	getFn := func(oid string) (result *SnmpPacket, err error) {
		switch getRequestType {
		case GetBulkRequest:
			return x.GetBulk([]string{oid}, uint8(x.NonRepeaters), uint8(maxReps))
		case GetNextRequest:
			return x.GetNext([]string{oid})
		default:
			return nil, fmt.Errorf("Unsupported request type: %d", getRequestType)
		}
	}

RequestLoop:
	for {

		requests++
		response, err := getFn(oid)
		if err != nil {
			return err
		}
		if len(response.Variables) == 0 {
			break RequestLoop
		}

		if response.Error == NoSuchName {
			x.Logger.Print("Walk terminated with NoSuchName")
			break RequestLoop
		}

		for k, v := range response.Variables {
			if v.Type == EndOfMibView || v.Type == NoSuchObject || v.Type == NoSuchInstance {
				x.Logger.Printf("BulkWalk terminated with type 0x%x", v.Type)
				break RequestLoop
			}
			if !strings.HasPrefix(v.Name, rootOid) {
				// Not in the requested root range.
				// if this is the first request, and the first variable in that request
				// and this condition is triggered - the first result is out of range
				// need to perform a regular get request
				// this request has been too narrowly defined to be found with a getNext
				// Issue #78
				if requests == 1 && k == 0 {
					err = x.getToWalk(rootOid, walkFn)
					if err != nil {
						return err
					}
				}
				break RequestLoop
			}
			if v.Name == oid {
				return fmt.Errorf("OID not increasing: %s", v.Name)
			}
			// Report our pdu
			if err := walkFn(v); err != nil {
				return err
			}
		}
		// Save last oid for next request
		oid = response.Variables[len(response.Variables)-1].Name
	}
	x.Logger.Printf("BulkWalk completed in %d requests", requests)
	return nil
}

func (x *GoSNMP) walkAll(getRequestType PDUType, rootOid string) (results []SnmpPDU, err error) {
	err = x.walk(getRequestType, rootOid, func(dataUnit SnmpPDU) error {
		results = append(results, dataUnit)
		return nil
	})
	return results, err
}

func (x *GoSNMP) getToWalk(rootOid string, walkFn WalkFunc) error {
	response, err := x.Get([]string{rootOid})
	if err != nil {
		return err
	}

	for _, v := range response.Variables {
		err = walkFn(v)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Sequential WARC file reader library supporting record-at-time compression
// this requires go1.4 due to its usage of compress/gzip.Reader#Multistream
//
// Currently only works on record-at-time compressed .gz files
//
// [Example implementation](https://github.com/sebcat/warc-urls)
package warc

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"errors"
	"io"
	"strconv"
	"strings"
)

var (
	ErrMalformedRecord = errors.New("malformed record")
	ErrNonWARCRecord   = errors.New("non-WARC/1.0 record")
	ErrOffsetOverflow  = errors.New("offset overflow")
	ErrNotASeeker      = errors.New("the underlying stream is not seekable")
)

// a combo of a buffered reader and an offset counter.
// we're doing this because the alternative approach of
// wrapping bufio steals a bit of time since the reader
// will be called in a tight loop when used as an
// io.ByteReader inside
// compress/flate.(*decompressor).moreBits (go1.4.2)
type reader struct {
	r io.Reader

	rbuf     [4096]byte
	nbufleft int
	rpos     int

	// number of bytes having been read from this reader
	// (i.e., not from r)
	nread int64
}

func (r *reader) next() (n int, err error) {
	n, err = r.r.Read(r.rbuf[:])
	r.nbufleft = n
	r.rpos = 0
	return
}

func (r *reader) Read(p []byte) (n int, err error) {
	ncopy := len(p)
	if r.nbufleft == 0 {
		ncopy, err = r.next()
	}

	if ncopy > r.nbufleft {
		ncopy = r.nbufleft
	}

	n = copy(p, r.rbuf[r.rpos:r.rpos+r.nbufleft])
	r.rpos += n
	r.nread += int64(n)
	r.nbufleft -= n
	return
}

// This is a bit of a bottle neck, as it's called often
func (r *reader) ReadByte() (c byte, err error) {
	if r.nbufleft == 0 {
		if _, err = r.next(); r.nbufleft == 0 {
			return 0, err
		}
	}

	c = r.rbuf[r.rpos]
	r.nread++
	r.rpos++
	r.nbufleft--
	return
}

type Offset int64

type Reader struct {
	r    *reader
	zr   *gzip.Reader
	last int64
}

type NamedField struct {
	Name  string
	Value string
}

type NamedFields []NamedField

type Record struct {
	Fields NamedFields
	Block  []byte
}

func (r *Record) Bytes() []byte {
	var b bytes.Buffer
	b.Write([]byte("WARC/1.0\r\n"))
	for _, field := range r.Fields {
		b.Write([]byte(field.Name + ": " + field.Value + "\r\n"))
	}

	b.Write([]byte("\r\n"))
	b.Write([]byte(r.Block))
	b.Write([]byte("\r\n\r\n"))
	return b.Bytes()
}

func (r *Record) FromBytes(rec []byte) error {
	// TODO: Maybe use net/textproto for all of this in here?
	parts := bytes.SplitN(rec, []byte("\r\n\r\n"), 2)
	if len(parts) != 2 {
		return ErrMalformedRecord
	}

	hdr, block := parts[0], parts[1]
	r.Block = block
	for ix, hdrline := range bytes.Split(hdr, []byte("\r\n")) {
		if ix == 0 {
			if string(hdrline) != "WARC/1.0" {
				return ErrNonWARCRecord
			}

			continue
		}

		l := bytes.SplitN(hdrline, []byte(":"), 2)
		if len(l) != 2 {
			return ErrMalformedRecord
		}

		r.Fields = append(r.Fields, NamedField{
			Name:  string(bytes.Trim(l[0], "\r\n\t ")),
			Value: string(bytes.Trim(l[1], "\r\n\t ")),
		})
	}

	lenStr := r.Fields.Value("content-length")
	if i, err := strconv.Atoi(lenStr); err == nil {
		r.Block = r.Block[:i]
	}

	return nil
}

func (f NamedFields) Value(name string) string {
	for _, el := range f {
		if strings.EqualFold(el.Name, name) {
			return el.Value
		}
	}

	return ""
}

func NewReader(r io.Reader) *Reader {
	return &Reader{
		r: &reader{
			r: r,
		},
	}
}

func NewGZIPReader(reader io.Reader) (r *Reader, err error) {
	r = NewReader(reader)
	r.zr, err = gzip.NewReader(r.r)
	if err != nil {
		return nil, err
	}

	r.zr.Multistream(false)
	return r, nil
}

func (r *Reader) gzipRecord() ([]byte, error) {
	var rec bytes.Buffer
	_, err := io.Copy(&rec, r.zr)
	if err != nil {
		return nil, err
	}

	if r.last == r.r.nread {
		return nil, io.EOF
	}

	r.last = r.r.nread
	r.zr.Reset(r.r)
	r.zr.Multistream(false)
	return rec.Bytes(), nil
}

func (r *Reader) plainRecord() ([]byte, error) {
	// TODO: Implement
	return nil, errors.New("support for non record-at-time compressed WARC files not yet implemented")
}

func (r *Reader) record() ([]byte, error) {
	if r.zr != nil {
		return r.gzipRecord()
	} else {
		return r.plainRecord()
	}
}

// Scans a stream for a raw WARC record. Doesn't do any
// validation or parsing. Useful for concurrency pipelines
// where the parsing and message handling is fanned out to
// multiple goroutines. See Record#FromBytes
func (r *Reader) NextRaw() ([]byte, error) {
	return r.record()
}

// Scans and parses a WARC record from a stream.
// returns io.EOF when done
func (r *Reader) Next() (*Record, error) {
	var res Record

	rec, err := r.record()
	if err != nil {
		return nil, err
	}

	if err := res.FromBytes(rec); err != nil {
		return nil, err
	}

	return &res, nil
}

// Scans a raw WARC record from a stream at a specific offset
// from the start of the stream.
// The original Reader passed to NewReader must implement the
// io.Seeker interface. The Reader stream will be at the
// position after the read record on successful return.
func (r *Reader) NextRawAt(offset Offset) ([]byte, error) {
	if seeker, ok := r.r.r.(io.Seeker); ok {
		if _, err := seeker.Seek(int64(offset), 0); err != nil {
			return nil, err
		}
	} else {
		return nil, ErrNotASeeker
	}

	return r.record()
}

// Scans and parses a WARC record from a stream at a specific offset
// from the start of the stream.
// The original Reader passed to NewReader must implement the
// io.Seeker interface. The Reader stream will be at the
// position after the read record on successful return.
func (r *Reader) NextAt(offset Offset) (*Record, error) {
	if seeker, ok := r.r.r.(io.Seeker); ok {
		if _, err := seeker.Seek(int64(offset), 0); err != nil {
			return nil, err
		}
	} else {
		return nil, ErrNotASeeker
	}

	return r.Next()
}

type cwriter struct {
	c Offset
	w io.Writer
}

func newCWriter(w io.Writer) *cwriter {
	return &cwriter{w: w}
}

func (w *cwriter) Write(p []byte) (n int, err error) {
	before := w.c
	n, err = w.w.Write(p)
	w.c += Offset(n)
	if w.c < before {
		err = ErrOffsetOverflow
	}

	return
}

func (w *cwriter) Offset() Offset {
	return w.c
}

type Writer struct {
	cw    *cwriter
	zw    *gzip.Writer
	index io.Writer
}

func NewIndexingWriter(w io.Writer, index io.Writer) *Writer {
	cw := newCWriter(w)
	zw := gzip.NewWriter(cw)
	return &Writer{cw: cw, zw: zw, index: index}
}

func NewWriter(w io.Writer) *Writer {
	return NewIndexingWriter(w, nil)
}

// Write a record. No validation of mandatory WARC fields is performed.
// The written record will be an independent GZIP stream.
func (w *Writer) WriteRecord(r *Record) error {
	rec := r.Bytes()
	offset := w.cw.Offset()
	_, err := w.zw.Write(rec)
	if err != nil {
		return err
	}

	if err := w.zw.Close(); err != nil {
		return err
	}

	w.zw.Reset(w.cw)
	if w.index != nil {
		if err := binary.Write(w.index, binary.LittleEndian, &offset); err != nil {
			return err
		}
	}

	return nil
}

// Read an Offset from a WARC index
func ReadOffset(r io.Reader) (Offset, error) {
	var offset Offset
	err := binary.Read(r, binary.LittleEndian, &offset)
	return offset, err
}
<commit_msg>Updated docs<commit_after>// WARC package supporting record-at-time compression.
// Requires go1.4 due to its usage of compress/gzip.Reader#Multistream
//
// Currently only works on record-at-time compressed .gz files.
//
// Supports indexed operations for concurrent reading. One use-case
// is to have one goroutine passing Offset's to a channel that
// multiple goroutines read from. Each of these goroutines have
// their own WARC Reader and can read and decompress a record
// independantly.
//
// [Example implementation](https://github.com/sebcat/warc-urls)
package warc

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"errors"
	"io"
	"strconv"
	"strings"
)

var (
	ErrMalformedRecord = errors.New("malformed record")
	ErrNonWARCRecord   = errors.New("non-WARC/1.0 record")
	ErrOffsetOverflow  = errors.New("offset overflow")
	ErrNotASeeker      = errors.New("the underlying stream is not seekable")
)

// a combo of a buffered reader and an offset counter.
// we're doing this because the alternative approach of
// wrapping bufio steals a bit of time since the reader
// will be called in a tight loop when used as an
// io.ByteReader inside
// compress/flate.(*decompressor).moreBits (go1.4.2)
type reader struct {
	r io.Reader

	rbuf     [4096]byte
	nbufleft int
	rpos     int

	// number of bytes having been read from this reader
	// (i.e., not from r)
	nread int64
}

func (r *reader) next() (n int, err error) {
	n, err = r.r.Read(r.rbuf[:])
	r.nbufleft = n
	r.rpos = 0
	return
}

func (r *reader) Read(p []byte) (n int, err error) {
	ncopy := len(p)
	if r.nbufleft == 0 {
		ncopy, err = r.next()
	}

	if ncopy > r.nbufleft {
		ncopy = r.nbufleft
	}

	n = copy(p, r.rbuf[r.rpos:r.rpos+r.nbufleft])
	r.rpos += n
	r.nread += int64(n)
	r.nbufleft -= n
	return
}

// This is a bit of a bottle neck, as it's called often
func (r *reader) ReadByte() (c byte, err error) {
	if r.nbufleft == 0 {
		if _, err = r.next(); r.nbufleft == 0 {
			return 0, err
		}
	}

	c = r.rbuf[r.rpos]
	r.nread++
	r.rpos++
	r.nbufleft--
	return
}

type Offset int64

type Reader struct {
	r    *reader
	zr   *gzip.Reader
	last int64
}

type NamedField struct {
	Name  string
	Value string
}

type NamedFields []NamedField

type Record struct {
	Fields NamedFields
	Block  []byte
}

func (r *Record) Bytes() []byte {
	var b bytes.Buffer
	b.Write([]byte("WARC/1.0\r\n"))
	for _, field := range r.Fields {
		b.Write([]byte(field.Name + ": " + field.Value + "\r\n"))
	}

	b.Write([]byte("\r\n"))
	b.Write([]byte(r.Block))
	b.Write([]byte("\r\n\r\n"))
	return b.Bytes()
}

func (r *Record) FromBytes(rec []byte) error {
	// TODO: Maybe use net/textproto for all of this in here?
	parts := bytes.SplitN(rec, []byte("\r\n\r\n"), 2)
	if len(parts) != 2 {
		return ErrMalformedRecord
	}

	hdr, block := parts[0], parts[1]
	r.Block = block
	for ix, hdrline := range bytes.Split(hdr, []byte("\r\n")) {
		if ix == 0 {
			if string(hdrline) != "WARC/1.0" {
				return ErrNonWARCRecord
			}

			continue
		}

		l := bytes.SplitN(hdrline, []byte(":"), 2)
		if len(l) != 2 {
			return ErrMalformedRecord
		}

		r.Fields = append(r.Fields, NamedField{
			Name:  string(bytes.Trim(l[0], "\r\n\t ")),
			Value: string(bytes.Trim(l[1], "\r\n\t ")),
		})
	}

	lenStr := r.Fields.Value("content-length")
	if i, err := strconv.Atoi(lenStr); err == nil {
		r.Block = r.Block[:i]
	}

	return nil
}

func (f NamedFields) Value(name string) string {
	for _, el := range f {
		if strings.EqualFold(el.Name, name) {
			return el.Value
		}
	}

	return ""
}

func NewReader(r io.Reader) *Reader {
	return &Reader{
		r: &reader{
			r: r,
		},
	}
}

func NewGZIPReader(reader io.Reader) (r *Reader, err error) {
	r = NewReader(reader)
	r.zr, err = gzip.NewReader(r.r)
	if err != nil {
		return nil, err
	}

	r.zr.Multistream(false)
	return r, nil
}

func (r *Reader) gzipRecord() ([]byte, error) {
	var rec bytes.Buffer
	_, err := io.Copy(&rec, r.zr)
	if err != nil {
		return nil, err
	}

	if r.last == r.r.nread {
		return nil, io.EOF
	}

	r.last = r.r.nread
	r.zr.Reset(r.r)
	r.zr.Multistream(false)
	return rec.Bytes(), nil
}

func (r *Reader) plainRecord() ([]byte, error) {
	// TODO: Implement
	return nil, errors.New("support for non record-at-time compressed WARC files not yet implemented")
}

func (r *Reader) record() ([]byte, error) {
	if r.zr != nil {
		return r.gzipRecord()
	} else {
		return r.plainRecord()
	}
}

// Scans a stream for a raw WARC record. Doesn't do any
// validation or parsing. Useful for concurrency pipelines
// where the parsing and message handling is fanned out to
// multiple goroutines. See Record#FromBytes
func (r *Reader) NextRaw() ([]byte, error) {
	return r.record()
}

// Scans and parses a WARC record from a stream.
// returns io.EOF when done
func (r *Reader) Next() (*Record, error) {
	var res Record

	rec, err := r.record()
	if err != nil {
		return nil, err
	}

	if err := res.FromBytes(rec); err != nil {
		return nil, err
	}

	return &res, nil
}

// Scans a raw WARC record from a stream at a specific offset
// from the start of the stream.
// The original Reader passed to NewReader must implement the
// io.Seeker interface. The Reader stream will be at the
// position after the read record on successful return.
func (r *Reader) NextRawAt(offset Offset) ([]byte, error) {
	if seeker, ok := r.r.r.(io.Seeker); ok {
		if _, err := seeker.Seek(int64(offset), 0); err != nil {
			return nil, err
		}
	} else {
		return nil, ErrNotASeeker
	}

	return r.record()
}

// Scans and parses a WARC record from a stream at a specific offset
// from the start of the stream.
// The original Reader passed to NewReader must implement the
// io.Seeker interface. The Reader stream will be at the
// position after the read record on successful return.
func (r *Reader) NextAt(offset Offset) (*Record, error) {
	if seeker, ok := r.r.r.(io.Seeker); ok {
		if _, err := seeker.Seek(int64(offset), 0); err != nil {
			return nil, err
		}
	} else {
		return nil, ErrNotASeeker
	}

	return r.Next()
}

type cwriter struct {
	c Offset
	w io.Writer
}

func newCWriter(w io.Writer) *cwriter {
	return &cwriter{w: w}
}

func (w *cwriter) Write(p []byte) (n int, err error) {
	before := w.c
	n, err = w.w.Write(p)
	w.c += Offset(n)
	if w.c < before {
		err = ErrOffsetOverflow
	}

	return
}

func (w *cwriter) Offset() Offset {
	return w.c
}

type Writer struct {
	cw    *cwriter
	zw    *gzip.Writer
	index io.Writer
}

func NewIndexingWriter(w io.Writer, index io.Writer) *Writer {
	cw := newCWriter(w)
	zw := gzip.NewWriter(cw)
	return &Writer{cw: cw, zw: zw, index: index}
}

func NewWriter(w io.Writer) *Writer {
	return NewIndexingWriter(w, nil)
}

// Write a record. No validation of mandatory WARC fields is performed.
// The written record will be an independent GZIP stream.
func (w *Writer) WriteRecord(r *Record) error {
	rec := r.Bytes()
	offset := w.cw.Offset()
	_, err := w.zw.Write(rec)
	if err != nil {
		return err
	}

	if err := w.zw.Close(); err != nil {
		return err
	}

	w.zw.Reset(w.cw)
	if w.index != nil {
		if err := binary.Write(w.index, binary.LittleEndian, &offset); err != nil {
			return err
		}
	}

	return nil
}

// Read an Offset from a WARC index
func ReadOffset(r io.Reader) (Offset, error) {
	var offset Offset
	err := binary.Read(r, binary.LittleEndian, &offset)
	return offset, err
}
<|endoftext|>


<commit_before>package emitter_test

import (
	"net"
	"sync"

	"github.com/cloudfoundry/dropsonde/emitter"
	"github.com/cloudfoundry/dropsonde/factories"
	"github.com/cloudfoundry/sonde-go/control"
	"github.com/gogo/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("UdpEmitter", func() {
	var testData = []byte("hello")

	Describe("Close()", func() {
		It("closes the UDP connection", func() {

			udpEmitter, _ := emitter.NewUdpEmitter("localhost:42420")

			udpEmitter.Close()

			err := udpEmitter.Emit(testData)
			Expect(err).To(HaveOccurred())
			Expect(err.Error()).To(ContainSubstring("use of closed network connection"))
		})
	})

	Describe("Emit()", func() {
		var udpEmitter emitter.ByteEmitter

		Context("when the agent is listening", func() {

			var agentListener net.PacketConn

			BeforeEach(func() {
				var err error
				agentListener, err = net.ListenPacket("udp4", "")
				Expect(err).ToNot(HaveOccurred())

				udpEmitter, err = emitter.NewUdpEmitter(agentListener.LocalAddr().String())
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				agentListener.Close()
			})

			It("should send the data", func(done Done) {
				err := udpEmitter.Emit(testData)
				Expect(err).ToNot(HaveOccurred())

				buffer := make([]byte, 4096)
				readCount, _, err := agentListener.ReadFrom(buffer)
				Expect(err).ToNot(HaveOccurred())
				Expect(buffer[:readCount]).To(Equal(testData))

				close(done)
			})
		})

		Context("when the agent is not listening", func() {
			BeforeEach(func() {
				udpEmitter, _ = emitter.NewUdpEmitter("localhost:12345")
			})

			It("should attempt to send the data", func() {
				err := udpEmitter.Emit(testData)
				Expect(err).ToNot(HaveOccurred())
			})

			Context("then the agent starts Listening", func() {
				It("should eventually send data", func(done Done) {
					err := udpEmitter.Emit(testData)
					Expect(err).ToNot(HaveOccurred())

					agentListener, err := net.ListenPacket("udp4", ":12345")
					Expect(err).ToNot(HaveOccurred())

					err = udpEmitter.Emit(testData)
					Expect(err).ToNot(HaveOccurred())

					buffer := make([]byte, 4096)
					readCount, _, err := agentListener.ReadFrom(buffer)
					Expect(err).ToNot(HaveOccurred())
					Expect(buffer[:readCount]).To(Equal(testData))

					close(done)
				})
			})
		})
	})

	Describe("NewUdpEmitter()", func() {
		Context("when ResolveUDPAddr fails", func() {
			It("returns an error", func() {
				emitter, err := emitter.NewUdpEmitter("invalid-address:")
				Expect(emitter).To(BeNil())
				Expect(err).To(HaveOccurred())
			})
		})

		Context("when all is good", func() {
			It("creates an emitter", func() {
				emitter, err := emitter.NewUdpEmitter("localhost:123")
				Expect(emitter).ToNot(BeNil())
				Expect(err).ToNot(HaveOccurred())
			})
		})
	})

	Describe("ListenForHeartbeatRequest", func() {
		var timesCalled int
		var lastControlMessage *control.ControlMessage
		var lock sync.Mutex

		var fakeResponder = func(controlMessage *control.ControlMessage) {
			lock.Lock()
			defer lock.Unlock()
			timesCalled++
			lastControlMessage = controlMessage
		}

		var getTimesCalled = func() int {
			lock.Lock()
			defer lock.Unlock()
			return timesCalled
		}

		var getReceivedHeartbeatRequest = func() *control.ControlMessage {
			lock.Lock()
			defer lock.Unlock()

			return lastControlMessage
		}

		BeforeEach(func() {
			lock = sync.Mutex{}
			timesCalled = 0
		})

		It("calls responder with the correct heartbeat request when when heartbeat is requested", func() {
			emitter, _ := emitter.NewUdpEmitter("localhost:123")
			go emitter.ListenForHeartbeatRequest(fakeResponder)

			Expect(timesCalled).To(BeZero())

			heartbeatRequest := newHeartbeatRequest()

			sendHeartbeatRequest(emitter.Address(), heartbeatRequest)
			Eventually(getTimesCalled).Should(Equal(1))
			Expect(getReceivedHeartbeatRequest()).To(Equal(heartbeatRequest))
		})

		It("responds to multiple heartbeat requests", func() {
			emitter, _ := emitter.NewUdpEmitter("localhost:123")
			go emitter.ListenForHeartbeatRequest(fakeResponder)
			sendHeartbeatRequest(emitter.Address(), newHeartbeatRequest())
			sendHeartbeatRequest(emitter.Address(), newHeartbeatRequest())

			Eventually(getTimesCalled).Should(Equal(2))
		})

		It("returns an error if listening on the UDP port fails", func() {
			emitter, _ := emitter.NewUdpEmitter("localhost:123")
			emitter.Close()
			err := emitter.ListenForHeartbeatRequest(fakeResponder)
			Expect(err).To(HaveOccurred())
			Expect(err).To(MatchError("use of closed network connection"))
		})
	})
})

func sendHeartbeatRequest(addr net.Addr, message *control.ControlMessage) {
	encodedMessage, _ := proto.Marshal(message)
	conn, _ := net.ListenPacket("udp4", "")
	conn.WriteTo(encodedMessage, addr)
}

func newHeartbeatRequest() *control.ControlMessage {
	id, _ := uuid.NewV4()

	return &control.ControlMessage{
		Origin:      proto.String("test"),
		Identifier:  factories.NewControlUUID(id),
		Timestamp:   proto.Int64(0),
		ControlType: control.ControlMessage_HeartbeatRequest.Enum(),
	}
}
<commit_msg>Update test to check for error substring<commit_after>package emitter_test

import (
	"net"
	"sync"

	"github.com/cloudfoundry/dropsonde/emitter"
	"github.com/cloudfoundry/dropsonde/factories"
	"github.com/cloudfoundry/sonde-go/control"
	"github.com/gogo/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("UdpEmitter", func() {
	var testData = []byte("hello")

	Describe("Close()", func() {
		It("closes the UDP connection", func() {

			udpEmitter, _ := emitter.NewUdpEmitter("localhost:42420")

			udpEmitter.Close()

			err := udpEmitter.Emit(testData)
			Expect(err).To(HaveOccurred())
			Expect(err.Error()).To(ContainSubstring("use of closed network connection"))
		})
	})

	Describe("Emit()", func() {
		var udpEmitter emitter.ByteEmitter

		Context("when the agent is listening", func() {

			var agentListener net.PacketConn

			BeforeEach(func() {
				var err error
				agentListener, err = net.ListenPacket("udp4", "")
				Expect(err).ToNot(HaveOccurred())

				udpEmitter, err = emitter.NewUdpEmitter(agentListener.LocalAddr().String())
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				agentListener.Close()
			})

			It("should send the data", func(done Done) {
				err := udpEmitter.Emit(testData)
				Expect(err).ToNot(HaveOccurred())

				buffer := make([]byte, 4096)
				readCount, _, err := agentListener.ReadFrom(buffer)
				Expect(err).ToNot(HaveOccurred())
				Expect(buffer[:readCount]).To(Equal(testData))

				close(done)
			})
		})

		Context("when the agent is not listening", func() {
			BeforeEach(func() {
				udpEmitter, _ = emitter.NewUdpEmitter("localhost:12345")
			})

			It("should attempt to send the data", func() {
				err := udpEmitter.Emit(testData)
				Expect(err).ToNot(HaveOccurred())
			})

			Context("then the agent starts Listening", func() {
				It("should eventually send data", func(done Done) {
					err := udpEmitter.Emit(testData)
					Expect(err).ToNot(HaveOccurred())

					agentListener, err := net.ListenPacket("udp4", ":12345")
					Expect(err).ToNot(HaveOccurred())

					err = udpEmitter.Emit(testData)
					Expect(err).ToNot(HaveOccurred())

					buffer := make([]byte, 4096)
					readCount, _, err := agentListener.ReadFrom(buffer)
					Expect(err).ToNot(HaveOccurred())
					Expect(buffer[:readCount]).To(Equal(testData))

					close(done)
				})
			})
		})
	})

	Describe("NewUdpEmitter()", func() {
		Context("when ResolveUDPAddr fails", func() {
			It("returns an error", func() {
				emitter, err := emitter.NewUdpEmitter("invalid-address:")
				Expect(emitter).To(BeNil())
				Expect(err).To(HaveOccurred())
			})
		})

		Context("when all is good", func() {
			It("creates an emitter", func() {
				emitter, err := emitter.NewUdpEmitter("localhost:123")
				Expect(emitter).ToNot(BeNil())
				Expect(err).ToNot(HaveOccurred())
			})
		})
	})

	Describe("ListenForHeartbeatRequest", func() {
		var timesCalled int
		var lastControlMessage *control.ControlMessage
		var lock sync.Mutex

		var fakeResponder = func(controlMessage *control.ControlMessage) {
			lock.Lock()
			defer lock.Unlock()
			timesCalled++
			lastControlMessage = controlMessage
		}

		var getTimesCalled = func() int {
			lock.Lock()
			defer lock.Unlock()
			return timesCalled
		}

		var getReceivedHeartbeatRequest = func() *control.ControlMessage {
			lock.Lock()
			defer lock.Unlock()

			return lastControlMessage
		}

		BeforeEach(func() {
			lock = sync.Mutex{}
			timesCalled = 0
		})

		It("calls responder with the correct heartbeat request when when heartbeat is requested", func() {
			emitter, _ := emitter.NewUdpEmitter("localhost:123")
			go emitter.ListenForHeartbeatRequest(fakeResponder)

			Expect(timesCalled).To(BeZero())

			heartbeatRequest := newHeartbeatRequest()

			sendHeartbeatRequest(emitter.Address(), heartbeatRequest)
			Eventually(getTimesCalled).Should(Equal(1))
			Expect(getReceivedHeartbeatRequest()).To(Equal(heartbeatRequest))
		})

		It("responds to multiple heartbeat requests", func() {
			emitter, _ := emitter.NewUdpEmitter("localhost:123")
			go emitter.ListenForHeartbeatRequest(fakeResponder)
			sendHeartbeatRequest(emitter.Address(), newHeartbeatRequest())
			sendHeartbeatRequest(emitter.Address(), newHeartbeatRequest())

			Eventually(getTimesCalled).Should(Equal(2))
		})

		It("returns an error if listening on the UDP port fails", func() {
			emitter, _ := emitter.NewUdpEmitter("localhost:123")
			emitter.Close()
			err := emitter.ListenForHeartbeatRequest(fakeResponder)
			Expect(err).To(HaveOccurred())
			Expect(err).To(MatchError(ContainSubstring("use of closed network connection")))
		})
	})
})

func sendHeartbeatRequest(addr net.Addr, message *control.ControlMessage) {
	encodedMessage, _ := proto.Marshal(message)
	conn, _ := net.ListenPacket("udp4", "")
	conn.WriteTo(encodedMessage, addr)
}

func newHeartbeatRequest() *control.ControlMessage {
	id, _ := uuid.NewV4()

	return &control.ControlMessage{
		Origin:      proto.String("test"),
		Identifier:  factories.NewControlUUID(id),
		Timestamp:   proto.Int64(0),
		ControlType: control.ControlMessage_HeartbeatRequest.Enum(),
	}
}
<|endoftext|>