<commit_before>package main

import (
  "bitbucket.org/jol/service/stdservice"
  "flag"
  "fmt"
  "github.com/foi-oss/schoolware/scenarios"
  "log"
  "os"
  "os/user"
  "strings"
)

var (
  // command line flag
  scenario = flag.String("scenario", "", "scenario to run")
  homedir  = flag.String("homedir", "~", "home directory")
)

func main() {
  flag.Usage = usage
  flag.Parse()

  u, _ := user.Current()
  args := strings.Join(os.Args[1:len(os.Args)-1], " ") + " -homedir=\"" + u.HomeDir + "\""

  stdservice.Run(&stdservice.Config{
    Name:            "schoolware",
    DisplayName:     "Schoolware",
    LongDescription: "School malware service",
    Start:           start,
    Stop:            stop,
    Args:            args,
  })
}

func start(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware started")

  if len(*scenario) == 0 {
    fmt.Fprintf(os.Stderr, "no scenario specified")
    l.Error("no scenario specified")
    return
  }

  if *homedir == "~" {
    u, _ := user.Current()
    scenarios.HomeDir = u.HomeDir
  } else {
    scenarios.HomeDir = *homedir
  }

  for _, s := range scenarios.All {
    if s.Name == *scenario {
      l.Info(fmt.Sprintf("scenario %s started", *scenario))

      err := s.Run(c)
      if err != nil {
        l.Error("scenario failed with: " + err.Error())
        log.Panicln("scenario failed with:", err.Error())
      }

      return
    }
  }

  fmt.Fprintf(os.Stderr, "unknown scenario specified")
}

func stop(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware is shutting down")
}

// usage prints list of known command-line options and scenarion descriptions
func usage() {
  fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
  fmt.Fprintf(os.Stderr, "  %s [--scenario=net|files|... OPTIONS] COMMAND\n\n", os.Args[0])

  fmt.Fprintln(os.Stderr, "Options:")
  flag.PrintDefaults()

  fmt.Fprintln(os.Stderr, "\nScenarios:")
  for _, s := range scenarios.All {
    fmt.Fprintf(os.Stderr, "  %s: %s\n", s.Name, s.Description)
  }
  fmt.Fprintln(os.Stderr, "\nOptions for each scenario are prefixed with its name.")
  fmt.Fprintln(os.Stderr, "\nCommands:")
  fmt.Fprintln(os.Stderr, "  run\t\timmediately run specified scenario\n"+
    "  install\tinstall background service\n"+
    "  start\t\tstart previously installed service\n"+
    "  stop\t\tstops the service\n"+
    "  remove\tremoves schoolware service from the systems")
}
<commit_msg>formatting<commit_after>package main

import (
  "bitbucket.org/jol/service/stdservice"
  "flag"
  "fmt"
  "github.com/foi-oss/schoolware/scenarios"
  "os"
  "os/user"
  "strings"
)

var (
  // command line flag
  scenario = flag.String("scenario", "", "scenario to run")
  homedir  = flag.String("homedir", "~", "home directory")
)

func main() {
  flag.Usage = usage
  flag.Parse()

  u, _ := user.Current()
  args := strings.Join(os.Args[1:len(os.Args)-1], " ") + " -homedir=\"" + u.HomeDir + "\""

  stdservice.Run(&stdservice.Config{
    Name:            "schoolware",
    DisplayName:     "Schoolware",
    LongDescription: "School malware service",
    Start:           start,
    Stop:            stop,
    Args:            args,
  })
}

func start(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware started")

  if len(*scenario) == 0 {
    l.Error("no scenario specified")
    return
  }

  if *homedir == "~" {
    u, _ := user.Current()
    scenarios.HomeDir = u.HomeDir
  } else {
    scenarios.HomeDir = *homedir
  }

  for _, s := range scenarios.All {
    if s.Name == *scenario {
      l.Info(fmt.Sprintf("scenario %s started", *scenario))

      err := s.Run(c)
      if err != nil {
        l.Error("scenario failed with: " + err.Error())
      }

      return
    }
  }

  l.Error("unknown scenario specified")
}

func stop(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware is shutting down")
}

// usage prints list of known command-line options and scenarion descriptions
func usage() {
  fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
  fmt.Fprintf(os.Stderr, "  %s [--scenario=net|files|... OPTIONS] COMMAND\n\n", os.Args[0])

  fmt.Fprintln(os.Stderr, "Options:")
  flag.PrintDefaults()

  fmt.Fprintln(os.Stderr, "\nScenarios:")
  for _, s := range scenarios.All {
    fmt.Fprintf(os.Stderr, "  %s: %s\n", s.Name, s.Description)
  }
  fmt.Fprintln(os.Stderr, "\nOptions for each scenario are prefixed with its name.")
  fmt.Fprintln(os.Stderr, "\nCommands:")
  fmt.Fprintln(os.Stderr, `
  run     immediately run specified scenario
  install install background service
  start   start previously installed service
  stop    stops the service
  remove  removes schoolware service from the systems`)
}
<|endoftext|>


<commit_before>package main

import (
  "bitbucket.org/jol/service/stdservice"
  "flag"
  "fmt"
  "github.com/foi-oss/schoolware/scenarios"
  "os"
  "os/user"
  "strings"
)

var (
  // command line flag
  scenario = flag.String("scenario", "", "scenario to run")
  homedir  = flag.String("homedir", "~", "home directory")
)

func main() {
  flag.Usage = usage
  flag.Parse()

  u, _ := user.Current()
  args := strings.Join(os.Args[1:len(os.Args)-1], " ") + " -homedir=\"" + u.HomeDir + "\""

  stdservice.Run(&stdservice.Config{
    Name:            "schoolware",
    DisplayName:     "Schoolware",
    LongDescription: "School malware service",
    Start:           start,
    Stop:            stop,
    Args:            args,
  })
}

func start(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware started")

  if len(*scenario) == 0 {
    l.Error("no scenario specified")
    return
  }

  if *homedir == "~" {
    u, _ := user.Current()
    scenarios.HomeDir = u.HomeDir
  } else {
    scenarios.HomeDir = *homedir
  }

  for _, s := range scenarios.All {
    if s.Name == *scenario {
      l.Info(fmt.Sprintf("scenario %s started", *scenario))

      err := s.Run(c)
      if err != nil {
        l.Error("scenario failed with: " + err.Error())
      }

      return
    }
  }

  l.Error("unknown scenario specified")
}

func stop(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware is shutting down")
}

// usage prints list of known command-line options and scenarion descriptions
func usage() {
  fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
  fmt.Fprintf(os.Stderr, "  %s [--scenario=net|files|... OPTIONS] COMMAND\n\n", os.Args[0])

  fmt.Fprintln(os.Stderr, "Options:")
  flag.PrintDefaults()

  fmt.Fprintln(os.Stderr, "\nScenarios:")
  for _, s := range scenarios.All {
    fmt.Fprintf(os.Stderr, "  %s: %s\n", s.Name, s.Description)
  }
  fmt.Fprintln(os.Stderr, "\nOptions for each scenario are prefixed with its name.")
  fmt.Fprintln(os.Stderr, "\nCommands:")
  fmt.Fprintln(os.Stderr, `
  run     immediately run specified scenario
  install install background service
  start   start previously installed service
  stop    stops the service
  remove  removes schoolware service from the systems`)
}
<commit_msg>do not error-out when no arguments are given<commit_after>package main

import (
  "bitbucket.org/jol/service/stdservice"
  "flag"
  "fmt"
  "github.com/foi-oss/schoolware/scenarios"
  "os"
  "os/user"
  "strings"
)

var (
  // command line flag
  scenario = flag.String("scenario", "", "scenario to run")
  homedir  = flag.String("homedir", "~", "home directory")
)

func main() {
  if len(os.Args) == 1 {
    fmt.Fprintf(os.Stderr, "Missing arguments. Seek --help")
    return
  }

  flag.Usage = usage
  flag.Parse()

  u, _ := user.Current()

  args := strings.Join(os.Args[1:len(os.Args)-1], " ") + " -homedir=\"" + u.HomeDir + "\""

  stdservice.Run(&stdservice.Config{
    Name:            "schoolware",
    DisplayName:     "Schoolware",
    LongDescription: "School malware service",
    Start:           start,
    Stop:            stop,
    Args:            args,
  })
}

func start(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware started")

  if len(*scenario) == 0 {
    l.Error("no scenario specified")
    return
  }

  if *homedir == "~" {
    u, _ := user.Current()
    scenarios.HomeDir = u.HomeDir
  } else {
    scenarios.HomeDir = *homedir
  }

  for _, s := range scenarios.All {
    if s.Name == *scenario {
      l.Info(fmt.Sprintf("scenario %s started", *scenario))

      err := s.Run(c)
      if err != nil {
        l.Error("scenario failed with: " + err.Error())
      }

      return
    }
  }

  l.Error("unknown scenario specified")
}

func stop(c *stdservice.Config) {
  l := c.Logger()
  l.Info("schoolware is shutting down")
}

// usage prints list of known command-line options and scenarion descriptions
func usage() {
  fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
  fmt.Fprintf(os.Stderr, "  %s [--scenario=net|files|... OPTIONS] COMMAND\n\n", os.Args[0])

  fmt.Fprintln(os.Stderr, "Options:")
  flag.PrintDefaults()

  fmt.Fprintln(os.Stderr, "\nScenarios:")
  for _, s := range scenarios.All {
    fmt.Fprintf(os.Stderr, "  %s: %s\n", s.Name, s.Description)
  }
  fmt.Fprintln(os.Stderr, "\nOptions for each scenario are prefixed with its name.")
  fmt.Fprintln(os.Stderr, "\nCommands:")
  fmt.Fprintln(os.Stderr, `
  run     immediately run specified scenario
  install install background service
  start   start previously installed service
  stop    stops the service
  remove  removes schoolware service from the systems`)
}
<|endoftext|>


<commit_before>package htmlinfo

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/html/charset"

	"github.com/dyatlov/go-oembed/oembed"
	"github.com/dyatlov/go-opengraph/opengraph"
	"golang.org/x/net/html"

	"github.com/dyatlov/go-readability"
)

// TouchIcon contains all icons parsed from page header, including Apple touch icons
type TouchIcon struct {
	URL        string `json:"url"`
	Type       string `json:"type"`
	Width      uint64 `json:"width"`
	Height     uint64 `json:"height"`
	IsScalable bool   `json:"is_scalable"`
}

// HTMLInfo contains information extracted from HTML page
type HTMLInfo struct {
	url *url.URL
	// http.Client instance to use, if nil then will be used default client
	Client *http.Client `json:"-"`
	// If it's true then parser will fetch oembed data from oembed url if possible
	AllowOembedFetching bool `json:"-"`
	// If it's true parser will extract main page content from html
	AllowMainContentExtraction bool `json:"-"`
	// We'll forward it to Oembed' fetchOembed method
	AcceptLanguage string `json:"-"`

	Title         string       `json:"title"`
	Description   string       `json:"description"`
	AuthorName    string       `json:"author_name"`
	CanonicalURL  string       `json:"canonical_url"`
	OembedJSONURL string       `json:"oembed_json_url"`
	OembedXMLURL  string       `json:"oembed_xml_url"`
	FaviconURL    string       `json:"favicon_url"`
	TouchIcons    []*TouchIcon `json:"touch_icons"`
	ImageSrcURL   string       `json:"image_src_url"`
	// Readability package is being used inside
	MainContent string               `json:"main_content"`
	OGInfo      *opengraph.OpenGraph `json:"opengraph"`
	OembedInfo  *oembed.Info         `json:"oembed"`
}

var (
	cleanHTMLTagsRegex    = regexp.MustCompile(`<.*?>`)
	replaceNewLinesRegex  = regexp.MustCompile(`[\r\n]+`)
	clearWhitespacesRegex = regexp.MustCompile(`\s+`)
	getImageRegex         = regexp.MustCompile(`(?i)<img[^>]+?src=("|')?(.*?)("|'|\s|>)`)
	linkWithIconsRegex    = regexp.MustCompile(`\b(icon|image_src)\b`)
	sizesRegex            = regexp.MustCompile(`(\d+)[^\d]+(\d+)`) // some websites use crazy unicode chars between height and width
)

// NewHTMLInfo return new instance of HTMLInfo
func NewHTMLInfo() *HTMLInfo {
	info := &HTMLInfo{AllowOembedFetching: true, AllowMainContentExtraction: true, OGInfo: opengraph.NewOpenGraph(), AcceptLanguage: "en-us"}
	return info
}

func (info *HTMLInfo) toAbsoluteURL(u string) string {
	if info.url == nil {
		return u
	}

	tu, _ := url.Parse(u)

	if tu != nil {
		if tu.Host == "" {
			tu.Scheme = info.url.Scheme
			tu.Host = info.url.Host
			tu.User = info.url.User
			tu.Opaque = info.url.Opaque
			if len(tu.Path) == 0 || tu.Path[0] != '/' {
				tu.Path = info.url.Path + tu.Path
			}
		} else if tu.Scheme == "" {
			tu.Scheme = info.url.Scheme
		}

		return tu.String()
	}

	return u
}

func (info *HTMLInfo) appendTouchIcons(url string, rel string, sizes []string) {
	for _, size := range sizes {
		icon := &TouchIcon{URL: url, Type: rel, IsScalable: (size == "any")}
		matches := sizesRegex.FindStringSubmatch(size)
		if len(matches) >= 3 {
			icon.Height, _ = strconv.ParseUint(matches[1], 10, 64)
			icon.Width, _ = strconv.ParseUint(matches[2], 10, 64)
		}
		info.TouchIcons = append(info.TouchIcons, icon)
	}
}

func (info *HTMLInfo) parseLinkIcon(attrs map[string]string) {
	rels := strings.Split(attrs["rel"], " ")
	url := info.toAbsoluteURL(attrs["href"])
	sizesString, present := attrs["sizes"]
	if !present {
		sizesString = "0x0"
	}
	sizes := strings.Split(sizesString, " ")

	for _, rel := range rels {
		if rel == "image_src" {
			info.ImageSrcURL = url
		} else if rel == "icon" {
			info.FaviconURL = url
			info.appendTouchIcons(url, rel, sizes)
		} else if rel == "apple-touch-icon" || rel == "apple-touch-icon-precomposed" {
			info.appendTouchIcons(url, rel, sizes)
		}
	}
}

func (info *HTMLInfo) parseHead(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.ElementNode && c.Data == "title" {
			if c.FirstChild != nil {
				info.Title = c.FirstChild.Data
			}
		} else if c.Type == html.ElementNode && c.Data == "link" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}
			if m["rel"] == "canonical" {
				info.CanonicalURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/json+oembed" {
				info.OembedJSONURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/xml+oembed" {
				info.OembedXMLURL = info.toAbsoluteURL(m["href"])
			} else if linkWithIconsRegex.MatchString(m["rel"]) {
				info.parseLinkIcon(m)
			}
		} else if c.Type == html.ElementNode && c.Data == "meta" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}

			if m["name"] == "description" {
				info.Description = m["content"]
			} else if m["name"] == "author" {
				info.AuthorName = m["content"]
			}

			info.OGInfo.ProcessMeta(m)
		}
	}
}

func (info *HTMLInfo) parseBody(n *html.Node) {
	if !info.AllowMainContentExtraction {
		return
	}

	buf := new(bytes.Buffer)
	err := html.Render(buf, n)
	if err != nil {
		return
	}
	bufStr := buf.String()
	doc, err := readability.NewDocument(bufStr)
	if err != nil {
		return
	}

	doc.WhitelistTags = []string{"div", "p", "img"}
	doc.WhitelistAttrs["img"] = []string{"src", "title", "alt"}

	content := doc.Content()
	content = html.UnescapeString(content)

	info.MainContent = strings.Trim(content, "\r\n\t ")
}

// Parse return information about page
// @param s - contains page source
// @params pageURL - contains URL from where the data was taken [optional]
// @params contentType - contains Content-Type header value [optional]
// if no url is given then parser won't attempt to parse oembed info
func (info *HTMLInfo) Parse(s io.Reader, pageURL *string, contentType *string) error {
	contentTypeStr := "text/html"
	if contentType != nil && len(*contentType) > 0 {
		contentTypeStr = *contentType
	}
	utf8s, err := charset.NewReader(s, contentTypeStr)
	if err != nil {
		return err
	}

	if pageURL != nil {
		tu, _ := url.Parse(*pageURL)
		info.url = tu
	}

	doc, err := html.Parse(utf8s)
	if err != nil {
		return err
	}

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.Type == html.ElementNode {
				if c.Data == "head" {
					info.parseHead(c)
					continue
				} else if c.Data == "body" {
					info.parseBody(c)
					continue
				}
			}
			f(c)
		}
	}
	f(doc)

	if info.AllowOembedFetching && pageURL != nil && len(info.OembedJSONURL) > 0 {
		pu, _ := url.Parse(info.OembedJSONURL)
		siteName := info.OGInfo.SiteName
		siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

		if len(siteName) == 0 {
			siteName = pu.Host
		}

		oiItem := &oembed.Item{EndpointURL: info.OembedJSONURL, ProviderName: siteName, ProviderURL: siteURL, IsEndpointURLComplete: true}
		oi, _ := oiItem.FetchOembed(oembed.Options{URL: *pageURL, Client: info.Client, AcceptLanguage: info.AcceptLanguage})
		if oi != nil && oi.Status < 300 {
			info.OembedInfo = oi
		}
	}

	return nil
}

func (info *HTMLInfo) trimText(text string, maxLen int) string {
	var numRunes = 0
	for index := range text {
		numRunes++
		if numRunes > maxLen {
			return text[:index-3] + "..."
		}
	}
	return text
}

// GenerateOembedFor return Oembed Info for given url based on previously parsed data
// The returned oembed data is also updated in info.OembedInfo
// Example:
//
// info := NewHTMLInfo()
// info.Parse(dataReader, &sourceURL)
// oembed := info.GenerateOembedFor(sourceURL)
func (info *HTMLInfo) GenerateOembedFor(pageURL string) *oembed.Info {
	pu, _ := url.Parse(pageURL)

	if pu == nil {
		return nil
	}

	siteName := info.OGInfo.SiteName
	siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

	if len(siteName) == 0 {
		siteName = pu.Host
	}

	title := info.OGInfo.Title
	if len(title) == 0 {
		title = info.Title
	}

	description := info.OGInfo.Description
	if len(description) == 0 {
		description = info.Description
		if len(description) == 0 {
			if len(info.MainContent) > 0 {
				description = cleanHTMLTagsRegex.ReplaceAllString(info.MainContent, " ")
				description = replaceNewLinesRegex.ReplaceAllString(description, " ")
				description = clearWhitespacesRegex.ReplaceAllString(description, " ")
				description = strings.Trim(description, " ")
				description = info.trimText(description, 200)
			}
		}
	}

	baseInfo := &oembed.Info{}

	baseInfo.Type = "link"
	baseInfo.URL = pageURL
	baseInfo.ProviderURL = siteURL
	baseInfo.ProviderName = siteName
	baseInfo.Title = title
	baseInfo.Description = description

	if len(info.ImageSrcURL) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.ImageSrcURL)
	}

	if len(info.OGInfo.Images) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.OGInfo.Images[0].URL)
		baseInfo.ThumbnailWidth = info.OGInfo.Images[0].Width
		baseInfo.ThumbnailHeight = info.OGInfo.Images[0].Height
	}

	if len(baseInfo.ThumbnailURL) == 0 && len(info.MainContent) > 0 {
		// get first image from body
		matches := getImageRegex.FindStringSubmatch(info.MainContent)
		if len(matches) > 0 {
			baseInfo.ThumbnailURL = info.toAbsoluteURL(matches[2])
		}
	}

	// first we check if there is link to oembed resource
	if info.OembedInfo != nil {
		info.OembedInfo.MergeWith(baseInfo)
		return info.OembedInfo
	}

	return baseInfo
}

// ToJSON return json represenation of structure, simple wrapper around json package
func (info *HTMLInfo) ToJSON() ([]byte, error) {
	return json.Marshal(info)
}

func (info *HTMLInfo) String() string {
	data, err := info.ToJSON()
	if err != nil {
		return err.Error()
	}
	return string(data[:])
}
<commit_msg>Fix trimText()<commit_after>package htmlinfo

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/html/charset"

	"github.com/dyatlov/go-oembed/oembed"
	"github.com/dyatlov/go-opengraph/opengraph"
	"golang.org/x/net/html"

	"github.com/dyatlov/go-readability"
)

// TouchIcon contains all icons parsed from page header, including Apple touch icons
type TouchIcon struct {
	URL        string `json:"url"`
	Type       string `json:"type"`
	Width      uint64 `json:"width"`
	Height     uint64 `json:"height"`
	IsScalable bool   `json:"is_scalable"`
}

// HTMLInfo contains information extracted from HTML page
type HTMLInfo struct {
	url *url.URL
	// http.Client instance to use, if nil then will be used default client
	Client *http.Client `json:"-"`
	// If it's true then parser will fetch oembed data from oembed url if possible
	AllowOembedFetching bool `json:"-"`
	// If it's true parser will extract main page content from html
	AllowMainContentExtraction bool `json:"-"`
	// We'll forward it to Oembed' fetchOembed method
	AcceptLanguage string `json:"-"`

	Title         string       `json:"title"`
	Description   string       `json:"description"`
	AuthorName    string       `json:"author_name"`
	CanonicalURL  string       `json:"canonical_url"`
	OembedJSONURL string       `json:"oembed_json_url"`
	OembedXMLURL  string       `json:"oembed_xml_url"`
	FaviconURL    string       `json:"favicon_url"`
	TouchIcons    []*TouchIcon `json:"touch_icons"`
	ImageSrcURL   string       `json:"image_src_url"`
	// Readability package is being used inside
	MainContent string               `json:"main_content"`
	OGInfo      *opengraph.OpenGraph `json:"opengraph"`
	OembedInfo  *oembed.Info         `json:"oembed"`
}

var (
	cleanHTMLTagsRegex    = regexp.MustCompile(`<.*?>`)
	replaceNewLinesRegex  = regexp.MustCompile(`[\r\n]+`)
	clearWhitespacesRegex = regexp.MustCompile(`\s+`)
	getImageRegex         = regexp.MustCompile(`(?i)<img[^>]+?src=("|')?(.*?)("|'|\s|>)`)
	linkWithIconsRegex    = regexp.MustCompile(`\b(icon|image_src)\b`)
	sizesRegex            = regexp.MustCompile(`(\d+)[^\d]+(\d+)`) // some websites use crazy unicode chars between height and width
)

// NewHTMLInfo return new instance of HTMLInfo
func NewHTMLInfo() *HTMLInfo {
	info := &HTMLInfo{AllowOembedFetching: true, AllowMainContentExtraction: true, OGInfo: opengraph.NewOpenGraph(), AcceptLanguage: "en-us"}
	return info
}

func (info *HTMLInfo) toAbsoluteURL(u string) string {
	if info.url == nil {
		return u
	}

	tu, _ := url.Parse(u)

	if tu != nil {
		if tu.Host == "" {
			tu.Scheme = info.url.Scheme
			tu.Host = info.url.Host
			tu.User = info.url.User
			tu.Opaque = info.url.Opaque
			if len(tu.Path) == 0 || tu.Path[0] != '/' {
				tu.Path = info.url.Path + tu.Path
			}
		} else if tu.Scheme == "" {
			tu.Scheme = info.url.Scheme
		}

		return tu.String()
	}

	return u
}

func (info *HTMLInfo) appendTouchIcons(url string, rel string, sizes []string) {
	for _, size := range sizes {
		icon := &TouchIcon{URL: url, Type: rel, IsScalable: (size == "any")}
		matches := sizesRegex.FindStringSubmatch(size)
		if len(matches) >= 3 {
			icon.Height, _ = strconv.ParseUint(matches[1], 10, 64)
			icon.Width, _ = strconv.ParseUint(matches[2], 10, 64)
		}
		info.TouchIcons = append(info.TouchIcons, icon)
	}
}

func (info *HTMLInfo) parseLinkIcon(attrs map[string]string) {
	rels := strings.Split(attrs["rel"], " ")
	url := info.toAbsoluteURL(attrs["href"])
	sizesString, present := attrs["sizes"]
	if !present {
		sizesString = "0x0"
	}
	sizes := strings.Split(sizesString, " ")

	for _, rel := range rels {
		if rel == "image_src" {
			info.ImageSrcURL = url
		} else if rel == "icon" {
			info.FaviconURL = url
			info.appendTouchIcons(url, rel, sizes)
		} else if rel == "apple-touch-icon" || rel == "apple-touch-icon-precomposed" {
			info.appendTouchIcons(url, rel, sizes)
		}
	}
}

func (info *HTMLInfo) parseHead(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.ElementNode && c.Data == "title" {
			if c.FirstChild != nil {
				info.Title = c.FirstChild.Data
			}
		} else if c.Type == html.ElementNode && c.Data == "link" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}
			if m["rel"] == "canonical" {
				info.CanonicalURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/json+oembed" {
				info.OembedJSONURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/xml+oembed" {
				info.OembedXMLURL = info.toAbsoluteURL(m["href"])
			} else if linkWithIconsRegex.MatchString(m["rel"]) {
				info.parseLinkIcon(m)
			}
		} else if c.Type == html.ElementNode && c.Data == "meta" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}

			if m["name"] == "description" {
				info.Description = m["content"]
			} else if m["name"] == "author" {
				info.AuthorName = m["content"]
			}

			info.OGInfo.ProcessMeta(m)
		}
	}
}

func (info *HTMLInfo) parseBody(n *html.Node) {
	if !info.AllowMainContentExtraction {
		return
	}

	buf := new(bytes.Buffer)
	err := html.Render(buf, n)
	if err != nil {
		return
	}
	bufStr := buf.String()
	doc, err := readability.NewDocument(bufStr)
	if err != nil {
		return
	}

	doc.WhitelistTags = []string{"div", "p", "img"}
	doc.WhitelistAttrs["img"] = []string{"src", "title", "alt"}

	content := doc.Content()
	content = html.UnescapeString(content)

	info.MainContent = strings.Trim(content, "\r\n\t ")
}

// Parse return information about page
// @param s - contains page source
// @params pageURL - contains URL from where the data was taken [optional]
// @params contentType - contains Content-Type header value [optional]
// if no url is given then parser won't attempt to parse oembed info
func (info *HTMLInfo) Parse(s io.Reader, pageURL *string, contentType *string) error {
	contentTypeStr := "text/html"
	if contentType != nil && len(*contentType) > 0 {
		contentTypeStr = *contentType
	}
	utf8s, err := charset.NewReader(s, contentTypeStr)
	if err != nil {
		return err
	}

	if pageURL != nil {
		tu, _ := url.Parse(*pageURL)
		info.url = tu
	}

	doc, err := html.Parse(utf8s)
	if err != nil {
		return err
	}

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.Type == html.ElementNode {
				if c.Data == "head" {
					info.parseHead(c)
					continue
				} else if c.Data == "body" {
					info.parseBody(c)
					continue
				}
			}
			f(c)
		}
	}
	f(doc)

	if info.AllowOembedFetching && pageURL != nil && len(info.OembedJSONURL) > 0 {
		pu, _ := url.Parse(info.OembedJSONURL)
		siteName := info.OGInfo.SiteName
		siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

		if len(siteName) == 0 {
			siteName = pu.Host
		}

		oiItem := &oembed.Item{EndpointURL: info.OembedJSONURL, ProviderName: siteName, ProviderURL: siteURL, IsEndpointURLComplete: true}
		oi, _ := oiItem.FetchOembed(oembed.Options{URL: *pageURL, Client: info.Client, AcceptLanguage: info.AcceptLanguage})
		if oi != nil && oi.Status < 300 {
			info.OembedInfo = oi
		}
	}

	return nil
}

func (info *HTMLInfo) trimText(text string, maxLen int) string {
	var numRunes = 0
	runes := []rune(text)
	for index := range runes {
		numRunes++
		if numRunes > maxLen {
			return string(runes[:index-3]) + "..."
		}
	}
	return text
}

// GenerateOembedFor return Oembed Info for given url based on previously parsed data
// The returned oembed data is also updated in info.OembedInfo
// Example:
//
// info := NewHTMLInfo()
// info.Parse(dataReader, &sourceURL)
// oembed := info.GenerateOembedFor(sourceURL)
func (info *HTMLInfo) GenerateOembedFor(pageURL string) *oembed.Info {
	pu, _ := url.Parse(pageURL)

	if pu == nil {
		return nil
	}

	siteName := info.OGInfo.SiteName
	siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

	if len(siteName) == 0 {
		siteName = pu.Host
	}

	title := info.OGInfo.Title
	if len(title) == 0 {
		title = info.Title
	}

	description := info.OGInfo.Description
	if len(description) == 0 {
		description = info.Description
		if len(description) == 0 {
			if len(info.MainContent) > 0 {
				description = cleanHTMLTagsRegex.ReplaceAllString(info.MainContent, " ")
				description = replaceNewLinesRegex.ReplaceAllString(description, " ")
				description = clearWhitespacesRegex.ReplaceAllString(description, " ")
				description = strings.Trim(description, " ")
				description = info.trimText(description, 200)
			}
		}
	}

	baseInfo := &oembed.Info{}

	baseInfo.Type = "link"
	baseInfo.URL = pageURL
	baseInfo.ProviderURL = siteURL
	baseInfo.ProviderName = siteName
	baseInfo.Title = title
	baseInfo.Description = description

	if len(info.ImageSrcURL) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.ImageSrcURL)
	}

	if len(info.OGInfo.Images) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.OGInfo.Images[0].URL)
		baseInfo.ThumbnailWidth = info.OGInfo.Images[0].Width
		baseInfo.ThumbnailHeight = info.OGInfo.Images[0].Height
	}

	if len(baseInfo.ThumbnailURL) == 0 && len(info.MainContent) > 0 {
		// get first image from body
		matches := getImageRegex.FindStringSubmatch(info.MainContent)
		if len(matches) > 0 {
			baseInfo.ThumbnailURL = info.toAbsoluteURL(matches[2])
		}
	}

	// first we check if there is link to oembed resource
	if info.OembedInfo != nil {
		info.OembedInfo.MergeWith(baseInfo)
		return info.OembedInfo
	}

	return baseInfo
}

// ToJSON return json represenation of structure, simple wrapper around json package
func (info *HTMLInfo) ToJSON() ([]byte, error) {
	return json.Marshal(info)
}

func (info *HTMLInfo) String() string {
	data, err := info.ToJSON()
	if err != nil {
		return err.Error()
	}
	return string(data[:])
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"math"
	"net/http"
	"strconv"

	"github.com/sad0vnikov/radish/config"
	"github.com/sad0vnikov/radish/http/responds"
	"github.com/sad0vnikov/radish/http/server"
	"github.com/sad0vnikov/radish/logger"
	"github.com/sad0vnikov/radish/redis/db"
)

//GetServersList is a http handler returning a list of avalable Redis instances
func GetServersList(w http.ResponseWriter, r *http.Request) (interface{}, error) {
	return config.Get().Servers, nil
}

type getKeysByMaskResponse struct {
	Keys       []string
	Page       int
	PagesCount int
}

const defaultPageSize = 100

//GetKeysByMask is a http handler returning a JSON list of keys satisfying given mask
//for server with the name given in 'server' query param
func GetKeysByMask(w http.ResponseWriter, r *http.Request) (interface{}, error) {

	const pageSize = defaultPageSize

	requestParams := server.GetURLParams(r)

	serverName := requestParams["server"]
	if len(serverName) == 0 {
		return nil, responds.NewBadRequestError("'server' param is mandatory")
	}
	mask := r.URL.Query().Get("mask")
	if len(mask) == 0 {
		mask = "*"
	}

	pageNumber := 1
	page := r.URL.Query().Get("page")
	if len(page) > 0 {
		paramPage, err := strconv.ParseInt(page, 0, 8)
		if err == nil {
			pageNumber = int(paramPage)
		}

	}

	keys, err := db.FindKeysByMask(serverName, mask)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	pageOffsetEnd := pageNumber * pageSize
	if pageOffsetEnd > len(keys) {
		pageOffsetEnd = len(keys)
	}

	pageOffsetStart := (pageNumber - 1) * pageSize
	if pageOffsetStart > len(keys) {
		return nil, responds.NewNotFoundError("page not found")
	}

	keysPage := keys[pageOffsetStart:pageOffsetEnd]
	pagesCount := int(math.Ceil(float64(len(keys)) / float64(pageSize)))

	responseContents := getKeysByMaskResponse{Keys: keysPage, Page: pageNumber, PagesCount: pagesCount}

	return responseContents, nil
}

type keyInfoResponse struct {
	PageSize   int
	PagesCount int
	KeyType    string
}

//GetKeyInfo returns key type, values pages count and page size
func GetKeyInfo(w http.ResponseWriter, r *http.Request) (interface{}, error) {
	requestParams := server.GetURLParams(r)

	serverName := requestParams["server"]
	if len(serverName) == 0 {
		return nil, responds.NewBadRequestError("'server' param is mandatory")
	}

	keyName := requestParams["key"]
	if len(keyName) == 0 {
		return nil, responds.NewBadRequestError("'key' param is mandatory")
	}

	keyExists, err := db.KeyExists(serverName, keyName)
	if err != nil {
		return nil, err
	}
	if !keyExists {
		return nil, responds.NewNotFoundError(fmt.Sprintf("key %v doesn't exist", keyName))
	}

	key, err := db.GetKeyInfo(serverName, keyName)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	response := keyInfoResponse{}
	response.PageSize = defaultPageSize
	response.PagesCount, err = key.PagesCount(defaultPageSize)
	if err != nil {
		logger.Error(err)
		return nil, err
	}
	response.KeyType = key.KeyType()

	return response, nil
}

type valuesResponse struct {
	KeyType    string
	Values     []RedisValue
	PageNum    int
	PagesCount int
}

//RedisValue is a struct that could represent any Redis value
//Value field is always present
//Key fields is only present in a redis Hash values
//Score field is only present in redis ZSet values
type RedisValue struct {
	Value string
	Key   string
	Score int64
}

//GetKeyValues returns a list of key values
func GetKeyValues(w http.ResponseWriter, r *http.Request) (interface{}, error) {
	requestParams := server.GetURLParams(r)

	serverName := requestParams["server"]
	if len(serverName) == 0 {
		return nil, responds.NewBadRequestError("'server' param is required")
	}

	keyName := requestParams["key"]
	if len(keyName) == 0 {
		return nil, responds.NewBadRequestError("'key' param is required")
	}

	keyExists, err := db.KeyExists(serverName, keyName)
	if err != nil {
		return nil, err
	}
	if !keyExists {
		return nil, responds.NewNotFoundError(fmt.Sprintf("key %v doesn't exist", keyName))
	}

	pageParam := requestParams["page"]
	pageNum := 0
	if len(pageParam) != 0 {
		if parsedPageParam, err := strconv.ParseInt(pageParam, 0, 0); err == nil {
			pageNum = int(parsedPageParam)
		}
	}

	key, err := db.GetKeyInfo(serverName, keyName)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	v, err := key.Values(pageNum, defaultPageSize)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	response := valuesResponse{}
	response.PageNum = pageNum
	pagesCount, err := key.PagesCount(defaultPageSize)
	if err != nil {
		logger.Error(err)
		return nil, err
	}
	response.PagesCount = pagesCount
	switch key.KeyType() {
	case db.RedisString:
		response.KeyType = key.KeyType()
		if str, ok := v.(string); ok {
			value := RedisValue{Value: str}
			response.Values = []RedisValue{value}
		}
		return response, nil

	case db.RedisList:
		response.KeyType = key.KeyType()
		if strings, ok := v.([]string); ok {
			values := []RedisValue{}
			for _, str := range strings {
				values = append(values, RedisValue{Value: str})
			}
			response.Values = values
		}
		return response, nil

	case db.RedisZset:
		response.KeyType = key.KeyType()
		if v, ok := v.([]db.ZSetMember); ok {
			values := []RedisValue{}
			for _, setMember := range v {
				values = append(values, RedisValue{Value: setMember.Member, Score: setMember.Score})
			}
			response.Values = values
		}
		response.PageNum = pageNum
		return response, nil
	case db.RedisHash:
		response.KeyType = key.KeyType()
		values := []RedisValue{}
		if v, ok := v.(map[string]string); ok {
			for hashKey, hashValue := range v {
				values = append(values, RedisValue{Key: hashKey, Value: hashValue})
			}
		}
		response.Values = values
		return response, nil
	case db.RedisSet:
		response.KeyType = key.KeyType()
		values := []RedisValue{}
		if v, ok := v.([]string); ok {
			for _, setMember := range v {
				values = append(values, RedisValue{Value: setMember})
			}
		}
		response.Values = values
		return response, nil

	default:
		return nil, fmt.Errorf("%v key has not-supported type %v", keyName, key.KeyType())
	}

}
<commit_msg>* keys list JSON deunification<commit_after>package api

import (
	"fmt"
	"math"
	"net/http"
	"strconv"

	"github.com/sad0vnikov/radish/config"
	"github.com/sad0vnikov/radish/http/responds"
	"github.com/sad0vnikov/radish/http/server"
	"github.com/sad0vnikov/radish/logger"
	"github.com/sad0vnikov/radish/redis/db"
)

//GetServersList is a http handler returning a list of avalable Redis instances
func GetServersList(w http.ResponseWriter, r *http.Request) (interface{}, error) {
	return config.Get().Servers, nil
}

type getKeysByMaskResponse struct {
	Keys       []string
	Page       int
	PagesCount int
}

const defaultPageSize = 100

//GetKeysByMask is a http handler returning a JSON list of keys satisfying given mask
//for server with the name given in 'server' query param
func GetKeysByMask(w http.ResponseWriter, r *http.Request) (interface{}, error) {

	const pageSize = defaultPageSize

	requestParams := server.GetURLParams(r)

	serverName := requestParams["server"]
	if len(serverName) == 0 {
		return nil, responds.NewBadRequestError("'server' param is mandatory")
	}
	mask := r.URL.Query().Get("mask")
	if len(mask) == 0 {
		mask = "*"
	}

	pageNumber := 1
	page := r.URL.Query().Get("page")
	if len(page) > 0 {
		paramPage, err := strconv.ParseInt(page, 0, 8)
		if err == nil {
			pageNumber = int(paramPage)
		}

	}

	keys, err := db.FindKeysByMask(serverName, mask)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	pageOffsetEnd := pageNumber * pageSize
	if pageOffsetEnd > len(keys) {
		pageOffsetEnd = len(keys)
	}

	pageOffsetStart := (pageNumber - 1) * pageSize
	if pageOffsetStart > len(keys) {
		return nil, responds.NewNotFoundError("page not found")
	}

	keysPage := keys[pageOffsetStart:pageOffsetEnd]
	pagesCount := int(math.Ceil(float64(len(keys)) / float64(pageSize)))

	responseContents := getKeysByMaskResponse{Keys: keysPage, Page: pageNumber, PagesCount: pagesCount}

	return responseContents, nil
}

type keyInfoResponse struct {
	PageSize   int
	PagesCount int
	KeyType    string
}

//GetKeyInfo returns key type, values pages count and page size
func GetKeyInfo(w http.ResponseWriter, r *http.Request) (interface{}, error) {
	requestParams := server.GetURLParams(r)

	serverName := requestParams["server"]
	if len(serverName) == 0 {
		return nil, responds.NewBadRequestError("'server' param is mandatory")
	}

	keyName := requestParams["key"]
	if len(keyName) == 0 {
		return nil, responds.NewBadRequestError("'key' param is mandatory")
	}

	keyExists, err := db.KeyExists(serverName, keyName)
	if err != nil {
		return nil, err
	}
	if !keyExists {
		return nil, responds.NewNotFoundError(fmt.Sprintf("key %v doesn't exist", keyName))
	}

	key, err := db.GetKeyInfo(serverName, keyName)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	response := keyInfoResponse{}
	response.PageSize = defaultPageSize
	response.PagesCount, err = key.PagesCount(defaultPageSize)
	if err != nil {
		logger.Error(err)
		return nil, err
	}
	response.KeyType = key.KeyType()

	return response, nil
}

type singleValueResponse struct {
	KeyType string
	Value   string
}

type listValuesResponse struct {
	KeyType    string
	Values     []string
	PageNum    int
	PagesCount int
}

type hashValuesResponse struct {
	KeyType    string
	Values     map[string]string
	PageNum    int
	PagesCount int
}

type setValuesResponse struct {
	KeyType    string
	Values     []string
	PageNum    int
	PagesCount int
}

type zsetValuesResponse struct {
	KeyType    string
	Values     []db.ZSetMember
	PageNum    int
	PagesCount int
}

//GetKeyValues returns a list of key values
func GetKeyValues(w http.ResponseWriter, r *http.Request) (interface{}, error) {
	requestParams := server.GetURLParams(r)

	serverName := requestParams["server"]
	if len(serverName) == 0 {
		return nil, responds.NewBadRequestError("'server' param is required")
	}

	keyName := requestParams["key"]
	if len(keyName) == 0 {
		return nil, responds.NewBadRequestError("'key' param is required")
	}

	keyExists, err := db.KeyExists(serverName, keyName)
	if err != nil {
		return nil, err
	}
	if !keyExists {
		return nil, responds.NewNotFoundError(fmt.Sprintf("key %v doesn't exist", keyName))
	}

	pageParam := requestParams["page"]
	pageNum := 0
	if len(pageParam) != 0 {
		if parsedPageParam, err := strconv.ParseInt(pageParam, 0, 0); err == nil {
			pageNum = int(parsedPageParam)
		}
	}

	key, err := db.GetKeyInfo(serverName, keyName)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	v, err := key.Values(pageNum, defaultPageSize)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	pagesCount, err := key.PagesCount(defaultPageSize)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	switch key.KeyType() {
	case db.RedisString:
		response := singleValueResponse{}
		response.KeyType = key.KeyType()
		if str, ok := v.(string); ok {
			response.Value = str
		}
		return response, nil

	case db.RedisList:
		response := listValuesResponse{}
		response.KeyType = key.KeyType()
		if strings, ok := v.([]string); ok {
			response.Values = strings
		}
		response.PageNum = pageNum
		response.PagesCount = pagesCount
		return response, nil

	case db.RedisZset:
		response := zsetValuesResponse{}
		response.KeyType = key.KeyType()
		if v, ok := v.([]db.ZSetMember); ok {
			response.Values = v
		}
		response.PageNum = pageNum
		response.PagesCount = pagesCount
		return response, nil
	case db.RedisHash:
		response := hashValuesResponse{}
		response.KeyType = key.KeyType()
		if v, ok := v.(map[string]string); ok {
			response.Values = v
		}
		response.PageNum = pageNum
		response.PagesCount = pagesCount
		return response, nil
	case db.RedisSet:
		response := setValuesResponse{}
		response.KeyType = key.KeyType()
		if v, ok := v.([]string); ok {
			response.Values = v
		}
		response.PageNum = pageNum
		response.PagesCount = pagesCount
		return response, nil

	default:
		return nil, fmt.Errorf("%v key has not-supported type %v", keyName, key.KeyType())
	}

}
<|endoftext|>


<commit_before>/*
The MIT License (MIT)

Copyright (c) 2016 winlin

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

/*
 This the main entrance of https-proxy, proxy to api or other http server.
*/
package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"github.com/ossrs/go-oryx-lib/https"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"net/url"
	"net/http/httputil"
	"net"
)

const server = "Oryx/0.0.2"

func main() {
	fmt.Println(server, "HTTP/HTTPS static server with API proxy.")

	var err error
	var httpPort, httpsPort int
	var httpsDomains, html, cacheFile string
	var useLetsEncrypt bool
	var ssCert, ssKey string
	var oproxy string
	flag.IntVar(&httpPort, "http", 0, "http listen at. 0 to disable http.")
	flag.IntVar(&httpsPort, "https", 0, "https listen at. 0 to disable https. 443 to serve. ")
	flag.StringVar(&httpsDomains, "domains", "", "the allow domains, empty to allow all. for example: ossrs.net,www.ossrs.net")
	flag.StringVar(&html, "root", "./html", "the www web root. support relative dir to argv[0].")
	flag.StringVar(&cacheFile, "cache", "./letsencrypt.cache", "the cache for https. support relative dir to argv[0].")
	flag.BoolVar(&useLetsEncrypt, "lets", false, "whether use letsencrypt CA. self sign if not.")
	flag.StringVar(&ssKey, "ssk", "server.key", "https self-sign key by(before server.cert): openssl genrsa -out server.key 2048")
	flag.StringVar(&ssCert, "ssc", "server.crt", "https self-sign cert by: openssl req -new -x509 -key server.key -out server.crt -days 365")
	flag.StringVar(&oproxy, "proxy", "", "proxy the matched path to backend, for example, -proxy http://127.0.0.1:8888/api/webrtc")
	flag.Parse()

	if httpsPort != 0 && httpsPort != 443 {
		fmt.Println("https must be 0(disabled) or 443(enabled)")
		os.Exit(-1)
	}
	if httpPort == 0 && httpsPort == 0 {
		flag.PrintDefaults()
		os.Exit(-1)
	}

	var proxyUrl *url.URL
	var proxy *httputil.ReverseProxy
	if oproxy != "" {
		if proxyUrl,err = url.Parse(oproxy); err != nil {
			fmt.Println("proxy is not legal url, proxy is", oproxy)
			os.Exit(-1)
		}

		proxy = &httputil.ReverseProxy{
			Director: func(r *http.Request) {
				r.URL.Scheme = proxyUrl.Scheme
				r.URL.Host = proxyUrl.Host
				if ip, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
					r.Header.Set("X-Real-IP", ip)
				}
				//fmt.Println(fmt.Sprintf("proxy http %v to %v", r.RemoteAddr, r.URL.String()))
			},
		}
	}

	if !path.IsAbs(cacheFile) && path.IsAbs(os.Args[0]) {
		cacheFile = path.Join(path.Dir(os.Args[0]), cacheFile)
	}
	if !path.IsAbs(html) && path.IsAbs(os.Args[0]) {
		html = path.Join(path.Dir(os.Args[0]), html)
	}

	fs := http.FileServer(http.Dir(html))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Server", server)

		if o := r.Header.Get("Origin"); len(o) > 0 {
			w.Header().Set("Access-Control-Allow-Origin", "*")
			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, HEAD, PUT, DELETE, OPTIONS")
			w.Header().Set("Access-Control-Expose-Headers", "Server,range,Content-Length,Content-Range")
			w.Header().Set("Access-Control-Allow-Headers", "origin,range,accept-encoding,referer,Cache-Control,X-Proxy-Authorization,X-Requested-With,Content-Type")
		}

		if proxyUrl != nil && strings.HasPrefix(r.URL.Path, proxyUrl.Path) {
			// For matched OPTIONS, directly return without response.
			if r.Method == "OPTIONS" {
				return
			}

			proxy.ServeHTTP(w, r)
			return
		}

		fs.ServeHTTP(w, r)
	})

	var protos []string
	if httpPort != 0 {
		protos = append(protos, fmt.Sprintf("http(:%v)", httpPort))
	}
	if httpsPort != 0 {
		s := httpsDomains
		if httpsDomains == "" {
			s = "all domains"
		}
		protos = append(protos, fmt.Sprintf("https(:%v, %v, %v)", httpsPort, s, cacheFile))

		if useLetsEncrypt {
			protos = append(protos, "letsencrypt")
		} else {
			protos = append(protos, fmt.Sprintf("self-sign(%v, %v)", ssKey, ssCert))
		}
	}
	fmt.Println(fmt.Sprintf("%v html root at %v", strings.Join(protos, ", "), string(html)))

	wg := sync.WaitGroup{}
	go func() {
		defer wg.Done()

		if httpPort == 0 {
			fmt.Println("http server disabled")
			return
		}

		if err := http.ListenAndServe(fmt.Sprintf(":%v", httpPort), nil); err != nil {
			panic(err)
		}
		fmt.Println("http server ok.")
	}()
	wg.Add(1)

	go func() {
		defer wg.Done()

		if httpsPort == 0 {
			fmt.Println("https server disabled")
			return
		}

		var m https.Manager

		if useLetsEncrypt {
			var domains []string
			if httpsDomains != "" {
				domains = strings.Split(httpsDomains, ",")
			}

			if m, err = https.NewLetsencryptManager("", domains, cacheFile); err != nil {
				panic(err)
			}
		} else {
			if m, err = https.NewSelfSignManager(ssCert, ssKey); err != nil {
				panic(err)
			}
		}

		svr := &http.Server{
			Addr: fmt.Sprintf(":%v", httpsPort),
			TLSConfig: &tls.Config{
				GetCertificate: m.GetCertificate,
			},
		}

		if err := svr.ListenAndServeTLS("", ""); err != nil {
			panic(err)
		}
		fmt.Println("https server ok.")
	}()
	wg.Add(1)

	wg.Wait()
}
<commit_msg>Refine log<commit_after>/*
The MIT License (MIT)

Copyright (c) 2016 winlin

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

/*
 This the main entrance of https-proxy, proxy to api or other http server.
*/
package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"github.com/ossrs/go-oryx-lib/https"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"path"
	"strings"
	"sync"
)

const server = "Oryx/0.0.2"

func main() {
	fmt.Println(server, "HTTP/HTTPS static server with API proxy.")

	var err error
	var httpPort, httpsPort int
	var httpsDomains, html, cacheFile string
	var useLetsEncrypt bool
	var ssCert, ssKey string
	var oproxy string
	flag.IntVar(&httpPort, "http", 0, "http listen at. 0 to disable http.")
	flag.IntVar(&httpsPort, "https", 0, "https listen at. 0 to disable https. 443 to serve. ")
	flag.StringVar(&httpsDomains, "domains", "", "the allow domains, empty to allow all. for example: ossrs.net,www.ossrs.net")
	flag.StringVar(&html, "root", "./html", "the www web root. support relative dir to argv[0].")
	flag.StringVar(&cacheFile, "cache", "./letsencrypt.cache", "the cache for https. support relative dir to argv[0].")
	flag.BoolVar(&useLetsEncrypt, "lets", false, "whether use letsencrypt CA. self sign if not.")
	flag.StringVar(&ssKey, "ssk", "server.key", "https self-sign key by(before server.cert): openssl genrsa -out server.key 2048")
	flag.StringVar(&ssCert, "ssc", "server.crt", "https self-sign cert by: openssl req -new -x509 -key server.key -out server.crt -days 365")
	flag.StringVar(&oproxy, "proxy", "", "proxy the matched path to backend, for example, -proxy http://127.0.0.1:8888/api/webrtc")
	flag.Parse()

	if httpsPort != 0 && httpsPort != 443 {
		fmt.Println("https must be 0(disabled) or 443(enabled)")
		os.Exit(-1)
	}
	if httpPort == 0 && httpsPort == 0 {
		flag.PrintDefaults()
		os.Exit(-1)
	}

	var proxyUrl *url.URL
	var proxy *httputil.ReverseProxy
	if oproxy != "" {
		if proxyUrl, err = url.Parse(oproxy); err != nil {
			fmt.Println("proxy is not legal url, proxy is", oproxy)
			os.Exit(-1)
		}

		proxy = &httputil.ReverseProxy{
			Director: func(r *http.Request) {
				r.URL.Scheme = proxyUrl.Scheme
				r.URL.Host = proxyUrl.Host
				if ip, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
					r.Header.Set("X-Real-IP", ip)
				}
				//fmt.Println(fmt.Sprintf("proxy http %v to %v", r.RemoteAddr, r.URL.String()))
			},
		}
	}

	if !path.IsAbs(cacheFile) && path.IsAbs(os.Args[0]) {
		cacheFile = path.Join(path.Dir(os.Args[0]), cacheFile)
	}
	if !path.IsAbs(html) && path.IsAbs(os.Args[0]) {
		html = path.Join(path.Dir(os.Args[0]), html)
	}

	fs := http.FileServer(http.Dir(html))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Server", server)

		if o := r.Header.Get("Origin"); len(o) > 0 {
			w.Header().Set("Access-Control-Allow-Origin", "*")
			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, HEAD, PUT, DELETE, OPTIONS")
			w.Header().Set("Access-Control-Expose-Headers", "Server,range,Content-Length,Content-Range")
			w.Header().Set("Access-Control-Allow-Headers", "origin,range,accept-encoding,referer,Cache-Control,X-Proxy-Authorization,X-Requested-With,Content-Type")
		}

		if proxyUrl != nil && strings.HasPrefix(r.URL.Path, proxyUrl.Path) {
			// For matched OPTIONS, directly return without response.
			if r.Method == "OPTIONS" {
				return
			}

			proxy.ServeHTTP(w, r)
			return
		}

		fs.ServeHTTP(w, r)
	})

	var protos []string
	if httpPort != 0 {
		protos = append(protos, fmt.Sprintf("http(:%v)", httpPort))
	}
	if httpsPort != 0 {
		s := httpsDomains
		if httpsDomains == "" {
			s = "all domains"
		}

		if useLetsEncrypt {
			protos = append(protos, fmt.Sprintf("https(:%v, %v, %v)", httpsPort, s, cacheFile))
		} else {
			protos = append(protos, fmt.Sprintf("https(:%v)", httpsPort))
		}

		if useLetsEncrypt {
			protos = append(protos, "letsencrypt")
		} else {
			protos = append(protos, fmt.Sprintf("self-sign(%v, %v)", ssKey, ssCert))
		}
	}
	fmt.Println(fmt.Sprintf("%v html root at %v", strings.Join(protos, ", "), string(html)))

	wg := sync.WaitGroup{}
	go func() {
		defer wg.Done()

		if httpPort == 0 {
			fmt.Println("http server disabled")
			return
		}

		if err := http.ListenAndServe(fmt.Sprintf(":%v", httpPort), nil); err != nil {
			panic(err)
		}
		fmt.Println("http server ok.")
	}()
	wg.Add(1)

	go func() {
		defer wg.Done()

		if httpsPort == 0 {
			fmt.Println("https server disabled")
			return
		}

		var m https.Manager

		if useLetsEncrypt {
			var domains []string
			if httpsDomains != "" {
				domains = strings.Split(httpsDomains, ",")
			}

			if m, err = https.NewLetsencryptManager("", domains, cacheFile); err != nil {
				panic(err)
			}
		} else {
			if m, err = https.NewSelfSignManager(ssCert, ssKey); err != nil {
				panic(err)
			}
		}

		svr := &http.Server{
			Addr: fmt.Sprintf(":%v", httpsPort),
			TLSConfig: &tls.Config{
				GetCertificate: m.GetCertificate,
			},
		}

		if err := svr.ListenAndServeTLS("", ""); err != nil {
			panic(err)
		}
		fmt.Println("https server ok.")
	}()
	wg.Add(1)

	wg.Wait()
}
<|endoftext|>


<commit_before>package netlinkAudit

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"syscall"
	"unsafe"
)

type AuditStatus struct {
	Mask          uint32 /* Bit mask for valid entries */
	Enabled       uint32 /* 1 = enabled, 0 = disabled */
	Failure       uint32 /* Failure-to-log action */
	Pid           uint32 /* pid of auditd process */
	Rate_limit    uint32 /* messages rate limit (per second) */
	Backlog_limit uint32 /* waiting messages limit */
	Lost          uint32 /* messages lost */
	Backlog       uint32 /* messages waiting in queue */
}

type AuditRuleData struct {
	Flags       uint32 /* AUDIT_PER_{TASK,CALL}, AUDIT_PREPEND */
	Action      uint32 /* AUDIT_NEVER, AUDIT_POSSIBLE, AUDIT_ALWAYS */
	Field_count uint32
	Mask        [AUDIT_BITMASK_SIZE]uint32 /* syscall(s) affected */
	Fields      [AUDIT_MAX_FIELDS]uint32
	Values      [AUDIT_MAX_FIELDS]uint32
	Fieldflags  [AUDIT_MAX_FIELDS]uint32
	Buflen      uint32    /* total length of string fields */
	Buf         [0]string /* string fields buffer */

}
type NetlinkSocket struct {
	fd  int
	lsa syscall.SockaddrNetlink
}

type NetlinkAuditRequest struct {
	Header syscall.NlMsghdr
	Data   []byte
}

var ParsedResult AuditStatus

func nativeEndian() binary.ByteOrder {
	var x uint32 = 0x01020304
	if *(*byte)(unsafe.Pointer(&x)) == 0x01 {
		return binary.BigEndian
	}
	return binary.LittleEndian
}

//The recvfrom in go takes only a byte [] to put the data recieved from the kernel that removes the need
//for having a separate audit_reply Struct for recieving data from kernel.
func (rr *NetlinkAuditRequest) ToWireFormat() []byte {
	b := make([]byte, rr.Header.Len)
	*(*uint32)(unsafe.Pointer(&b[0:4][0])) = rr.Header.Len
	*(*uint16)(unsafe.Pointer(&b[4:6][0])) = rr.Header.Type
	*(*uint16)(unsafe.Pointer(&b[6:8][0])) = rr.Header.Flags
	*(*uint32)(unsafe.Pointer(&b[8:12][0])) = rr.Header.Seq
	*(*uint32)(unsafe.Pointer(&b[12:16][0])) = rr.Header.Pid
	b = append(b[:], rr.Data[:]...)
	return b
}

func newNetlinkAuditRequest(proto, seq, family, sizeofData int) *NetlinkAuditRequest {
	rr := &NetlinkAuditRequest{}

	rr.Header.Len = uint32(syscall.NLMSG_HDRLEN + sizeofData)
	rr.Header.Type = uint16(proto)
	rr.Header.Flags = syscall.NLM_F_REQUEST | syscall.NLM_F_ACK
	rr.Header.Seq = uint32(seq)
	return rr
	//	return rr.ToWireFormat()
}

// Round the length of a netlink message up to align it properly.
func nlmAlignOf(msglen int) int {
	return (msglen + syscall.NLMSG_ALIGNTO - 1) & ^(syscall.NLMSG_ALIGNTO - 1)
}

func ParseAuditNetlinkMessage(b []byte) ([]syscall.NetlinkMessage, error) {
	var msgs []syscall.NetlinkMessage
	for len(b) >= syscall.NLMSG_HDRLEN {
		h, dbuf, dlen, err := netlinkMessageHeaderAndData(b)
		if err != nil {
			fmt.Println("Error in parsing")
			return nil, err
		}
		m := syscall.NetlinkMessage{Header: *h, Data: dbuf[:int(h.Len)-syscall.NLMSG_HDRLEN]}
		msgs = append(msgs, m)
		b = b[dlen:]
	}
	return msgs, nil
}

func netlinkMessageHeaderAndData(b []byte) (*syscall.NlMsghdr, []byte, int, error) {

	h := (*syscall.NlMsghdr)(unsafe.Pointer(&b[0]))
	if int(h.Len) < syscall.NLMSG_HDRLEN || int(h.Len) > len(b) {
		fmt.Println("Error due to....HDRLEN:", syscall.NLMSG_HDRLEN, " Header Length:", h.Len, " Length of BYTE Array:", len(b))
		return nil, nil, 0, syscall.EINVAL
	}
	return h, b[syscall.NLMSG_HDRLEN:], nlmAlignOf(int(h.Len)), nil
}

// This function makes a conncetion with kernel space and is to be used for all further socket communication

func GetNetlinkSocket() (*NetlinkSocket, error) {
	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_AUDIT) //connect to the socket of type RAW
	if err != nil {
		return nil, err
	}
	s := &NetlinkSocket{
		fd: fd,
	}
	s.lsa.Family = syscall.AF_NETLINK
	s.lsa.Groups = 0
	s.lsa.Pid = 0 //Kernel space pid is always set to be 0

	if err := syscall.Bind(fd, &s.lsa); err != nil {
		syscall.Close(fd)
		return nil, err
	}
	return s, nil
}

//To end the socket conncetion
func (s *NetlinkSocket) Close() {
	syscall.Close(s.fd)
}

func (s *NetlinkSocket) Send(request *NetlinkAuditRequest) error {
	if err := syscall.Sendto(s.fd, request.ToWireFormat(), 0, &s.lsa); err != nil {
		return err
	}
	return nil
}

func (s *NetlinkSocket) Receive(bytesize int, block int) ([]syscall.NetlinkMessage, error) {
	rb := make([]byte, bytesize)
	nr, _, err := syscall.Recvfrom(s.fd, rb, 0|block)
	//nr, _, err := syscall.Recvfrom(s, rb, syscall.MSG_PEEK|syscall.MSG_DONTWAIT)
	if err != nil {
		return nil, err
	}
	if nr < syscall.NLMSG_HDRLEN {
		return nil, syscall.EINVAL
	}
	rb = rb[:nr]
	//var tab []byte
	//append(tab, rb...)
	return ParseAuditNetlinkMessage(rb) //Or syscall.ParseNetlinkMessage(rb)
}

//func audit_send(socket, proto, Data * struct, sizeof struct)
//func audit_get_reply(socket, proto, Data* struct , block int)
func AuditSend(s *NetlinkSocket, proto int, data []byte, sizedata, seq int) error {

	wb := newNetlinkAuditRequest(proto, seq, syscall.AF_NETLINK, sizedata) //Need to work on sequence
	wb.Data = append(wb.Data[:], data[:]...)
	if err := s.Send(wb); err != nil {
		return err
	}
	return nil
}

func AuditGetReply(s *NetlinkSocket, bytesize, block, seq int) error {
done:
	for {
		msgs, err := s.Receive(bytesize, block) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, err := syscall.Getsockname(s.fd)
			if err != nil {
				return err
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL

			}

			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done
			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR")
				break done
				//return nil
			}
			if m.Header.Type == AUDIT_GET {
				fmt.Println("AUDIT_GET")
				//				break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MS")
				//break done
			}
			if m.Header.Type == AUDIT_LIST_RULES {
				fmt.Println("AUDIT_LIST_RULES")
				//break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MSG")
				//break done
			}
			if m.Header.Type == 1009 {
				fmt.Println("Watchlist")
			}

		}
	}
	return nil

}

func AuditSetEnabled(s *NetlinkSocket, seq int) error {
	var status AuditStatus
	status.Enabled = 1
	status.Mask = AUDIT_STATUS_ENABLED
	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), status)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}

	err = AuditSend(s, AUDIT_SET, buff.Bytes(), int(unsafe.Sizeof(status)), seq)
	if err != nil {
		return err
	}
	// Receiving IN JUST ONE TRY
	err = AuditGetReply(s, syscall.Getpagesize(), 0, seq)
	if err != nil {
		return err
	}
	return nil
}

func AuditIsEnabled(s *NetlinkSocket, seq int) error {
	fmt.Println("Now Sending AUDIT_GET for Checking if Audit is enabled or not \n")
	wb := newNetlinkAuditRequest(AUDIT_GET, seq, syscall.AF_NETLINK, 0)

	if err := s.Send(wb); err != nil {
		return err
	}

done:
	for {
		//Make the rb byte bigger because of large messages from Kernel doesn't fit in 4096
		msgs, err := s.Receive(MAX_AUDIT_MESSAGE_LENGTH, syscall.MSG_DONTWAIT) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, er := syscall.Getsockname(s.fd)
			if er != nil {
				return nil
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL
			}
			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done

			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR\n\n")
			}
			if m.Header.Type == AUDIT_GET {
				//Conversion of the data part written to AuditStatus struct
				//Nil error : successfuly parsed
				b := m.Data[:]
				buf := bytes.NewBuffer(b)
				var dumm AuditStatus
				err = binary.Read(buf, nativeEndian(), &dumm)
				ParsedResult = dumm
				//fmt.Println("\nstruct :", dumm, err)
				//fmt.Println("\nStatus: ", dumm.Enabled)

				fmt.Println("ENABLED")
				break done
			}

		}

	}
	return nil

}

func AuditAddRuleData(s *NetlinkSocket, rule *AuditRuleData, flags int, action int) error {

	if flags == AUDIT_FILTER_ENTRY {
		fmt.Println("Use of entry filter is deprecated")
		return nil
	}

	rule.Flags = uint32(flags)
	rule.Action = uint32(action)

	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), rule)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}
	seq := 0
	err = AuditSend(s, AUDIT_ADD_RULE, buff.Bytes(), int(unsafe.Sizeof(rule))+int(rule.Buflen), seq)

	//rc := syscall.Sendto(fd, AUDIT_ADD_RULE, rule, unsafe.Sizeof(auditstruct) + rule.buflen)
	//rc := syscall.Sendto(fd, rule, AUDIT_ADD_RULE, syscall.Getsockname(fd))
	if err != nil {
		fmt.Println("Error sending add rule data request ()")
		return err
	}
	return err
}

/* How the file should look like
-- seprate constant, stuct to function
-- have a library function for different things like list all rules etc
-- have a main function like audit_send/get_reply
*/

/* Form of main function
package main

import (
	"fmt"
	"github.com/..../netlinkAudit"
)
func main() {
	s, err := netlinkAudit.GetNetlinkSocket()
	if err != nil {
		fmt.Println(err)
	}
	defer s.Close()

	netlinkAudit.AuditSetEnabled(s, 1)
	err = netlinkAudit.AuditIsEnabled(s, 2)
	fmt.Println("parsedResult")
	fmt.Println(netlinkAudit.ParsedResult)
	if err == nil {
		fmt.Println("Horrah")
	}

}

*/
<commit_msg>fixed seq<commit_after>package netlinkAudit

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"syscall"
	"unsafe"
)

type AuditStatus struct {
	Mask          uint32 /* Bit mask for valid entries */
	Enabled       uint32 /* 1 = enabled, 0 = disabled */
	Failure       uint32 /* Failure-to-log action */
	Pid           uint32 /* pid of auditd process */
	Rate_limit    uint32 /* messages rate limit (per second) */
	Backlog_limit uint32 /* waiting messages limit */
	Lost          uint32 /* messages lost */
	Backlog       uint32 /* messages waiting in queue */
}

type AuditRuleData struct {
	Flags       uint32 /* AUDIT_PER_{TASK,CALL}, AUDIT_PREPEND */
	Action      uint32 /* AUDIT_NEVER, AUDIT_POSSIBLE, AUDIT_ALWAYS */
	Field_count uint32
	Mask        [AUDIT_BITMASK_SIZE]uint32 /* syscall(s) affected */
	Fields      [AUDIT_MAX_FIELDS]uint32
	Values      [AUDIT_MAX_FIELDS]uint32
	Fieldflags  [AUDIT_MAX_FIELDS]uint32
	Buflen      uint32    /* total length of string fields */
	Buf         [0]string /* string fields buffer */

}
type NetlinkSocket struct {
	fd  int
	lsa syscall.SockaddrNetlink
}

type NetlinkAuditRequest struct {
	Header syscall.NlMsghdr
	Data   []byte
}

var ParsedResult AuditStatus

func nativeEndian() binary.ByteOrder {
	var x uint32 = 0x01020304
	if *(*byte)(unsafe.Pointer(&x)) == 0x01 {
		return binary.BigEndian
	}
	return binary.LittleEndian
}

//The recvfrom in go takes only a byte [] to put the data recieved from the kernel that removes the need
//for having a separate audit_reply Struct for recieving data from kernel.
func (rr *NetlinkAuditRequest) ToWireFormat() []byte {
	b := make([]byte, rr.Header.Len)
	*(*uint32)(unsafe.Pointer(&b[0:4][0])) = rr.Header.Len
	*(*uint16)(unsafe.Pointer(&b[4:6][0])) = rr.Header.Type
	*(*uint16)(unsafe.Pointer(&b[6:8][0])) = rr.Header.Flags
	*(*uint32)(unsafe.Pointer(&b[8:12][0])) = rr.Header.Seq
	*(*uint32)(unsafe.Pointer(&b[12:16][0])) = rr.Header.Pid
	b = append(b[:], rr.Data[:]...)
	return b
}

func newNetlinkAuditRequest(proto, seq, family, sizeofData int) *NetlinkAuditRequest {
	rr := &NetlinkAuditRequest{}

	rr.Header.Len = uint32(syscall.NLMSG_HDRLEN + sizeofData)
	rr.Header.Type = uint16(proto)
	rr.Header.Flags = syscall.NLM_F_REQUEST | syscall.NLM_F_ACK
	rr.Header.Seq = uint32(seq)
	return rr
	//	return rr.ToWireFormat()
}

// Round the length of a netlink message up to align it properly.
func nlmAlignOf(msglen int) int {
	return (msglen + syscall.NLMSG_ALIGNTO - 1) & ^(syscall.NLMSG_ALIGNTO - 1)
}

func ParseAuditNetlinkMessage(b []byte) ([]syscall.NetlinkMessage, error) {
	var msgs []syscall.NetlinkMessage
	for len(b) >= syscall.NLMSG_HDRLEN {
		h, dbuf, dlen, err := netlinkMessageHeaderAndData(b)
		if err != nil {
			fmt.Println("Error in parsing")
			return nil, err
		}
		m := syscall.NetlinkMessage{Header: *h, Data: dbuf[:int(h.Len)-syscall.NLMSG_HDRLEN]}
		msgs = append(msgs, m)
		b = b[dlen:]
	}
	return msgs, nil
}

func netlinkMessageHeaderAndData(b []byte) (*syscall.NlMsghdr, []byte, int, error) {

	h := (*syscall.NlMsghdr)(unsafe.Pointer(&b[0]))
	if int(h.Len) < syscall.NLMSG_HDRLEN || int(h.Len) > len(b) {
		fmt.Println("Error due to....HDRLEN:", syscall.NLMSG_HDRLEN, " Header Length:", h.Len, " Length of BYTE Array:", len(b))
		return nil, nil, 0, syscall.EINVAL
	}
	return h, b[syscall.NLMSG_HDRLEN:], nlmAlignOf(int(h.Len)), nil
}

// This function makes a conncetion with kernel space and is to be used for all further socket communication

func GetNetlinkSocket() (*NetlinkSocket, error) {
	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_AUDIT) //connect to the socket of type RAW
	if err != nil {
		return nil, err
	}
	s := &NetlinkSocket{
		fd: fd,
	}
	s.lsa.Family = syscall.AF_NETLINK
	s.lsa.Groups = 0
	s.lsa.Pid = 0 //Kernel space pid is always set to be 0

	if err := syscall.Bind(fd, &s.lsa); err != nil {
		syscall.Close(fd)
		return nil, err
	}
	return s, nil
}

//To end the socket conncetion
func (s *NetlinkSocket) Close() {
	syscall.Close(s.fd)
}

func (s *NetlinkSocket) Send(request *NetlinkAuditRequest) error {
	if err := syscall.Sendto(s.fd, request.ToWireFormat(), 0, &s.lsa); err != nil {
		return err
	}
	return nil
}

func (s *NetlinkSocket) Receive(bytesize int, block int) ([]syscall.NetlinkMessage, error) {
	rb := make([]byte, bytesize)
	nr, _, err := syscall.Recvfrom(s.fd, rb, 0|block)
	//nr, _, err := syscall.Recvfrom(s, rb, syscall.MSG_PEEK|syscall.MSG_DONTWAIT)
	if err != nil {
		return nil, err
	}
	if nr < syscall.NLMSG_HDRLEN {
		return nil, syscall.EINVAL
	}
	rb = rb[:nr]
	//var tab []byte
	//append(tab, rb...)
	return ParseAuditNetlinkMessage(rb) //Or syscall.ParseNetlinkMessage(rb)
}

//func audit_send(socket, proto, Data * struct, sizeof struct)
//func audit_get_reply(socket, proto, Data* struct , block int)
func AuditSend(s *NetlinkSocket, proto int, data []byte, sizedata, seq int) error {

	wb := newNetlinkAuditRequest(proto, seq, syscall.AF_NETLINK, sizedata) //Need to work on sequence
	wb.Data = append(wb.Data[:], data[:]...)
	if err := s.Send(wb); err != nil {
		return err
	}
	return nil
}

func AuditGetReply(s *NetlinkSocket, bytesize, block, seq int) error {
done:
	for {
		msgs, err := s.Receive(bytesize, block) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, err := syscall.Getsockname(s.fd)
			if err != nil {
				return err
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL

			}

			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done
			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR")
				break done
				//return nil
			}
			if m.Header.Type == AUDIT_GET {
				fmt.Println("AUDIT_GET")
				//				break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MS")
				//break done
			}
			if m.Header.Type == AUDIT_LIST_RULES {
				fmt.Println("AUDIT_LIST_RULES")
				//break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MSG")
				//break done
			}
			if m.Header.Type == 1009 {
				fmt.Println("Watchlist")
			}

		}
	}
	return nil

}

func AuditSetEnabled(s *NetlinkSocket, seq int) error {
	var status AuditStatus
	status.Enabled = 1
	status.Mask = AUDIT_STATUS_ENABLED
	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), status)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}

	err = AuditSend(s, AUDIT_SET, buff.Bytes(), int(unsafe.Sizeof(status)), seq)
	if err != nil {
		return err
	}
	// Receiving IN JUST ONE TRY
	err = AuditGetReply(s, syscall.Getpagesize(), 0, seq)
	if err != nil {
		return err
	}
	return nil
}

func AuditIsEnabled(s *NetlinkSocket, seq int) error {
	fmt.Println("Now Sending AUDIT_GET for Checking if Audit is enabled or not \n")
	wb := newNetlinkAuditRequest(AUDIT_GET, seq, syscall.AF_NETLINK, 0)

	if err := s.Send(wb); err != nil {
		return err
	}

done:
	for {
		//Make the rb byte bigger because of large messages from Kernel doesn't fit in 4096
		msgs, err := s.Receive(MAX_AUDIT_MESSAGE_LENGTH, syscall.MSG_DONTWAIT) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, er := syscall.Getsockname(s.fd)
			if er != nil {
				return nil
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL
			}
			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done

			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR\n\n")
			}
			if m.Header.Type == AUDIT_GET {
				//Conversion of the data part written to AuditStatus struct
				//Nil error : successfuly parsed
				b := m.Data[:]
				buf := bytes.NewBuffer(b)
				var dumm AuditStatus
				err = binary.Read(buf, nativeEndian(), &dumm)
				ParsedResult = dumm
				//fmt.Println("\nstruct :", dumm, err)
				//fmt.Println("\nStatus: ", dumm.Enabled)

				fmt.Println("ENABLED")
				break done
			}

		}

	}
	return nil

}

func AuditAddRuleData(s *NetlinkSocket, rule *AuditRuleData, flags int, action int) error {

	if flags == AUDIT_FILTER_ENTRY {
		fmt.Println("Use of entry filter is deprecated")
		return nil
	}

	rule.Flags = uint32(flags)
	rule.Action = uint32(action)

	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), rule)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}
	seq := 1 //Should be set accordingly
	err = AuditSend(s, AUDIT_ADD_RULE, buff.Bytes(), int(unsafe.Sizeof(rule))+int(rule.Buflen), seq)

	//rc := syscall.Sendto(fd, AUDIT_ADD_RULE, rule, unsafe.Sizeof(auditstruct) + rule.buflen)
	//rc := syscall.Sendto(fd, rule, AUDIT_ADD_RULE, syscall.Getsockname(fd))
	if err != nil {
		fmt.Println("Error sending add rule data request ()")
		return err
	}
	return err
}

/* How the file should look like
-- seprate constant, stuct to function
-- have a library function for different things like list all rules etc
-- have a main function like audit_send/get_reply
*/

/* Form of main function
package main

import (
	"fmt"
	"github.com/..../netlinkAudit"
)
func main() {
	s, err := netlinkAudit.GetNetlinkSocket()
	if err != nil {
		fmt.Println(err)
	}
	defer s.Close()

	netlinkAudit.AuditSetEnabled(s, 1)
	err = netlinkAudit.AuditIsEnabled(s, 2)
	fmt.Println("parsedResult")
	fmt.Println(netlinkAudit.ParsedResult)
	if err == nil {
		fmt.Println("Horrah")
	}

}

*/
<|endoftext|>


<commit_before>package gateway

import (
	"github.com/labstack/echo"
)

func apiRoute(c echo.Context) error {
	return  nil
}
<commit_msg>route 分发<commit_after>package gateway

import (
	"github.com/labstack/echo"
)
//统一入口 跟route分发
func apiRoute(c echo.Context) error {
	// 生成request_id
	// 获取debug选项
	// 查询是否存在此Api
	// 记录请求IP、参数
	// api不存在，返回错误
	// 记录请求的API信息
	// 生成url
	// 使用fasthttp    或者(http) (grpc) 用于通信
	// 设置Method
	// 透传参数
	// 透传cookie
	// 设置X-FORWARD-FOR 		记录真实IP
	// 请求
	// 拼接url
	return  nil
}
<|endoftext|>


<commit_before>package gcm

import (
	"github.com/alexjlockwood/gcm"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server"
	"github.com/smancke/guble/store"

	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"sync"
	//"runtime"
	"runtime"
)

// GCM_REGISTRATIONS_SCHEMA is the default sqlite schema for gcm
const GCM_REGISTRATIONS_SCHEMA = "gcm_registration"

// GCMConnector is the structure for handling the communication with Google Cloud Messaging
type GCMConnector struct {
	router             server.Router
	kvStore            store.KVStore
	prefix             string
	channelFromRouter  chan server.MsgAndRoute
	closeRouteByRouter chan server.Route
	stopChan           chan bool
	sender             *gcm.Sender
	workersNumber      int
	waitGroup          sync.WaitGroup
}

// NewGCMConnector creates a new gcmConnector without starting it
func NewGCMConnector(router server.Router, prefix string, gcmAPIKey string) (*GCMConnector, error) {

	kvStore, err := router.KVStore()
	if err != nil {
		return nil, err
	}

	//TODO Cosmin: check with dev-team the number of GCM workers, below
	gcm := &GCMConnector{
		router:            router,
		kvStore:           kvStore,
		prefix:            prefix,
		channelFromRouter: make(chan server.MsgAndRoute, 1000),
		stopChan:          make(chan bool, 1),
		sender:            &gcm.Sender{ApiKey: gcmAPIKey},
		workersNumber:     runtime.GOMAXPROCS(0),
	}

	return gcm, nil
}

// Start opens the connector, start more goroutines / workers to handle messages coming from the router
func (conn *GCMConnector) Start() error {
	broadcastRoute := server.NewRoute(removeTrailingSlash(conn.prefix)+"/broadcast", conn.channelFromRouter, "gcm_connector", "gcm_connector")
	conn.router.Subscribe(broadcastRoute)
	go func() {
		//TODO Cosmin: should loadSubscriptions() be taken out of this goroutine, and executed before ?
		// even if startup-time is longer, the routes are guaranteed to be there right after Start() returns
		conn.loadSubscriptions()

		protocol.Debug("number of GCM workers: %v", conn.workersNumber)
		for i := 1; i <= conn.workersNumber; i++ {
			protocol.Debug("starting GCM worker %v", i)
			go conn.loopSendOrBroadcastMessage()
		}
		conn.waitGroup.Add(conn.workersNumber)
	}()
	return nil
}

// Stop signals the closing of GCMConnector
func (conn *GCMConnector) Stop() error {
	protocol.Debug("GCM Stop()")
	close(conn.stopChan)
	conn.waitGroup.Wait()
	return nil
}

// Check returns nil if health-check succeeds, or an error if health-check fails
func (conn *GCMConnector) Check() error {
	return nil
}

// loopSendOrBroadcastMessage awaits in a loop for messages from router to be forwarded to GCM,
// until the stop-channel is closed
func (conn *GCMConnector) loopSendOrBroadcastMessage() {
	for {
		select {
		case msg := <-conn.channelFromRouter:
			if string(msg.Message.Path) == removeTrailingSlash(conn.prefix)+"/broadcast" {
				go conn.broadcastMessage(msg)
			} else {
				go conn.sendMessage(msg)
			}
		case _, opened := <-conn.stopChan:
			if !opened {
				protocol.Debug("GCM worker stopping")
				conn.waitGroup.Done()
				return
			}
		}
	}
}

func (conn *GCMConnector) sendMessage(msg server.MsgAndRoute) {
	gcmID := msg.Route.ApplicationID

	payload := conn.parseMessageToMap(msg.Message)

	var messageToGcm = gcm.NewMessage(payload, gcmID)
	protocol.Info("sending message to %v ...", gcmID)
	result, err := conn.sender.Send(messageToGcm, 5)
	if err != nil {
		protocol.Err("error sending message to GCM gcmID=%v: %v", gcmID, err.Error())
		return
	}

	errorJSON := result.Results[0].Error
	if errorJSON != "" {
		conn.handleJSONError(errorJSON, gcmID, msg.Route)
	} else {
		protocol.Debug("delivered message to GCM gcmID=%v: %v", gcmID, errorJSON)
	}

	// we only send to one receiver,
	// so we know that we can replace the old id with the first registration id (=canonical id)
	if result.CanonicalIDs != 0 {
		conn.replaceSubscriptionWithCanonicalID(msg.Route, result.Results[0].RegistrationID)
	}
}

func (conn *GCMConnector) parseMessageToMap(msg *protocol.Message) map[string]interface{} {
	payload := map[string]interface{}{}
	if msg.Body[0] == '{' {
		json.Unmarshal(msg.Body, &payload)
	} else {
		payload["message"] = msg.BodyAsString()
	}
	protocol.Debug("parsed message is: %v", payload)
	return payload
}

func (conn *GCMConnector) broadcastMessage(msg server.MsgAndRoute) {
	topic := msg.Message.Path
	payload := conn.parseMessageToMap(msg.Message)
	protocol.Info("broadcasting message with topic %v ...", string(topic))

	subscriptions := conn.kvStore.Iterate(GCM_REGISTRATIONS_SCHEMA, "")
	count := 0
	for {
		select {
		case entry, ok := <-subscriptions:
			if !ok {
				protocol.Info("send message to %v receivers", count)
				return
			}
			gmcID := entry[0]
			//TODO collect 1000 gcmIds and send them in one request!
			broadcastMessage := gcm.NewMessage(payload, gmcID)
			go func() {
				//TODO error handling of response!
				_, err := conn.sender.Send(broadcastMessage, 3)
				protocol.Debug("sent broadcast message to gcmID=%v", gmcID)
				if err != nil {
					protocol.Err("error sending broadcast message to gcmID=%v: %v", gmcID, err.Error())
				}
			}()
			count++
		}
	}
}

func (conn *GCMConnector) replaceSubscriptionWithCanonicalID(route *server.Route, newGcmID string) {
	oldGcmID := route.ApplicationID
	topic := string(route.Path)
	userID := route.UserID

	protocol.Info("replacing old gcmID %v with canonicalId %v", oldGcmID, newGcmID)

	conn.removeSubscription(route, oldGcmID)
	conn.subscribe(topic, userID, newGcmID)
}

func (conn *GCMConnector) handleJSONError(jsonError string, gcmID string, route *server.Route) {
	if jsonError == "NotRegistered" {
		protocol.Debug("remove not registered GCM registration gcmID=%v", gcmID)
		conn.removeSubscription(route, gcmID)
	} else if jsonError == "InvalidRegistration" {
		protocol.Err("the gcmID=%v is not registered. %v", gcmID, jsonError)
	} else {
		protocol.Err("unexpected error while sending to GCM gcmID=%v: %v", gcmID, jsonError)
	}
}

// GetPrefix is used to satisfy the HTTP handler interface
func (conn *GCMConnector) GetPrefix() string {
	return conn.prefix
}

func (conn *GCMConnector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		protocol.Err("Only HTTP POST METHOD SUPPORTED but received type=[%s]", r.Method)
		http.Error(w, "Permission Denied", http.StatusMethodNotAllowed)
		return
	}

	userID, gcmID, topic, err := conn.parseParams(r.URL.Path)
	if err != nil {
		http.Error(w, "Invalid Parameters in request", http.StatusBadRequest)
		return
	}
	conn.subscribe(topic, userID, gcmID)

	fmt.Fprintf(w, "registered: %v\n", topic)
}

// parseParams will parse the HTTP URL with format /gcm/:userid/:gcmid/subscribe/*topic
// returning the parsed Params, or error if the request is not in the correct format
func (conn *GCMConnector) parseParams(path string) (userID, gcmID, topic string, err error) {
	subscribePrefixPath := "subscribe"
	currentURLPath := removeTrailingSlash(path)

	if strings.HasPrefix(currentURLPath, conn.prefix) != true {
		err = errors.New("GCM request is not starting with gcm prefix")
		return
	}
	pathAfterPrefix := strings.TrimPrefix(currentURLPath, conn.prefix)

	splitParams := strings.SplitN(pathAfterPrefix, "/", 3)
	if len(splitParams) != 3 {
		err = errors.New("GCM request has wrong number of params")
		return
	}
	userID = splitParams[0]
	gcmID = splitParams[1]

	if strings.HasPrefix(splitParams[2], subscribePrefixPath+"/") != true {
		err = errors.New("GCM request third param is not subscribe")
		return
	}
	topic = strings.TrimPrefix(splitParams[2], subscribePrefixPath)
	return userID, gcmID, topic, nil
}

func (conn *GCMConnector) subscribe(topic string, userID string, gcmID string) {
	protocol.Info("GCM connector registration to userID=%q, gcmID=%q: %q", userID, gcmID, topic)

	route := server.NewRoute(topic, conn.channelFromRouter, gcmID, userID)

	conn.router.Subscribe(route)
	conn.saveSubscription(userID, topic, gcmID)
}

func (conn *GCMConnector) removeSubscription(route *server.Route, gcmID string) {
	conn.router.Unsubscribe(route)
	conn.kvStore.Delete(GCM_REGISTRATIONS_SCHEMA, gcmID)
}

func (conn *GCMConnector) saveSubscription(userID, topic, gcmID string) {
	conn.kvStore.Put(GCM_REGISTRATIONS_SCHEMA, gcmID, []byte(userID+":"+topic))
}

func (conn *GCMConnector) loadSubscriptions() {
	subscriptions := conn.kvStore.Iterate(GCM_REGISTRATIONS_SCHEMA, "")
	count := 0
	for {
		select {
		case entry, ok := <-subscriptions:
			if !ok {
				protocol.Info("renewed %v GCM subscriptions", count)
				return
			}
			gcmID := entry[0]
			splitValue := strings.SplitN(entry[1], ":", 2)
			userID := splitValue[0]
			topic := splitValue[1]

			protocol.Debug("renewing GCM subscription: userID=%v, topic=%v, gcmID=%v", userID, topic, gcmID)
			route := server.NewRoute(topic, conn.channelFromRouter, gcmID, userID)
			conn.router.Subscribe(route)
			count++
		}
	}
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<commit_msg>checking if router channel is still open inside loopSendOrBroadcastMessage(); defer Done; Add(1) inside the goroutine<commit_after>package gcm

import (
	"github.com/alexjlockwood/gcm"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server"
	"github.com/smancke/guble/store"

	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"sync"
	//"runtime"
	"runtime"
)

// GCM_REGISTRATIONS_SCHEMA is the default sqlite schema for gcm
const GCM_REGISTRATIONS_SCHEMA = "gcm_registration"

// GCMConnector is the structure for handling the communication with Google Cloud Messaging
type GCMConnector struct {
	router             server.Router
	kvStore            store.KVStore
	prefix             string
	channelFromRouter  chan server.MsgAndRoute
	closeRouteByRouter chan server.Route
	stopChan           chan bool
	sender             *gcm.Sender
	workersNumber      int
	waitGroup          sync.WaitGroup
}

// NewGCMConnector creates a new gcmConnector without starting it
func NewGCMConnector(router server.Router, prefix string, gcmAPIKey string) (*GCMConnector, error) {

	kvStore, err := router.KVStore()
	if err != nil {
		return nil, err
	}

	//TODO Cosmin: check with dev-team the number of GCM workers, below
	gcm := &GCMConnector{
		router:            router,
		kvStore:           kvStore,
		prefix:            prefix,
		channelFromRouter: make(chan server.MsgAndRoute, 1000),
		stopChan:          make(chan bool, 1),
		sender:            &gcm.Sender{ApiKey: gcmAPIKey},
		workersNumber:     runtime.GOMAXPROCS(0),
	}

	return gcm, nil
}

// Start opens the connector, start more goroutines / workers to handle messages coming from the router
func (conn *GCMConnector) Start() error {
	broadcastRoute := server.NewRoute(removeTrailingSlash(conn.prefix)+"/broadcast", conn.channelFromRouter, "gcm_connector", "gcm_connector")
	conn.router.Subscribe(broadcastRoute)
	go func() {
		//TODO Cosmin: should loadSubscriptions() be taken out of this goroutine, and executed before ?
		// even if startup-time is longer, the routes are guaranteed to be there right after Start() returns
		conn.loadSubscriptions()

		protocol.Debug("number of GCM workers: %v", conn.workersNumber)
		for i := 1; i <= conn.workersNumber; i++ {
			protocol.Debug("starting GCM worker %v", i)
			go conn.loopSendOrBroadcastMessage()
		}
	}()
	return nil
}

// Stop signals the closing of GCMConnector
func (conn *GCMConnector) Stop() error {
	protocol.Debug("GCM Stop()")
	close(conn.stopChan)
	conn.waitGroup.Wait()
	return nil
}

// Check returns nil if health-check succeeds, or an error if health-check fails
func (conn *GCMConnector) Check() error {
	return nil
}

// loopSendOrBroadcastMessage awaits in a loop for messages from router to be forwarded to GCM,
// until the stop-channel is closed
func (conn *GCMConnector) loopSendOrBroadcastMessage() {
	defer conn.waitGroup.Done()
	conn.waitGroup.Add(1)
	for {
		select {
		case msg, opened := <-conn.channelFromRouter:
			if opened {
				if string(msg.Message.Path) == removeTrailingSlash(conn.prefix) + "/broadcast" {
					go conn.broadcastMessage(msg)
				} else {
					go conn.sendMessage(msg)
				}
			}
		case <-conn.stopChan:
			protocol.Debug("stopping GCM worker")
			return
		}
	}
}

func (conn *GCMConnector) sendMessage(msg server.MsgAndRoute) {
	gcmID := msg.Route.ApplicationID

	payload := conn.parseMessageToMap(msg.Message)

	var messageToGcm = gcm.NewMessage(payload, gcmID)
	protocol.Info("sending message to %v ...", gcmID)
	result, err := conn.sender.Send(messageToGcm, 5)
	if err != nil {
		protocol.Err("error sending message to GCM gcmID=%v: %v", gcmID, err.Error())
		return
	}

	errorJSON := result.Results[0].Error
	if errorJSON != "" {
		conn.handleJSONError(errorJSON, gcmID, msg.Route)
	} else {
		protocol.Debug("delivered message to GCM gcmID=%v: %v", gcmID, errorJSON)
	}

	// we only send to one receiver,
	// so we know that we can replace the old id with the first registration id (=canonical id)
	if result.CanonicalIDs != 0 {
		conn.replaceSubscriptionWithCanonicalID(msg.Route, result.Results[0].RegistrationID)
	}
}

func (conn *GCMConnector) parseMessageToMap(msg *protocol.Message) map[string]interface{} {
	payload := map[string]interface{}{}
	if msg.Body[0] == '{' {
		json.Unmarshal(msg.Body, &payload)
	} else {
		payload["message"] = msg.BodyAsString()
	}
	protocol.Debug("parsed message is: %v", payload)
	return payload
}

func (conn *GCMConnector) broadcastMessage(msg server.MsgAndRoute) {
	topic := msg.Message.Path
	payload := conn.parseMessageToMap(msg.Message)
	protocol.Info("broadcasting message with topic %v ...", string(topic))

	subscriptions := conn.kvStore.Iterate(GCM_REGISTRATIONS_SCHEMA, "")
	count := 0
	for {
		select {
		case entry, ok := <-subscriptions:
			if !ok {
				protocol.Info("send message to %v receivers", count)
				return
			}
			gmcID := entry[0]
			//TODO collect 1000 gcmIds and send them in one request!
			broadcastMessage := gcm.NewMessage(payload, gmcID)
			go func() {
				//TODO error handling of response!
				_, err := conn.sender.Send(broadcastMessage, 3)
				protocol.Debug("sent broadcast message to gcmID=%v", gmcID)
				if err != nil {
					protocol.Err("error sending broadcast message to gcmID=%v: %v", gmcID, err.Error())
				}
			}()
			count++
		}
	}
}

func (conn *GCMConnector) replaceSubscriptionWithCanonicalID(route *server.Route, newGcmID string) {
	oldGcmID := route.ApplicationID
	topic := string(route.Path)
	userID := route.UserID

	protocol.Info("replacing old gcmID %v with canonicalId %v", oldGcmID, newGcmID)

	conn.removeSubscription(route, oldGcmID)
	conn.subscribe(topic, userID, newGcmID)
}

func (conn *GCMConnector) handleJSONError(jsonError string, gcmID string, route *server.Route) {
	if jsonError == "NotRegistered" {
		protocol.Debug("remove not registered GCM registration gcmID=%v", gcmID)
		conn.removeSubscription(route, gcmID)
	} else if jsonError == "InvalidRegistration" {
		protocol.Err("the gcmID=%v is not registered. %v", gcmID, jsonError)
	} else {
		protocol.Err("unexpected error while sending to GCM gcmID=%v: %v", gcmID, jsonError)
	}
}

// GetPrefix is used to satisfy the HTTP handler interface
func (conn *GCMConnector) GetPrefix() string {
	return conn.prefix
}

func (conn *GCMConnector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		protocol.Err("Only HTTP POST METHOD SUPPORTED but received type=[%s]", r.Method)
		http.Error(w, "Permission Denied", http.StatusMethodNotAllowed)
		return
	}

	userID, gcmID, topic, err := conn.parseParams(r.URL.Path)
	if err != nil {
		http.Error(w, "Invalid Parameters in request", http.StatusBadRequest)
		return
	}
	conn.subscribe(topic, userID, gcmID)

	fmt.Fprintf(w, "registered: %v\n", topic)
}

// parseParams will parse the HTTP URL with format /gcm/:userid/:gcmid/subscribe/*topic
// returning the parsed Params, or error if the request is not in the correct format
func (conn *GCMConnector) parseParams(path string) (userID, gcmID, topic string, err error) {
	subscribePrefixPath := "subscribe"
	currentURLPath := removeTrailingSlash(path)

	if strings.HasPrefix(currentURLPath, conn.prefix) != true {
		err = errors.New("GCM request is not starting with gcm prefix")
		return
	}
	pathAfterPrefix := strings.TrimPrefix(currentURLPath, conn.prefix)

	splitParams := strings.SplitN(pathAfterPrefix, "/", 3)
	if len(splitParams) != 3 {
		err = errors.New("GCM request has wrong number of params")
		return
	}
	userID = splitParams[0]
	gcmID = splitParams[1]

	if strings.HasPrefix(splitParams[2], subscribePrefixPath+"/") != true {
		err = errors.New("GCM request third param is not subscribe")
		return
	}
	topic = strings.TrimPrefix(splitParams[2], subscribePrefixPath)
	return userID, gcmID, topic, nil
}

func (conn *GCMConnector) subscribe(topic string, userID string, gcmID string) {
	protocol.Info("GCM connector registration to userID=%q, gcmID=%q: %q", userID, gcmID, topic)

	route := server.NewRoute(topic, conn.channelFromRouter, gcmID, userID)

	conn.router.Subscribe(route)
	conn.saveSubscription(userID, topic, gcmID)
}

func (conn *GCMConnector) removeSubscription(route *server.Route, gcmID string) {
	conn.router.Unsubscribe(route)
	conn.kvStore.Delete(GCM_REGISTRATIONS_SCHEMA, gcmID)
}

func (conn *GCMConnector) saveSubscription(userID, topic, gcmID string) {
	conn.kvStore.Put(GCM_REGISTRATIONS_SCHEMA, gcmID, []byte(userID+":"+topic))
}

func (conn *GCMConnector) loadSubscriptions() {
	subscriptions := conn.kvStore.Iterate(GCM_REGISTRATIONS_SCHEMA, "")
	count := 0
	for {
		select {
		case entry, ok := <-subscriptions:
			if !ok {
				protocol.Info("renewed %v GCM subscriptions", count)
				return
			}
			gcmID := entry[0]
			splitValue := strings.SplitN(entry[1], ":", 2)
			userID := splitValue[0]
			topic := splitValue[1]

			protocol.Debug("renewing GCM subscription: userID=%v, topic=%v, gcmID=%v", userID, topic, gcmID)
			route := server.NewRoute(topic, conn.channelFromRouter, gcmID, userID)
			conn.router.Subscribe(route)
			count++
		}
	}
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<|endoftext|>