<commit_before>package ablog

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/johnnylee/util"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// The default root prefix is "/".
var RootPrefix = "/"

func Main() {
	// Remove output dir.
	_ = os.RemoveAll("output")

	// Walk content directory, copying directories and static content.
	fmt.Println("Copying static content...")
	walk := func(path string, info os.FileInfo, err error) error {
		outPath := filepath.Join("output", path[7:])

		if info.IsDir() {
			fmt.Println("  Creating directory:", outPath)
			if err := os.MkdirAll(outPath, 0777); err != nil {
				exitErr(err, "Failed to create directory: "+outPath)
			}
			return nil
		}

		if path[len(path)-3:] != ".md" {
			fmt.Println("  Linking file:", outPath)
			if err := os.Link(path, outPath); err != nil {
				exitErr(err, "Failed to link file: "+path)
			}
		}

		return nil
	}

	if err := filepath.Walk("content", walk); err != nil {
		exitErr(err, "Failed to walk content directory.")
	}

	// Load templates.
	tmpl := template.Must(template.ParseGlob("template/*"))

	dir := NewDir(nil, "content", 0)
	dir.render(tmpl)
}

// ----------------------------------------------------------------------------
func exitErr(err error, msg string) {
	fmt.Println(msg)
	fmt.Println("Error:", err.Error())
	os.Exit(1)
}

func glob(pattern string) (matches []string) {
	paths, err := filepath.Glob(pattern)
	if err != nil {
		exitErr(err, "Failed to glob files: "+pattern)
	}

	sort.Sort(sort.Reverse(sort.StringSlice(paths)))
	return paths
}

// ----------------------------------------------------------------------------
type ADir struct {
	contentPath string // The path to directory under "content".
	outPath     string // The output path for the directory.

	RelPath string // The relative path in the output tree, beginning "/".

	Parent *ADir // Parent directory.
	Level  int   // The directory nesting level. 0 is root.

	Files []*AFile // Files in the directory.
	Dirs  []*ADir  // Sub-directories.

	// Previous and next directories.
	PrevDir *ADir
	NextDir *ADir

	// Sorted list of file tags in current directory, and recursively.
	FileTags          []string
	FileTagsRecursive []string
}

func NewDir(parent *ADir, dirPath string, level int) *ADir {
	fmt.Println("Processing directory:", dirPath)

	dir := ADir{}
	dir.contentPath = dirPath
	dir.outPath = filepath.Join("output", dirPath[7:])
	fmt.Println("  Output path:", dir.outPath)

	dir.Parent = parent
	dir.Level = level

	// Loading.
	dir.loadFiles()
	dir.loadDirs()
	dir.loadTags()

	return &dir
}

func (dir *ADir) loadFiles() {
	for _, path := range glob(filepath.Join(dir.contentPath, "*.md")) {
		fmt.Println("  Processing file:", path)
		dir.Files = append(dir.Files, NewAFile(dir, path))
	}

	// Set file prev/next pointers.
	for i, file := range dir.Files {
		if i != 0 {
			file.PrevFile = dir.Files[i-1]
		}
		if i < len(dir.Files)-1 {
			file.NextFile = dir.Files[i+1]
		}
	}
}

func (dir *ADir) loadDirs() {
	for _, path := range glob(filepath.Join(dir.contentPath, "*/")) {
		if !util.IsDir(path) {
			continue
		}
		dir.Dirs = append(dir.Dirs, NewDir(dir, path, dir.Level+1))
	}

	// Set dir prev/next pointers.
	for i, d := range dir.Dirs {
		if i != 0 {
			d.PrevDir = dir.Dirs[i-1]
		}
		if i < len(dir.Dirs)-1 {
			d.NextDir = dir.Dirs[i+1]
		}
	}
}

func (dir *ADir) loadTags() {
	fmt.Println("  Loading tags...")

	distinct := func(tagsList ...[]string) (out []string) {
		var m map[string]struct{}

		for _, tags := range tagsList {
			for _, tag := range tags {
				m[tag] = struct{}{}
			}
		}

		for k := range m {
			out = append(out, k)
		}
		sort.Strings(out)
		return
	}

	var tagsList [][]string
	for _, file := range dir.Files {
		tagsList = append(tagsList, file.Tags)
	}

	dir.FileTags = distinct(tagsList...)

	for _, subDir := range dir.Dirs {
		tagsList = append(tagsList, subDir.FileTagsRecursive)
	}

	dir.FileTagsRecursive = distinct(tagsList...)
}

// SubDir: Get a sub-directory by name.
func (dir *ADir) SubDir(name string) *ADir {
	for _, dir := range dir.Dirs {
		if filepath.Base(dir.outPath) == name {
			return dir
		}
	}
	return nil
}

// FilesRecursive: Return files in this and any sub directory.
func (dir *ADir) FilesRecursive() (files []*AFile) {
	files = append(files, dir.Files...)
	for _, subDir := range dir.Dirs {
		files = append(files, subDir.Files...)
	}
	return
}

// TaggedFilesAll: Return files in directory having all the given tags.
func (dir *ADir) TaggedFilesAll(tags ...string) (files []*AFile) {
	for _, file := range dir.Files {
		if file.HasTagsAll(tags...) {
			files = append(files, file)
		}
	}
	return
}

// TaggedFilesAllRecursive: Recursive version of TaggedFilesAll.
func (dir *ADir) TaggedFilesAllRecursive(tags ...string) (files []*AFile) {
	for _, file := range dir.FilesRecursive() {
		if file.HasTagsAll(tags...) {
			files = append(files, file)
		}
	}
	return
}

// TaggedFilesAny: Return files in directory having any the given tags.
func (dir *ADir) TaggedFilesAny(tags ...string) (files []*AFile) {
	for _, file := range dir.Files {
		if file.HasTagsAny(tags...) {
			files = append(files, file)
		}
	}
	return
}

// TaggedFilesAnyRecursive: Recursive version of TaggedFilesAny.
func (dir *ADir) TaggedFilesAnyRecursive(tags ...string) (files []*AFile) {
	for _, file := range dir.FilesRecursive() {
		if file.HasTagsAny(tags...) {
			files = append(files, file)
		}
	}
	return
}

func (dir *ADir) render(tmpl *template.Template) {
	fmt.Println("Rendering directory: " + dir.outPath)

	for _, file := range dir.Files {
		file.render(tmpl)
	}

	for _, subDir := range dir.Dirs {
		subDir.render(tmpl)
	}
}

// ----------------------------------------------------------------------------
type AFile struct {
	Parent *ADir // The parent directory.
	Level  int   // The parent directory's nesting level.

	mdPath  string // The path to the markdown content file.
	outPath string // The html output path.

	Url          string // The URL of the rendered HTML file.
	RootPrefix   string // The root path prefix.
	RootRelative string // Relative path to root dir.

	// Previous and next files in the directory.
	PrevFile *AFile
	NextFile *AFile

	// The content rendered from the markdownfile.
	Content template.HTML

	// Meta-data from the markdown file below.
	Template string   // The template to use to render this file.
	Tags     []string // Tags for your use.
	Title    string   // The title.
	Author   string   // The author.

	// Timestamps for creation / modification of the content.
	Created  struct{ Year, Month, Day int }
	Modified struct{ Year, Month, Day int }
}

func NewAFile(parent *ADir, mdPath string) *AFile {
	file := AFile{}
	file.Parent = parent
	file.Level = parent.Level

	file.mdPath = mdPath

	// Set the output path.
	file.outPath = filepath.Join(parent.outPath, filepath.Base(mdPath))
	file.outPath = file.outPath[:len(file.outPath)-2] + "html"

	file.Url = filepath.Join(RootPrefix, file.outPath[7:])
	file.RootPrefix = RootPrefix
	file.RootRelative = strings.Repeat("../", file.Level)

	// Load metadata and content from markdown file.
	data, err := ioutil.ReadFile(mdPath)
	if err != nil {
		exitErr(err, "When reading file: "+mdPath)
	}

	meta := bytes.SplitN(data, []byte("----"), 2)[0]
	if err = json.Unmarshal(meta, &file); err != nil {
		exitErr(err, "When reading metadata for file: "+mdPath)
	}

	input := bytes.SplitAfterN(data, []byte("----"), 2)[1]
	file.Content = template.HTML(markdown(input))

	return &file
}

func (file *AFile) UrlRelative(baseFile *AFile) string {
	path, err := filepath.Rel(baseFile.Parent.outPath, file.outPath)
	if err != nil {
		exitErr(err, "When computing relative path")
	}
	return path
}

// HasTag: Return true if the file has the given tag.
func (file *AFile) HasTag(tag string) bool {
	for _, t := range file.Tags {
		if tag == t {
			return true
		}
	}
	return false
}

// HasTagsAll: Return true if the file has all the given tags.
func (file *AFile) HasTagsAll(tags ...string) bool {
	for _, t := range tags {
		if !file.HasTag(t) {
			return false
		}
	}
	return true
}

// HasTagsAny: Return true if the file has any of the given tags.
func (file *AFile) HasTagsAny(tags ...string) bool {
	for _, t := range tags {
		if file.HasTag(t) {
			return true
		}
	}
	return false
}

// BaseName: Return the bare html filename.
func (file *AFile) BaseName() string {
	return filepath.Base(file.outPath)
}

// FirstParagraph: Return the first paragraph of the file. The returned HTML
// will contain the opening and closing
 tags.
func (file *AFile) FirstParagraph() template.HTML {
	return template.HTML(
		bytes.SplitAfterN(
			[]byte(file.Content), []byte("
"), 2)[0])
}

// FormatCreated: Format the creation date using Go's date formatting function.
// The reference time is "Mon Jan 2 15:04:05 MST 2006"
func (file *AFile) FormatCreated(fmt string) string {
	d := time.Date(
		file.Created.Year, time.Month(file.Created.Month), file.Created.Day,
		12, 0, 0, 0, time.UTC)
	return d.Format(fmt)
}

// FormatModified: The same as FormatCreated, but for the modification date.
func (file *AFile) FormatModified(fmt string) string {
	d := time.Date(
		file.Modified.Year, time.Month(file.Modified.Month), file.Modified.Day,
		12, 0, 0, 0, time.UTC)
	return d.Format(fmt)
}

func (file *AFile) render(tmpl *template.Template) {
	fmt.Println("Rendering file:", file.mdPath)
	fmt.Println("  Output path:", file.outPath)
	fmt.Println("  Template:   ", file.Template)
	fmt.Println("  URL:        ", file.Url)

	// Open output file for writing.
	f, err := os.Create(file.outPath)
	if err != nil {
		exitErr(err, "Failed to create output file: "+file.outPath)
	}
	defer f.Close()

	if err = tmpl.ExecuteTemplate(f, file.Template, file); err != nil {
		exitErr(err, "Failed to render template: "+file.Template)
	}
}
<commit_msg>Don't process files that begin with a ".".<commit_after>package ablog

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/johnnylee/util"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// The default root prefix is "/".
var RootPrefix = "/"

func Main() {
	// Remove output dir.
	_ = os.RemoveAll("output")

	// Walk content directory, copying directories and static content.
	fmt.Println("Copying static content...")
	walk := func(path string, info os.FileInfo, err error) error {
		outPath := filepath.Join("output", path[7:])

		if info.IsDir() {
			fmt.Println("  Creating directory:", outPath)
			if err := os.MkdirAll(outPath, 0777); err != nil {
				exitErr(err, "Failed to create directory: "+outPath)
			}
			return nil
		}

		if path[len(path)-3:] != ".md" {
			fmt.Println("  Linking file:", outPath)
			if err := os.Link(path, outPath); err != nil {
				exitErr(err, "Failed to link file: "+path)
			}
		}

		return nil
	}

	if err := filepath.Walk("content", walk); err != nil {
		exitErr(err, "Failed to walk content directory.")
	}

	// Load templates.
	tmpl := template.Must(template.ParseGlob("template/*"))

	dir := NewDir(nil, "content", 0)
	dir.render(tmpl)
}

// ----------------------------------------------------------------------------
func exitErr(err error, msg string) {
	fmt.Println(msg)
	fmt.Println("Error:", err.Error())
	os.Exit(1)
}

func glob(pattern string) (matches []string) {
	paths, err := filepath.Glob(pattern)
	if err != nil {
		exitErr(err, "Failed to glob files: "+pattern)
	}

	sort.Sort(sort.Reverse(sort.StringSlice(paths)))
	return paths
}

// ----------------------------------------------------------------------------
type ADir struct {
	contentPath string // The path to directory under "content".
	outPath     string // The output path for the directory.

	RelPath string // The relative path in the output tree, beginning "/".

	Parent *ADir // Parent directory.
	Level  int   // The directory nesting level. 0 is root.

	Files []*AFile // Files in the directory.
	Dirs  []*ADir  // Sub-directories.

	// Previous and next directories.
	PrevDir *ADir
	NextDir *ADir

	// Sorted list of file tags in current directory, and recursively.
	FileTags          []string
	FileTagsRecursive []string
}

func NewDir(parent *ADir, dirPath string, level int) *ADir {
	fmt.Println("Processing directory:", dirPath)

	dir := ADir{}
	dir.contentPath = dirPath
	dir.outPath = filepath.Join("output", dirPath[7:])
	fmt.Println("  Output path:", dir.outPath)

	dir.Parent = parent
	dir.Level = level

	// Loading.
	dir.loadFiles()
	dir.loadDirs()
	dir.loadTags()

	return &dir
}

func (dir *ADir) loadFiles() {
	for _, path := range glob(filepath.Join(dir.contentPath, "[^.]*.md")) {
		fmt.Println("  Processing file:", path)
		dir.Files = append(dir.Files, NewAFile(dir, path))
	}

	// Set file prev/next pointers.
	for i, file := range dir.Files {
		if i != 0 {
			file.PrevFile = dir.Files[i-1]
		}
		if i < len(dir.Files)-1 {
			file.NextFile = dir.Files[i+1]
		}
	}
}

func (dir *ADir) loadDirs() {
	for _, path := range glob(filepath.Join(dir.contentPath, "*/")) {
		if !util.IsDir(path) {
			continue
		}
		dir.Dirs = append(dir.Dirs, NewDir(dir, path, dir.Level+1))
	}

	// Set dir prev/next pointers.
	for i, d := range dir.Dirs {
		if i != 0 {
			d.PrevDir = dir.Dirs[i-1]
		}
		if i < len(dir.Dirs)-1 {
			d.NextDir = dir.Dirs[i+1]
		}
	}
}

func (dir *ADir) loadTags() {
	fmt.Println("  Loading tags...")

	distinct := func(tagsList ...[]string) (out []string) {
		m := make(map[string]struct{})

		for _, tags := range tagsList {
			for _, tag := range tags {
				m[tag] = struct{}{}
			}
		}

		for k := range m {
			out = append(out, k)
		}
		sort.Strings(out)
		return
	}

	var tagsList [][]string
	for _, file := range dir.Files {
		tagsList = append(tagsList, file.Tags)
	}

	dir.FileTags = distinct(tagsList...)

	for _, subDir := range dir.Dirs {
		tagsList = append(tagsList, subDir.FileTagsRecursive)
	}

	dir.FileTagsRecursive = distinct(tagsList...)
}

// SubDir: Get a sub-directory by name.
func (dir *ADir) SubDir(name string) *ADir {
	for _, dir := range dir.Dirs {
		if filepath.Base(dir.outPath) == name {
			return dir
		}
	}
	return nil
}

// FilesRecursive: Return files in this and any sub directory.
func (dir *ADir) FilesRecursive() (files []*AFile) {
	files = append(files, dir.Files...)
	for _, subDir := range dir.Dirs {
		files = append(files, subDir.Files...)
	}
	return
}

// TaggedFilesAll: Return files in directory having all the given tags.
func (dir *ADir) TaggedFilesAll(tags ...string) (files []*AFile) {
	for _, file := range dir.Files {
		if file.HasTagsAll(tags...) {
			files = append(files, file)
		}
	}
	return
}

// TaggedFilesAllRecursive: Recursive version of TaggedFilesAll.
func (dir *ADir) TaggedFilesAllRecursive(tags ...string) (files []*AFile) {
	for _, file := range dir.FilesRecursive() {
		if file.HasTagsAll(tags...) {
			files = append(files, file)
		}
	}
	return
}

// TaggedFilesAny: Return files in directory having any the given tags.
func (dir *ADir) TaggedFilesAny(tags ...string) (files []*AFile) {
	for _, file := range dir.Files {
		if file.HasTagsAny(tags...) {
			files = append(files, file)
		}
	}
	return
}

// TaggedFilesAnyRecursive: Recursive version of TaggedFilesAny.
func (dir *ADir) TaggedFilesAnyRecursive(tags ...string) (files []*AFile) {
	for _, file := range dir.FilesRecursive() {
		if file.HasTagsAny(tags...) {
			files = append(files, file)
		}
	}
	return
}

func (dir *ADir) render(tmpl *template.Template) {
	fmt.Println("Rendering directory: " + dir.outPath)

	for _, file := range dir.Files {
		file.render(tmpl)
	}

	for _, subDir := range dir.Dirs {
		subDir.render(tmpl)
	}
}

// ----------------------------------------------------------------------------
type AFile struct {
	Parent *ADir // The parent directory.
	Level  int   // The parent directory's nesting level.

	mdPath  string // The path to the markdown content file.
	outPath string // The html output path.

	Url          string // The URL of the rendered HTML file.
	RootPrefix   string // The root path prefix.
	RootRelative string // Relative path to root dir.

	// Previous and next files in the directory.
	PrevFile *AFile
	NextFile *AFile

	// The content rendered from the markdown file.
	Content template.HTML

	// Meta-data from the markdown file below.
	Template string   // The template to use to render this file.
	Tags     []string // Tags for your use.
	Title    string   // The title.
	Author   string   // The author.

	// Timestamps for creation / modification of the content.
	Created  struct{ Year, Month, Day int }
	Modified struct{ Year, Month, Day int }
}

func NewAFile(parent *ADir, mdPath string) *AFile {
	file := AFile{}
	file.Parent = parent
	file.Level = parent.Level

	file.mdPath = mdPath

	// Set the output path.
	file.outPath = filepath.Join(parent.outPath, filepath.Base(mdPath))
	file.outPath = file.outPath[:len(file.outPath)-2] + "html"

	file.Url = filepath.Join(RootPrefix, file.outPath[7:])
	file.RootPrefix = RootPrefix
	file.RootRelative = strings.Repeat("../", file.Level)

	// Load metadata and content from markdown file.
	data, err := ioutil.ReadFile(mdPath)
	if err != nil {
		exitErr(err, "When reading file: "+mdPath)
	}

	meta := bytes.SplitN(data, []byte("----"), 2)[0]
	if err = json.Unmarshal(meta, &file); err != nil {
		exitErr(err, "When reading metadata for file: "+mdPath)
	}

	input := bytes.SplitAfterN(data, []byte("----"), 2)[1]
	file.Content = template.HTML(markdown(input))

	return &file
}

// UrlRelative: Return the relative path to file from baseFile.
func (file *AFile) UrlRelative(baseFile *AFile) string {
	path, err := filepath.Rel(baseFile.Parent.outPath, file.outPath)
	if err != nil {
		exitErr(err, "When computing relative path")
	}
	return path
}

// HasTag: Return true if the file has the given tag.
func (file *AFile) HasTag(tag string) bool {
	for _, t := range file.Tags {
		if tag == t {
			return true
		}
	}
	return false
}

// HasTagsAll: Return true if the file has all the given tags.
func (file *AFile) HasTagsAll(tags ...string) bool {
	for _, t := range tags {
		if !file.HasTag(t) {
			return false
		}
	}
	return true
}

// HasTagsAny: Return true if the file has any of the given tags.
func (file *AFile) HasTagsAny(tags ...string) bool {
	for _, t := range tags {
		if file.HasTag(t) {
			return true
		}
	}
	return false
}

// BaseName: Return the bare html filename.
func (file *AFile) BaseName() string {
	return filepath.Base(file.outPath)
}

// FirstParagraph: Return the first paragraph of the file. The returned HTML
// will contain the opening and closing
 tags.
func (file *AFile) FirstParagraph() template.HTML {
	return template.HTML(
		bytes.SplitAfterN(
			[]byte(file.Content), []byte("
"), 2)[0])
}

// FormatCreated: Format the creation date using Go's date formatting function.
// The reference time is "Mon Jan 2 15:04:05 MST 2006"
func (file *AFile) FormatCreated(fmt string) string {
	d := time.Date(
		file.Created.Year, time.Month(file.Created.Month), file.Created.Day,
		12, 0, 0, 0, time.UTC)
	return d.Format(fmt)
}

// FormatModified: The same as FormatCreated, but for the modification date.
func (file *AFile) FormatModified(fmt string) string {
	d := time.Date(
		file.Modified.Year, time.Month(file.Modified.Month), file.Modified.Day,
		12, 0, 0, 0, time.UTC)
	return d.Format(fmt)
}

func (file *AFile) render(tmpl *template.Template) {
	fmt.Println("Rendering file:", file.mdPath)
	fmt.Println("  Output path:", file.outPath)
	fmt.Println("  Template:   ", file.Template)
	fmt.Println("  URL:        ", file.Url)

	// Open output file for writing.
	f, err := os.Create(file.outPath)
	if err != nil {
		exitErr(err, "Failed to create output file: "+file.outPath)
	}
	defer f.Close()

	if err = tmpl.ExecuteTemplate(f, file.Template, file); err != nil {
		exitErr(err, "Failed to render template: "+file.Template)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"os"
	"strings"
)

var body *goquery.Selection

func main() {
	var doc *goquery.Document

	loc := os.Args[1]
	if strings.HasPrefix(loc, "http") {
		doc, _ = goquery.NewDocument(loc)
	} else {
		b, _ := ioutil.ReadFile(loc)
		doc, _ = goquery.NewDocumentFromReader(bytes.NewReader(b))
	}

	// find article body
	body = doc.Find("body")
	if len(os.Args) > 2 {
		body = doc.Find(os.Args[2])
	}

	var good []string
	var bad []string
	var c int

	c = countHeadings()
	if c == 0 {
		bad = append(bad, "The text does not contain any subheadings. Add at least one subheading.")
	} else {
		good = append(good, fmt.Sprintf("The text contains %d subheadings.", c))
	}

	if countWords(body) <= 300 {
		bad = append(bad, "You have far too little content, please add some content to enable a good analysis.")
	} else {
		good = append(good, "The text contains more than 300 words.")
	}

	c = countParagraphsWithWords(150)
	if c >= 1 {
		bad = append(bad, fmt.Sprintf("%d of the paragraphs contains more than the recommended maximum of 150 words.", c))
	} else {
		good = append(good, "None of the paragraphs contain too many words.")
	}

	c = countHeadingFollowedByWords(300)
	if c > 1 {
		bad = append(bad, fmt.Sprintf("%d of the subheadings is followed by more than the recommended maximum of 300 words.", c))
	} else {
		good = append(good, "No subheadings are followed by more than 300 words.")
	}

	fmt.Printf("Analysing %s\n", loc)
	fmt.Printf("\n# The good\n")
	for _, l := range good {
		fmt.Printf("- %s\n", l)
	}
	if len(good) == 0 {
		fmt.Print("...\n")
	}

	fmt.Printf("\n# The bad\n")
	for _, l := range bad {
		fmt.Printf("- %s\n", l)
	}
	if len(bad) == 0 {
		fmt.Print("...\n")
	}

}

func countHeadingFollowedByWords(l int) int {
	count := 0
	body.Find("h2, h3, h4, h5").Each(func(i int, s *goquery.Selection) {
		sub := s.NextUntil("h2, h3, h4, h5")
		wordCount := countWords(sub)
		if wordCount > l {
			count++
		}
	})

	return count
}

// count subheadings, > 1
func countHeadings() int {
	return body.Find("h2, h3, h4, h5").Length()
}

// count words, > 300
func countWords(s *goquery.Selection) int {
	return len(strings.Split(s.Text(), " "))
}

// check paragraph length
func countParagraphsWithWords(l int) int {
	count := 0

	body.Find("p").Each(func(i int, s *goquery.Selection) {
		wordCount := countWords(s)
		if wordCount > l {
			count++
		}
	})

	return count
}

//  maximum 25% of sentences with over 20 words

// passive voice

// transition words

// flesch reading ease test
<commit_msg>remove stray line<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"os"
	"strings"
)

var body *goquery.Selection

func main() {
	var doc *goquery.Document

	loc := os.Args[1]
	if strings.HasPrefix(loc, "http") {
		doc, _ = goquery.NewDocument(loc)
	} else {
		b, _ := ioutil.ReadFile(loc)
		doc, _ = goquery.NewDocumentFromReader(bytes.NewReader(b))
	}

	// find article body
	body = doc.Find("body")
	if len(os.Args) > 2 {
		body = doc.Find(os.Args[2])
	}

	var good []string
	var bad []string
	var c int

	c = countHeadings()
	if c == 0 {
		bad = append(bad, "The text does not contain any subheadings. Add at least one subheading.")
	} else {
		good = append(good, fmt.Sprintf("The text contains %d subheadings.", c))
	}

	if countWords(body) <= 300 {
		bad = append(bad, "You have far too little content, please add some content to enable a good analysis.")
	} else {
		good = append(good, "The text contains more than 300 words.")
	}

	c = countParagraphsWithWords(150)
	if c >= 1 {
		bad = append(bad, fmt.Sprintf("%d of the paragraphs contains more than the recommended maximum of 150 words.", c))
	} else {
		good = append(good, "None of the paragraphs contain too many words.")
	}

	c = countHeadingFollowedByWords(300)
	if c > 1 {
		bad = append(bad, fmt.Sprintf("%d of the subheadings is followed by more than the recommended maximum of 300 words.", c))
	} else {
		good = append(good, "No subheadings are followed by more than 300 words.")
	}

	fmt.Printf("Analysing %s\n", loc)
	fmt.Printf("\n# The good\n")
	for _, l := range good {
		fmt.Printf("- %s\n", l)
	}
	if len(good) == 0 {
		fmt.Print("...\n")
	}

	fmt.Printf("\n# The bad\n")
	for _, l := range bad {
		fmt.Printf("- %s\n", l)
	}
	if len(bad) == 0 {
		fmt.Print("...\n")
	}
}

func countHeadingFollowedByWords(l int) int {
	count := 0
	body.Find("h2, h3, h4, h5").Each(func(i int, s *goquery.Selection) {
		sub := s.NextUntil("h2, h3, h4, h5")
		wordCount := countWords(sub)
		if wordCount > l {
			count++
		}
	})

	return count
}

// count subheadings, > 1
func countHeadings() int {
	return body.Find("h2, h3, h4, h5").Length()
}

// count words, > 300
func countWords(s *goquery.Selection) int {
	return len(strings.Split(s.Text(), " "))
}

// check paragraph length
func countParagraphsWithWords(l int) int {
	count := 0

	body.Find("p").Each(func(i int, s *goquery.Selection) {
		wordCount := countWords(s)
		if wordCount > l {
			count++
		}
	})

	return count
}

//  maximum 25% of sentences with over 20 words

// passive voice

// transition words

// flesch reading ease test
<|endoftext|>


<commit_before>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/nuclio/nuclio/pkg/dashboard"
	"github.com/nuclio/nuclio/pkg/platform"
	"github.com/nuclio/nuclio/pkg/restful"
)

type invocationResource struct {
	*resource
	nodeAddresses []string
}

// called after initialization
func (tr *invocationResource) OnAfterInitialize() error {

	// all methods
	for _, registrar := range []func(string, http.HandlerFunc){
		tr.GetRouter().Get,
		tr.GetRouter().Post,
		tr.GetRouter().Put,
		tr.GetRouter().Delete,
		tr.GetRouter().Patch,
		tr.GetRouter().Options,
	} {
		registrar("/*", tr.handleRequest)
	}

	return nil
}

func (tr *invocationResource) handleRequest(responseWriter http.ResponseWriter, request *http.Request) {
	path := request.Header.Get("x-nuclio-path")
	functionName := request.Header.Get("x-nuclio-function-name")
	invokeVia := tr.getInvokeVia(request.Header.Get("x-nuclio-invoke-via"))

	// get namespace from request or use the provided default
	functionNamespace := tr.getNamespaceOrDefault(request.Header.Get("x-nuclio-function-namespace"))

	// if user prefixed path with "/", remove it
	path = strings.TrimLeft(path, "/")

	if functionName == "" || functionNamespace == "" {
		responseWriter.WriteHeader(http.StatusBadRequest)
		responseWriter.Write([]byte(`{"error": "Function name must be provided"}`)) // nolint: errcheck
		return
	}

	requestBody, err := ioutil.ReadAll(request.Body)
	if err != nil {
		responseWriter.WriteHeader(http.StatusInternalServerError)
		responseWriter.Write([]byte(`{"error": "Failed to read request body"}`)) // nolint: errcheck
		return
	}

	// resolve the function host
	invocationResult, err := tr.getPlatform().CreateFunctionInvocation(&platform.CreateFunctionInvocationOptions{
		Name:      functionName,
		Namespace: functionNamespace,
		Path:      path,
		Method:    request.Method,
		Headers:   request.Header,
		Body:      requestBody,
		Via:       invokeVia,
	})

	if err != nil {
		tr.Logger.WarnWith("Failed to invoke function", "err", err)

		responseWriter.WriteHeader(http.StatusInternalServerError)
		responseWriter.Write([]byte(`{"error": "Failed to invoke function"}`)) // nolint: errcheck
		return
	}

	// set headers
	for headerName, headerValue := range invocationResult.Headers {

		// don't send nuclio headers to the actual function
		if !strings.HasPrefix(headerName, "x-nuclio") {
			responseWriter.Header().Set(headerName, headerValue[0])
		}
	}

	responseWriter.Header().Set("Content-Type", "application/json")
	responseWriter.WriteHeader(invocationResult.StatusCode)
	responseWriter.Write(invocationResult.Body) // nolint: errcheck
}

func (tr *invocationResource) getInvokeVia(invokeViaName string) platform.InvokeViaType {
	switch invokeViaName {
	// erd: For now, if the UI asked for external IP, force using "via any". "Any" should try external IP
	// and then domain name, which is better
	// case "external-ip":
	// 	 return platform.InvokeViaExternalIP
	case "loadbalancer":
		return platform.InvokeViaLoadBalancer
	case "domain-name":
		return platform.InvokeViaDomainName
	default:
		return platform.InvokeViaAny
	}
}

// register the resource
var invocationResourceInstance = &invocationResource{
	resource: newResource("api/function_invocations", []restful.ResourceMethod{}),
}

func init() {
	invocationResourceInstance.Resource = invocationResourceInstance
	invocationResourceInstance.Register(dashboard.DashboardResourceRegistrySingleton)
}
<commit_msg>[Fix] Set appropriate content type on function invocation response (#1084)<commit_after>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/nuclio/nuclio/pkg/dashboard"
	"github.com/nuclio/nuclio/pkg/platform"
	"github.com/nuclio/nuclio/pkg/restful"
)

type invocationResource struct {
	*resource
	nodeAddresses []string
}

// called after initialization
func (tr *invocationResource) OnAfterInitialize() error {

	// all methods
	for _, registrar := range []func(string, http.HandlerFunc){
		tr.GetRouter().Get,
		tr.GetRouter().Post,
		tr.GetRouter().Put,
		tr.GetRouter().Delete,
		tr.GetRouter().Patch,
		tr.GetRouter().Options,
	} {
		registrar("/*", tr.handleRequest)
	}

	return nil
}

func (tr *invocationResource) handleRequest(responseWriter http.ResponseWriter, request *http.Request) {
	path := request.Header.Get("x-nuclio-path")
	functionName := request.Header.Get("x-nuclio-function-name")
	invokeVia := tr.getInvokeVia(request.Header.Get("x-nuclio-invoke-via"))

	// get namespace from request or use the provided default
	functionNamespace := tr.getNamespaceOrDefault(request.Header.Get("x-nuclio-function-namespace"))

	// if user prefixed path with "/", remove it
	path = strings.TrimLeft(path, "/")

	if functionName == "" || functionNamespace == "" {
		responseWriter.WriteHeader(http.StatusBadRequest)
		responseWriter.Write([]byte(`{"error": "Function name must be provided"}`)) // nolint: errcheck
		return
	}

	requestBody, err := ioutil.ReadAll(request.Body)
	if err != nil {
		responseWriter.WriteHeader(http.StatusInternalServerError)
		responseWriter.Write([]byte(`{"error": "Failed to read request body"}`)) // nolint: errcheck
		return
	}

	// resolve the function host
	invocationResult, err := tr.getPlatform().CreateFunctionInvocation(&platform.CreateFunctionInvocationOptions{
		Name:      functionName,
		Namespace: functionNamespace,
		Path:      path,
		Method:    request.Method,
		Headers:   request.Header,
		Body:      requestBody,
		Via:       invokeVia,
	})

	// defaults to json
	responseWriter.Header().Set("Content-Type", "application/json")

	if err != nil {
		tr.Logger.WarnWith("Failed to invoke function", "err", err)

		responseWriter.WriteHeader(http.StatusInternalServerError)
		responseWriter.Write([]byte(`{"error": "Failed to invoke function"}`)) // nolint: errcheck
		return
	}

	// set headers
	for headerName, headerValue := range invocationResult.Headers {

		// don't send nuclio headers to the actual function
		if !strings.HasPrefix(headerName, "x-nuclio") {
			responseWriter.Header().Set(headerName, headerValue[0])
		}
	}

	responseWriter.WriteHeader(invocationResult.StatusCode)
	responseWriter.Write(invocationResult.Body) // nolint: errcheck
}

func (tr *invocationResource) getInvokeVia(invokeViaName string) platform.InvokeViaType {
	switch invokeViaName {
	// erd: For now, if the UI asked for external IP, force using "via any". "Any" should try external IP
	// and then domain name, which is better
	// case "external-ip":
	// 	 return platform.InvokeViaExternalIP
	case "loadbalancer":
		return platform.InvokeViaLoadBalancer
	case "domain-name":
		return platform.InvokeViaDomainName
	default:
		return platform.InvokeViaAny
	}
}

// register the resource
var invocationResourceInstance = &invocationResource{
	resource: newResource("api/function_invocations", []restful.ResourceMethod{}),
}

func init() {
	invocationResourceInstance.Resource = invocationResourceInstance
	invocationResourceInstance.Register(dashboard.DashboardResourceRegistrySingleton)
}
<|endoftext|>


<commit_before>package sqlstore

import (
	"database/sql"
	"time"

	"github.com/usefathom/fathom/pkg/models"
)

func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStats, error) {
	stats := &models.SiteStats{New: false}
	query := db.Rebind(`SELECT * FROM daily_site_stats WHERE site_id = ? AND date = ? LIMIT 1`)
	err := db.Get(stats, query, siteID, date.Format("2006-01-02"))
	return stats, mapError(err)
}

func (db *sqlstore) SaveSiteStats(s *models.SiteStats) error {
	if s.New {
		return db.insertSiteStats(s)
	}

	return db.updateSiteStats(s)
}

func (db *sqlstore) insertSiteStats(s *models.SiteStats) error {
	query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
	_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
	return err
}

func (db *sqlstore) updateSiteStats(s *models.SiteStats) error {
	query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE site_id = ? AND date = ?`)
	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format("2006-01-02"))
	return err
}

func (db *sqlstore) GetSiteStatsPerDay(siteID int64, startDate time.Time, endDate time.Time) ([]*models.SiteStats, error) {
	results := []*models.SiteStats{}
	sql := `SELECT * FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	err := db.Select(&results, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return results, err
}

func (db *sqlstore) GetAggregatedSiteStats(siteID int64, startDate time.Time, endDate time.Time) (*models.SiteStats, error) {
	stats := &models.SiteStats{}
	query := db.Rebind(`SELECT
		COALESCE(SUM(pageviews), 0) AS pageviews,
		COALESCE(SUM(visitors), 0) AS visitors,
		COALESCE(SUM(sessions), 0) AS sessions,
		COALESCE(ROUND(SUM(pageviews*avg_duration) / NULLIF(SUM(pageviews), 0), 4), 0.00) AS avg_duration,
		COALESCE(ROUND(SUM(sessions*bounce_rate) / NULLIF(SUM(sessions), 0), 4), 0.00) AS bounce_rate
	 FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ? LIMIT 1`)
	err := db.Get(stats, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return stats, mapError(err)
}

func (db *sqlstore) GetTotalSiteViews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
	sql := `SELECT COALESCE(SUM(pageviews), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total int64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetTotalSiteVisitors(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
	sql := `SELECT COALESCE(SUM(visitors), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total int64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetTotalSiteSessions(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
	sql := `SELECT COALESCE(SUM(sessions), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total int64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetAverageSiteDuration(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
	sql := `SELECT COALESCE(ROUND(SUM(pageviews*avg_duration)/SUM(pageviews), 4), 0.00) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total float64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetAverageSiteBounceRate(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
	sql := `SELECT COALESCE(ROUND(SUM(sessions*bounce_rate)/SUM(sessions), 4), 0.00) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total float64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) {
	var siteTrackingID string
	var total int64
	if err := db.Get(&siteTrackingID, db.Rebind(`SELECT tracking_id FROM sites WHERE id = ?`), siteID); err != nil && err != sql.ErrNoRows {
		return 0, mapError(err)
	}

	sql := `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND ( duration = 0 OR is_bounce = TRUE) AND  timestamp > ?`
	query := db.Rebind(sql)
	if err := db.Get(&total, query, siteTrackingID, time.Now().Add(-5*time.Minute)); err != nil {
		return 0, mapError(err)
	}

	return total, nil
}
<commit_msg>account for empty site_tracking_id in realtime visitor query<commit_after>package sqlstore

import (
	"database/sql"
	"time"

	"github.com/usefathom/fathom/pkg/models"
)

func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStats, error) {
	stats := &models.SiteStats{New: false}
	query := db.Rebind(`SELECT * FROM daily_site_stats WHERE site_id = ? AND date = ? LIMIT 1`)
	err := db.Get(stats, query, siteID, date.Format("2006-01-02"))
	return stats, mapError(err)
}

func (db *sqlstore) SaveSiteStats(s *models.SiteStats) error {
	if s.New {
		return db.insertSiteStats(s)
	}

	return db.updateSiteStats(s)
}

func (db *sqlstore) insertSiteStats(s *models.SiteStats) error {
	query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
	_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
	return err
}

func (db *sqlstore) updateSiteStats(s *models.SiteStats) error {
	query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE site_id = ? AND date = ?`)
	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format("2006-01-02"))
	return err
}

func (db *sqlstore) GetSiteStatsPerDay(siteID int64, startDate time.Time, endDate time.Time) ([]*models.SiteStats, error) {
	results := []*models.SiteStats{}
	sql := `SELECT * FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	err := db.Select(&results, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return results, err
}

func (db *sqlstore) GetAggregatedSiteStats(siteID int64, startDate time.Time, endDate time.Time) (*models.SiteStats, error) {
	stats := &models.SiteStats{}
	query := db.Rebind(`SELECT
		COALESCE(SUM(pageviews), 0) AS pageviews,
		COALESCE(SUM(visitors), 0) AS visitors,
		COALESCE(SUM(sessions), 0) AS sessions,
		COALESCE(ROUND(SUM(pageviews*avg_duration) / NULLIF(SUM(pageviews), 0), 4), 0.00) AS avg_duration,
		COALESCE(ROUND(SUM(sessions*bounce_rate) / NULLIF(SUM(sessions), 0), 4), 0.00) AS bounce_rate
	 FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ? LIMIT 1`)
	err := db.Get(stats, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return stats, mapError(err)
}

func (db *sqlstore) GetTotalSiteViews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
	sql := `SELECT COALESCE(SUM(pageviews), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total int64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetTotalSiteVisitors(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
	sql := `SELECT COALESCE(SUM(visitors), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total int64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetTotalSiteSessions(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
	sql := `SELECT COALESCE(SUM(sessions), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total int64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetAverageSiteDuration(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
	sql := `SELECT COALESCE(ROUND(SUM(pageviews*avg_duration)/SUM(pageviews), 4), 0.00) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total float64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetAverageSiteBounceRate(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
	sql := `SELECT COALESCE(ROUND(SUM(sessions*bounce_rate)/SUM(sessions), 4), 0.00) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
	query := db.Rebind(sql)
	var total float64
	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
	return total, mapError(err)
}

func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) {
	var siteTrackingID string
	var total int64
	if err := db.Get(&siteTrackingID, db.Rebind(`SELECT tracking_id FROM sites WHERE id = ?`), siteID); err != nil && err != sql.ErrNoRows {
		return 0, mapError(err)
	}

	sql := `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR ( ? = 1 AND site_tracking_id = "" )) AND ( duration = 0 OR is_bounce = TRUE) AND  timestamp > ?`
	query := db.Rebind(sql)
	if err := db.Get(&total, query, siteTrackingID, siteID, time.Now().Add(-5*time.Minute)); err != nil {
		return 0, mapError(err)
	}

	return total, nil
}
<|endoftext|>


<commit_before>package hyperspace

import (
	"sort"
	"sync"
	"time"

	"nimona.io/pkg/object"

	"nimona.io/internal/rand"
	"nimona.io/pkg/bloom"
	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/discovery"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/exchange"
	"nimona.io/pkg/log"
	"nimona.io/pkg/peer"
)

var (
	peerType               = new(peer.Peer).GetType()
	peerLookupRequestType  = new(peer.LookupRequest).GetType()
	peerLookupResponseType = new(peer.LookupResponse).GetType()
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

type (
	// Discoverer hyperspace
	Discoverer struct {
		context   context.Context
		peerstore discovery.PeerStorer
		exchange  exchange.Exchange
		local     *peer.LocalPeer
	}
)

// NewDiscoverer returns a new hyperspace discoverer
func NewDiscoverer(
	ctx context.Context,
	ps discovery.PeerStorer,
	exc exchange.Exchange,
	local *peer.LocalPeer,
	bootstrapPeers []*peer.Peer,
) (*Discoverer, error) {
	r := &Discoverer{
		context:   ctx,
		peerstore: ps,
		local:     local,
		exchange:  exc,
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/Discoverer"),
	)

	objectSub := r.exchange.Subscribe(
		exchange.FilterByObjectType(
			peerType,
			peerLookupRequestType,
			peerLookupResponseType,
		),
	)

	go exchange.HandleEnvelopeSubscription(objectSub, r.handleObject)

	// get in touch with bootstrap nodes
	go func() {
		if err := r.bootstrap(ctx, bootstrapPeers); err != nil {
			logger.Error("could not bootstrap", log.Error(err))
		}

		// publish content
		if err := r.publishContentHashes(ctx); err != nil {
			logger.Error("could not publish initial content hashes", log.Error(err))
		}

		// subsequently try to get fresh peers every 5 minutes
		ticker := time.NewTicker(5 * time.Minute)
		for range ticker.C {
			if _, err := r.Lookup(
				context.Background(),
				peer.LookupByContentType("nimona.io/peer.Peer"),
			); err != nil {
				logger.Error("could not refresh peers", log.Error(err))
			}
			if err := r.publishContentHashes(ctx); err != nil {
				logger.Error("could not refresh content hashes", log.Error(err))
			}
		}
	}()

	return r, nil
}

// Lookup finds and returns peer infos from a fingerprint
func (r *Discoverer) Lookup(
	ctx context.Context,
	opts ...peer.LookupOption,
) (<-chan *peer.Peer, error) {
	opt := peer.ParseLookupOptions(opts...)

	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/resolver.Lookup"),
	)
	logger.Debug("looking up")

	bl := bloom.New(opt.Lookups...)

	// create channel to keep peers we find
	peers := make(chan *peer.Peer, 100)

	// send content requests to recipients
	req := &peer.LookupRequest{
		Nonce: rand.String(12),
		Bloom: bl,
	}
	reqObject := req.ToObject()

	peerLookupResponses := make(chan *exchange.Envelope)

	// listen for lookup responses
	resSub := r.exchange.Subscribe(
		exchange.FilterByObjectType(peerLookupResponseType),
		func(e *exchange.Envelope) bool {
			v := e.Payload.Get("nonce:s")
			rn, ok := v.(string)
			return ok && rn == req.Nonce
		},
	)
	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			peerLookupResponses <- e
		}
	}()

	// create channel for the peers we need to ask
	initialRecipients := make(chan crypto.PublicKey, 100)
	recipients := make(chan crypto.PublicKey)
	// keep a record of who responded
	recipientsResponded := map[crypto.PublicKey]bool{}
	recipientsRespondedLock := sync.RWMutex{}

	go func() {
		for {
			recipient := <-recipients
			// check if we've already asked them
			recipientsRespondedLock.RLock()
			if _, asked := recipientsResponded[recipient]; asked {
				recipientsRespondedLock.RUnlock()
				continue
			}
			recipientsRespondedLock.RUnlock()
			recipientsRespondedLock.Lock()
			// else mark them as already been asked
			recipientsResponded[recipient] = false
			recipientsRespondedLock.Unlock()
			// and finally ask them
			err := r.exchange.Send(
				ctx,
				reqObject,
				peer.LookupByOwner(recipient),
				exchange.WithLocalDiscoveryOnly(),
				exchange.WithAsync(),
			)
			if err != nil {
				logger.Debug("could send request to peer", log.Error(err))
			}
			logger.Debug("asked peer", log.String("peer", recipient.String()))
		}
	}()

	go func() {
		timeout := time.NewTimer(time.Second * 10)
		defer close(peers)
		defer close(recipients)
		defer resSub.Cancel()
		for {
			select {
			case <-ctx.Done():
				logger.Debug("ctx done, giving up")
				return
			case <-timeout.C:
				logger.Debug("timeout done, giving up")
				return
			case r := <-initialRecipients:
				recipients <- r
			case e := <-peerLookupResponses:
				res := &peer.LookupResponse{}
				if err := res.FromObject(e.Payload); err != nil {
					continue
				}
				recipientsRespondedLock.Lock()
				recipientsResponded[e.Sender] = true
				recipientsRespondedLock.Unlock()
				for _, p := range res.Peers {
					// add peers to our peerstore
					r.peerstore.Add(p, false)
					// if the peer matches the query, add it to our results
					if opt.Match(p) {
						peers <- p
					}
					// push peer to the list peers we might want to ask next
					recipients <- p.PublicKey()
				}
				recipientsRespondedLock.RLock()
				allDone := true
				for _, answered := range recipientsResponded {
					if !answered {
						allDone = false
					}
				}
				recipientsRespondedLock.RUnlock()
				if allDone {
					return
				}
			}
		}
	}()

	aps, err := r.peerstore.Lookup(ctx, peer.LookupOnlyLocal())
	if err != nil {
		logger.Error("error getting all peers", log.Error(err))
		return nil, err
	}

	pps := []*peer.Peer{}
	for p := range aps {
		pps = append(pps, p)
	}
	cps := getClosest(pps, bl)
	cps = r.withoutOwnPeer(cps)
	for _, p := range cps {
		initialRecipients <- p.PublicKey()
	}
	close(initialRecipients)

	return peers, nil
}

func (r *Discoverer) handleObject(
	e *exchange.Envelope,
) error {
	// attempt to recover correlation id from request id
	ctx := r.context

	// handle payload
	o := e.Payload
	switch o.GetType() {
	case peerType:
		v := &peer.Peer{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handlePeer(ctx, v)
	case peerLookupRequestType:
		v := &peer.LookupRequest{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handlePeerLookup(ctx, v, e)
	case peerLookupResponseType:
		v := &peer.LookupResponse{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		for _, p := range v.Peers {
			r.handlePeer(ctx, p)
		}
	}
	return nil
}

func (r *Discoverer) handlePeer(
	ctx context.Context,
	p *peer.Peer,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/resolver.handlePeer"),
		log.String("peer.publicKey", p.PublicKey().String()),
		log.Strings("peer.addresses", p.Addresses),
	)
	logger.Debug("adding peer to store")
	r.peerstore.Add(p, false)
}

func (r *Discoverer) handlePeerLookup(
	ctx context.Context,
	q *peer.LookupRequest,
	e *exchange.Envelope,
) {
	ctx = context.FromContext(ctx)
	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/resolver.handlePeerLookup"),
		log.String("e.sender", e.Sender.String()),
		log.Any("query.bloom", q.Bloom),
	)

	logger.Debug("handling peer lookup")

	aps, err := r.peerstore.Lookup(ctx, peer.LookupOnlyLocal())
	if err != nil {
		return
	}
	pps := []*peer.Peer{}
	for p := range aps {
		pps = append(pps, p)
	}
	cps := getClosest(pps, q.Bloom)
	cps = append(cps, r.local.GetSignedPeer())
	cps = peer.Unique(cps)

	ctx = context.New(
		context.WithParent(ctx),
	)

	res := &peer.LookupResponse{
		Nonce: q.Nonce,
		Peers: cps,
	}

	err = r.exchange.Send(
		ctx,
		res.ToObject(),
		peer.LookupByOwner(e.Sender),
		exchange.WithLocalDiscoveryOnly(),
		exchange.WithAsync(),
	)
	if err != nil {
		logger.Debug("could not send lookup response",
			log.Error(err),
		)
	}
	logger.With(
		log.Int("n", len(cps)),
	).Debug("handling done, sent n peers")
}

func (r *Discoverer) bootstrap(
	ctx context.Context,
	bootstrapPeers []*peer.Peer,
) error {
	logger := log.FromContext(ctx)
	opts := []exchange.Option{
		exchange.WithLocalDiscoveryOnly(),
		exchange.WithAsync(),
	}
	nonce := rand.String(6)
	q := &peer.LookupRequest{
		Nonce: nonce,
		Bloom: r.local.GetSignedPeer().Bloom,
	}
	o := q.ToObject()
	for _, p := range bootstrapPeers {
		logger.Debug("connecting to bootstrap", log.Strings("addresses", p.Addresses))
		err := r.exchange.SendToPeer(ctx, o, p, opts...)
		if err != nil {
			logger.Debug("could not send request to bootstrap", log.Error(err))
		}
	}
	return nil
}

func (r *Discoverer) publishContentHashes(
	ctx context.Context,
) error {
	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/Discoverer.publishContentHashes"),
	)
	cb := r.local.GetSignedPeer()
	aps, err := r.peerstore.Lookup(ctx, peer.LookupOnlyLocal())
	if err != nil {
		return err
	}
	pps := []*peer.Peer{}
	for p := range aps {
		pps = append(pps, p)
	}
	cps := getClosest(pps, cb.Bloom)
	fs := []crypto.PublicKey{}
	for _, c := range cps {
		fs = append(fs, c.Owners...)
	}
	if len(fs) == 0 {
		logger.Debug("couldn't find peers to tell")
		return errors.New("no peers to tell")
	}

	logger.With(
		log.Int("n", len(fs)),
		log.Any("bloom", cb.Bloom),
	).Debug("trying to tell n peers")

	opts := []exchange.Option{
		exchange.WithLocalDiscoveryOnly(),
		exchange.WithAsync(),
	}

	o := cb.ToObject()
	sig, err := object.NewSignature(r.local.GetPeerPrivateKey(), o)
	if err != nil {
		logger.With(
			log.Error(err),
		).Error("could not sign object")
		return errors.Wrap(err, errors.New("could not sign object"))
	}

	o = o.AddSignature(sig)
	for _, f := range fs {
		err := r.exchange.Send(ctx, o, peer.LookupByOwner(f), opts...)
		if err != nil {
			logger.Debug("could not send request", log.Error(err))
		}
	}
	return nil
}

func (r *Discoverer) withoutOwnPeer(ps []*peer.Peer) []*peer.Peer {
	lp := r.local.GetPeerPublicKey().String()
	pm := map[string]*peer.Peer{}
	for _, p := range ps {
		for _, s := range p.Owners {
			pm[s.String()] = p
		}
	}
	nps := []*peer.Peer{}
	for f, p := range pm {
		if f == lp {
			continue
		}
		nps = append(nps, p)
	}
	return nps
}

// getClosest returns peers that closest resemble the query
func getClosest(ps []*peer.Peer, q bloom.Bloom) []*peer.Peer {
	type kv struct {
		bloomIntersection int
		peer              *peer.Peer
	}

	r := []kv{}
	for _, p := range ps {
		r = append(r, kv{
			bloomIntersection: intersectionCount(
				q.Bloom(),
				p.Bloom,
			),
			peer: p,
		})
	}

	sort.Slice(r, func(i, j int) bool {
		return r[i].bloomIntersection < r[j].bloomIntersection
	})

	fs := []*peer.Peer{}
	for i, c := range r {
		fs = append(fs, c.peer)
		if i > 10 { // TODO make limit configurable
			break
		}
	}

	return fs
}

func intersectionCount(a, b []int64) int {
	m := make(map[int64]uint64)
	for _, k := range a {
		m[k] |= (1 << 0)
	}
	for _, k := range b {
		m[k] |= (1 << 1)
	}

	i := 0
	for _, v := range m {
		a := v&(1<<0) != 0
		b := v&(1<<1) != 0
		if a && b {
			i++
		}
	}

	return i
}
<commit_msg>fix(discovery/hyperspace): reduce locking during lookup<commit_after>package hyperspace

import (
	"sort"
	"time"

	"nimona.io/pkg/object"

	"nimona.io/internal/rand"
	"nimona.io/pkg/bloom"
	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/discovery"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/exchange"
	"nimona.io/pkg/log"
	"nimona.io/pkg/peer"
)

var (
	peerType               = new(peer.Peer).GetType()
	peerLookupRequestType  = new(peer.LookupRequest).GetType()
	peerLookupResponseType = new(peer.LookupResponse).GetType()
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

type (
	// Discoverer hyperspace
	Discoverer struct {
		context   context.Context
		peerstore discovery.PeerStorer
		exchange  exchange.Exchange
		local     *peer.LocalPeer
	}
)

// NewDiscoverer returns a new hyperspace discoverer
func NewDiscoverer(
	ctx context.Context,
	ps discovery.PeerStorer,
	exc exchange.Exchange,
	local *peer.LocalPeer,
	bootstrapPeers []*peer.Peer,
) (*Discoverer, error) {
	r := &Discoverer{
		context:   ctx,
		peerstore: ps,
		local:     local,
		exchange:  exc,
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/Discoverer"),
	)

	objectSub := r.exchange.Subscribe(
		exchange.FilterByObjectType(
			peerType,
			peerLookupRequestType,
			peerLookupResponseType,
		),
	)

	go exchange.HandleEnvelopeSubscription(objectSub, r.handleObject)

	// get in touch with bootstrap nodes
	go func() {
		if err := r.bootstrap(ctx, bootstrapPeers); err != nil {
			logger.Error("could not bootstrap", log.Error(err))
		}

		// publish content
		if err := r.publishContentHashes(ctx); err != nil {
			logger.Error("could not publish initial content hashes", log.Error(err))
		}

		// subsequently try to get fresh peers every 5 minutes
		ticker := time.NewTicker(5 * time.Minute)
		for range ticker.C {
			if _, err := r.Lookup(
				context.Background(),
				peer.LookupByContentType("nimona.io/peer.Peer"),
			); err != nil {
				logger.Error("could not refresh peers", log.Error(err))
			}
			if err := r.publishContentHashes(ctx); err != nil {
				logger.Error("could not refresh content hashes", log.Error(err))
			}
		}
	}()

	return r, nil
}

// Lookup finds and returns peer infos from a fingerprint
func (r *Discoverer) Lookup(
	ctx context.Context,
	opts ...peer.LookupOption,
) (<-chan *peer.Peer, error) {
	opt := peer.ParseLookupOptions(opts...)

	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/resolver.Lookup"),
	)
	logger.Debug("looking up")

	bl := bloom.New(opt.Lookups...)

	// create channel to keep peers we find
	peers := make(chan *peer.Peer, 100)

	// send content requests to recipients
	req := &peer.LookupRequest{
		Nonce: rand.String(12),
		Bloom: bl,
	}
	reqObject := req.ToObject()

	peerLookupResponses := make(chan *exchange.Envelope)

	// listen for lookup responses
	resSub := r.exchange.Subscribe(
		exchange.FilterByObjectType(peerLookupResponseType),
		func(e *exchange.Envelope) bool {
			v := e.Payload.Get("nonce:s")
			rn, ok := v.(string)
			return ok && rn == req.Nonce
		},
	)
	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			peerLookupResponses <- e
		}
	}()

	// create channel for the peers we need to ask
	initialRecipients := make(chan crypto.PublicKey, 100)

	queryPeer := func(pk crypto.PublicKey) {
		err := r.exchange.Send(
			ctx,
			reqObject,
			peer.LookupByOwner(pk),
			exchange.WithLocalDiscoveryOnly(),
			exchange.WithAsync(),
		)
		if err != nil {
			logger.Debug("could send request to peer", log.Error(err))
		}
		logger.Debug("asked peer", log.String("peer", pk.String()))
	}

	go func() {
		// keep a record of who responded
		recipientsResponded := map[crypto.PublicKey]bool{}
		// just in case timeout
		// TODO maybe figure out if the ctx as a timeout before adding one
		timeout := time.NewTimer(time.Second * 10)
		defer close(peers)
		defer resSub.Cancel()
		for {
			select {
			case <-ctx.Done():
				logger.Debug("ctx done, giving up")
				return
			case <-timeout.C:
				logger.Debug("timeout done, giving up")
				return
			case r := <-initialRecipients:
				// mark peer as asked
				recipientsResponded[r] = false
				// ask recipient
				queryPeer(r)
			case e := <-peerLookupResponses:
				res := &peer.LookupResponse{}
				if err := res.FromObject(e.Payload); err != nil {
					continue
				}
				// mark sender as responded
				recipientsResponded[e.Sender] = true
				for _, p := range res.Peers {
					// add peers to our peerstore
					r.peerstore.Add(p, false)
					// if the peer matches the query, add it to our results
					if opt.Match(p) {
						peers <- p
					}
					// check if we've already asked this peer
					if _, asked := recipientsResponded[p.PublicKey()]; asked {
						// if so, move on
						continue
					}
					// else mark peer as asked
					recipientsResponded[p.PublicKey()] = false
					// and ask them
					queryPeer(p.PublicKey())
				}
				allDone := true
				for _, answered := range recipientsResponded {
					if !answered {
						allDone = false
					}
				}
				if allDone {
					return
				}
			}
		}
	}()

	aps, err := r.peerstore.Lookup(ctx, peer.LookupOnlyLocal())
	if err != nil {
		logger.Error("error getting all peers", log.Error(err))
		return nil, err
	}

	pps := []*peer.Peer{}
	for p := range aps {
		pps = append(pps, p)
	}
	cps := getClosest(pps, bl)
	cps = r.withoutOwnPeer(cps)
	for _, p := range cps {
		initialRecipients <- p.PublicKey()
	}
	close(initialRecipients)

	return peers, nil
}

func (r *Discoverer) handleObject(
	e *exchange.Envelope,
) error {
	// attempt to recover correlation id from request id
	ctx := r.context

	// handle payload
	o := e.Payload
	switch o.GetType() {
	case peerType:
		v := &peer.Peer{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handlePeer(ctx, v)
	case peerLookupRequestType:
		v := &peer.LookupRequest{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handlePeerLookup(ctx, v, e)
	case peerLookupResponseType:
		v := &peer.LookupResponse{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		for _, p := range v.Peers {
			r.handlePeer(ctx, p)
		}
	}
	return nil
}

func (r *Discoverer) handlePeer(
	ctx context.Context,
	p *peer.Peer,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/resolver.handlePeer"),
		log.String("peer.publicKey", p.PublicKey().String()),
		log.Strings("peer.addresses", p.Addresses),
	)
	logger.Debug("adding peer to store")
	r.peerstore.Add(p, false)
}

func (r *Discoverer) handlePeerLookup(
	ctx context.Context,
	q *peer.LookupRequest,
	e *exchange.Envelope,
) {
	ctx = context.FromContext(ctx)
	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/resolver.handlePeerLookup"),
		log.String("e.sender", e.Sender.String()),
		log.Any("query.bloom", q.Bloom),
	)

	logger.Debug("handling peer lookup")

	aps, err := r.peerstore.Lookup(ctx, peer.LookupOnlyLocal())
	if err != nil {
		return
	}
	pps := []*peer.Peer{}
	for p := range aps {
		pps = append(pps, p)
	}
	cps := getClosest(pps, q.Bloom)
	cps = append(cps, r.local.GetSignedPeer())
	cps = peer.Unique(cps)

	ctx = context.New(
		context.WithParent(ctx),
	)

	res := &peer.LookupResponse{
		Nonce: q.Nonce,
		Peers: cps,
	}

	err = r.exchange.Send(
		ctx,
		res.ToObject(),
		peer.LookupByOwner(e.Sender),
		exchange.WithLocalDiscoveryOnly(),
		exchange.WithAsync(),
	)
	if err != nil {
		logger.Debug("could not send lookup response",
			log.Error(err),
		)
	}
	logger.With(
		log.Int("n", len(cps)),
	).Debug("handling done, sent n peers")
}

func (r *Discoverer) bootstrap(
	ctx context.Context,
	bootstrapPeers []*peer.Peer,
) error {
	logger := log.FromContext(ctx)
	opts := []exchange.Option{
		exchange.WithLocalDiscoveryOnly(),
		exchange.WithAsync(),
	}
	nonce := rand.String(6)
	q := &peer.LookupRequest{
		Nonce: nonce,
		Bloom: r.local.GetSignedPeer().Bloom,
	}
	o := q.ToObject()
	for _, p := range bootstrapPeers {
		logger.Debug("connecting to bootstrap", log.Strings("addresses", p.Addresses))
		err := r.exchange.SendToPeer(ctx, o, p, opts...)
		if err != nil {
			logger.Debug("could not send request to bootstrap", log.Error(err))
		}
	}
	return nil
}

func (r *Discoverer) publishContentHashes(
	ctx context.Context,
) error {
	logger := log.FromContext(ctx).With(
		log.String("method", "hyperspace/Discoverer.publishContentHashes"),
	)
	cb := r.local.GetSignedPeer()
	aps, err := r.peerstore.Lookup(ctx, peer.LookupOnlyLocal())
	if err != nil {
		return err
	}
	pps := []*peer.Peer{}
	for p := range aps {
		pps = append(pps, p)
	}
	cps := getClosest(pps, cb.Bloom)
	fs := []crypto.PublicKey{}
	for _, c := range cps {
		fs = append(fs, c.Owners...)
	}
	if len(fs) == 0 {
		logger.Debug("couldn't find peers to tell")
		return errors.New("no peers to tell")
	}

	logger.With(
		log.Int("n", len(fs)),
		log.Any("bloom", cb.Bloom),
	).Debug("trying to tell n peers")

	opts := []exchange.Option{
		exchange.WithLocalDiscoveryOnly(),
		exchange.WithAsync(),
	}

	o := cb.ToObject()
	sig, err := object.NewSignature(r.local.GetPeerPrivateKey(), o)
	if err != nil {
		logger.With(
			log.Error(err),
		).Error("could not sign object")
		return errors.Wrap(err, errors.New("could not sign object"))
	}

	o = o.AddSignature(sig)
	for _, f := range fs {
		err := r.exchange.Send(ctx, o, peer.LookupByOwner(f), opts...)
		if err != nil {
			logger.Debug("could not send request", log.Error(err))
		}
	}
	return nil
}

func (r *Discoverer) withoutOwnPeer(ps []*peer.Peer) []*peer.Peer {
	lp := r.local.GetPeerPublicKey().String()
	pm := map[string]*peer.Peer{}
	for _, p := range ps {
		for _, s := range p.Owners {
			pm[s.String()] = p
		}
	}
	nps := []*peer.Peer{}
	for f, p := range pm {
		if f == lp {
			continue
		}
		nps = append(nps, p)
	}
	return nps
}

// getClosest returns peers that closest resemble the query
func getClosest(ps []*peer.Peer, q bloom.Bloom) []*peer.Peer {
	type kv struct {
		bloomIntersection int
		peer              *peer.Peer
	}

	r := []kv{}
	for _, p := range ps {
		r = append(r, kv{
			bloomIntersection: intersectionCount(
				q.Bloom(),
				p.Bloom,
			),
			peer: p,
		})
	}

	sort.Slice(r, func(i, j int) bool {
		return r[i].bloomIntersection < r[j].bloomIntersection
	})

	fs := []*peer.Peer{}
	for i, c := range r {
		fs = append(fs, c.peer)
		if i > 10 { // TODO make limit configurable
			break
		}
	}

	return fs
}

func intersectionCount(a, b []int64) int {
	m := make(map[int64]uint64)
	for _, k := range a {
		m[k] |= (1 << 0)
	}
	for _, k := range b {
		m[k] |= (1 << 1)
	}

	i := 0
	for _, v := range m {
		a := v&(1<<0) != 0
		b := v&(1<<1) != 0
		if a && b {
			i++
		}
	}

	return i
}
<|endoftext|>


<commit_before>package workflow

import (
	"database/sql"
	"fmt"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

func loadJoins(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, u *sdk.User) ([]sdk.WorkflowNodeJoin, error) {
	joinIDs := []int64{}
	_, err := db.Select(&joinIDs, "select id from workflow_node_join where workflow_id = $1", w.ID)
	if err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, sdk.WrapError(err, "loadJoins> Unable to load join IDs on workflow %d", w.ID)
	}

	joins := []sdk.WorkflowNodeJoin{}
	for _, id := range joinIDs {
		j, errJ := loadJoin(db, store, w, id, u)
		if errJ != nil {
			return nil, sdk.WrapError(errJ, "loadJoins> Unable to load join %d on workflow %d", id, w.ID)
		}
		joins = append(joins, *j)
	}

	return joins, nil
}

func loadJoin(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, id int64, u *sdk.User) (*sdk.WorkflowNodeJoin, error) {
	dbjoin := Join{}
	//Load the join
	if err := db.SelectOne(&dbjoin, "select * from workflow_node_join where id = $1 and workflow_id = $2", id, w.ID); err != nil {
		return nil, sdk.WrapError(err, "loadJoin> Unable to load join %d", id)
	}
	dbjoin.WorkflowID = w.ID

	//Load sources
	if _, err := db.Select(&dbjoin.SourceNodeIDs, "select workflow_node_id from workflow_node_join_source where workflow_node_join_id = $1", id); err != nil {
		return nil, sdk.WrapError(err, "loadJoin> Unable to load join %d sources", id)
	}
	j := sdk.WorkflowNodeJoin(dbjoin)

	for _, id := range j.SourceNodeIDs {
		j.SourceNodeRefs = append(j.SourceNodeRefs, fmt.Sprintf("%d", id))
	}

	//Select triggers id
	var triggerIDs []int64
	if _, err := db.Select(&triggerIDs, "select id from workflow_node_join_trigger where  workflow_node_join_id = $1", id); err != nil {
		if err == sql.ErrNoRows {
			return nil, sdk.WrapError(err, "loadJoin> Unable to load join triggers id for join %d", id)
		}
		return nil, sdk.WrapError(err, "loadJoin> Unable to load join triggers id for join %d", id)
	}

	//Load trigegrs
	for _, t := range triggerIDs {
		jt, err := loadJoinTrigger(db, store, w, &j, t, u)
		if err != nil {
			return nil, sdk.WrapError(err, "loadJoin> Unable to load join trigger %d", t)
		}
		j.Triggers = append(j.Triggers, *jt)
	}

	return &j, nil
}

func loadJoinTrigger(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, node *sdk.WorkflowNodeJoin, id int64, u *sdk.User) (*sdk.WorkflowNodeJoinTrigger, error) {
	dbtrigger := JoinTrigger{}
	//Load the trigger
	if err := db.SelectOne(&dbtrigger, "select * from workflow_node_join_trigger where workflow_node_join_id = $1 and id = $2", node.ID, id); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, sdk.WrapError(err, "loadJoinTrigger> Unable to load trigger %d", id)
	}

	t := sdk.WorkflowNodeJoinTrigger(dbtrigger)
	//Load node destination
	if t.WorkflowDestNodeID != 0 {
		dest, err := loadNode(db, store, w, t.WorkflowDestNodeID, u)
		if err != nil {
			return nil, sdk.WrapError(err, "loadJoinTrigger> Unable to load destination node %d", t.WorkflowDestNodeID)
		}
		t.WorkflowDestNode = *dest
	}

	return &t, nil
}

func findNodeByRefInWorkflow(ref string, w *sdk.Workflow) *sdk.WorkflowNode {
	r := findNodeByRef(ref, w.Root)
	if r != nil {
		return r
	}

	for i := range w.Joins {
		j := &w.Joins[i]
		for ti := range j.Triggers {
			t := &j.Triggers[ti]
			r := findNodeByRef(ref, &t.WorkflowDestNode)
			if r != nil {
				return r
			}
		}
	}

	return nil
}

func findNodeByRef(ref string, n *sdk.WorkflowNode) *sdk.WorkflowNode {
	log.Debug("findNodeByRef> finding node %s in node %d (%s) on %s", ref, n.ID, n.Ref, n.Pipeline.Name)
	if n.Ref == ref {
		return n
	}
	for _, t := range n.Triggers {
		n1 := findNodeByRef(ref, &t.WorkflowDestNode)
		if n1 != nil {
			return n1
		}
	}
	return nil
}

func insertJoin(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.WorkflowNodeJoin, u *sdk.User) error {
	log.Debug("insertOrUpdateJoin> %#v", n)
	n.WorkflowID = w.ID
	n.ID = 0
	n.SourceNodeIDs = nil
	dbJoin := Join(*n)

	//Check references to sources
	if len(n.SourceNodeIDs) == 0 {
		if len(n.SourceNodeRefs) == 0 {
			return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Invalid joins references")
		}

		for _, s := range n.SourceNodeRefs {
			//Search references
			var foundRef = findNodeByRefInWorkflow(s, w)
			if foundRef == nil {
				return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Invalid joins references")
			}
			log.Debug("insertOrUpdateJoin> Found reference %s : %d on %s", s, foundRef.ID, foundRef.Pipeline.Name)
			if foundRef.ID == 0 {
				log.Debug("insertOrUpdateJoin> insert or update reference node (%s) %d on %s", s, foundRef.ID, foundRef.Pipeline.Name)
				if err := insertNode(db, w, foundRef, u, true); err != nil {
					return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Unable to insert or update source node")
				}
			}
			n.SourceNodeIDs = append(n.SourceNodeIDs, foundRef.ID)
		}
	}

	//Insert the join
	if err := db.Insert(&dbJoin); err != nil {
		return sdk.WrapError(err, "insertOrUpdateJoin> Unable to insert workflow node join")
	}
	n.ID = dbJoin.ID

	//Setup destination triggers
	for i := range n.Triggers {
		t := &n.Triggers[i]
		if err := insertJoinTrigger(db, w, n, t, u); err != nil {
			return sdk.WrapError(err, "insertOrUpdateJoin> Unable to insert or update join trigger")
		}
	}

	//Insert associations with sources
	query := "insert into workflow_node_join_source(workflow_node_id, workflow_node_join_id) values ($1, $2)"
	for _, source := range n.SourceNodeIDs {
		if _, err := db.Exec(query, source, n.ID); err != nil {
			return sdk.WrapError(err, "insertOrUpdateJoin> Unable to insert associations between node %d and join %d", source, n.ID)
		}
	}

	return nil
}

func insertJoinTrigger(db gorp.SqlExecutor, w *sdk.Workflow, j *sdk.WorkflowNodeJoin, trigger *sdk.WorkflowNodeJoinTrigger, u *sdk.User) error {
	trigger.WorkflowNodeJoinID = j.ID
	trigger.ID = 0

	//Setup destination node
	if err := insertNode(db, w, &trigger.WorkflowDestNode, u, false); err != nil {
		return sdk.WrapError(err, "insertOrUpdateJoinTrigger> Unable to setup destination node")
	}
	trigger.WorkflowDestNodeID = trigger.WorkflowDestNode.ID

	//Insert trigger
	dbt := JoinTrigger(*trigger)
	if err := db.Insert(&dbt); err != nil {
		return sdk.WrapError(err, "insertOrUpdateJoinTrigger> Unable to insert trigger")
	}
	trigger.ID = dbt.ID
	trigger.WorkflowDestNode.TriggerJoinSrcID = trigger.ID

	// Update node trigger ID
	if err := updateWorkflowTriggerJoinSrc(db, &trigger.WorkflowDestNode); err != nil {
		return sdk.WrapError(err, "insertTrigger> Unable to update node %d for trigger %d", trigger.WorkflowDestNode.ID, trigger.ID)
	}

	return nil
}

func deleteJoin(db gorp.SqlExecutor, n sdk.WorkflowNodeJoin) error {
	j := Join(n)
	if _, err := db.Delete(&j); err != nil {
		return sdk.WrapError(err, "deleteJoin> Unable to delete join %d", j.ID)
	}
	return nil
}
<commit_msg>fix (api): load join trigger with trigger not found (#1522)<commit_after>package workflow

import (
	"database/sql"
	"fmt"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

func loadJoins(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, u *sdk.User) ([]sdk.WorkflowNodeJoin, error) {
	joinIDs := []int64{}
	_, err := db.Select(&joinIDs, "select id from workflow_node_join where workflow_id = $1", w.ID)
	if err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, sdk.WrapError(err, "loadJoins> Unable to load join IDs on workflow %d", w.ID)
	}

	joins := []sdk.WorkflowNodeJoin{}
	for _, id := range joinIDs {
		j, errJ := loadJoin(db, store, w, id, u)
		if errJ != nil {
			return nil, sdk.WrapError(errJ, "loadJoins> Unable to load join %d on workflow %d", id, w.ID)
		}
		joins = append(joins, *j)
	}

	return joins, nil
}

func loadJoin(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, id int64, u *sdk.User) (*sdk.WorkflowNodeJoin, error) {
	dbjoin := Join{}
	//Load the join
	if err := db.SelectOne(&dbjoin, "select * from workflow_node_join where id = $1 and workflow_id = $2", id, w.ID); err != nil {
		return nil, sdk.WrapError(err, "loadJoin> Unable to load join %d", id)
	}
	dbjoin.WorkflowID = w.ID

	//Load sources
	if _, err := db.Select(&dbjoin.SourceNodeIDs, "select workflow_node_id from workflow_node_join_source where workflow_node_join_id = $1", id); err != nil {
		return nil, sdk.WrapError(err, "loadJoin> Unable to load join %d sources", id)
	}
	j := sdk.WorkflowNodeJoin(dbjoin)

	for _, id := range j.SourceNodeIDs {
		j.SourceNodeRefs = append(j.SourceNodeRefs, fmt.Sprintf("%d", id))
	}

	//Select triggers id
	var triggerIDs []int64
	if _, err := db.Select(&triggerIDs, "select id from workflow_node_join_trigger where  workflow_node_join_id = $1", id); err != nil {
		if err == sql.ErrNoRows {
			return nil, sdk.WrapError(err, "loadJoin> Unable to load join triggers id for join %d", id)
		}
		return nil, sdk.WrapError(err, "loadJoin> Unable to load join triggers id for join %d", id)
	}

	//Load trigegrs
	for _, t := range triggerIDs {
		jt, err := loadJoinTrigger(db, store, w, &j, t, u)
		if err != nil {
			return nil, sdk.WrapError(err, "loadJoin> Unable to load join trigger %d", t)
		}
		//If the trigger has not been found, skip it
		if jt == nil {
			log.Warning("workflow.loadJoin> Trigger id=%d not found bu referenced by join_id %d", t, id)
			continue
		}
		j.Triggers = append(j.Triggers, *jt)
	}

	return &j, nil
}

func loadJoinTrigger(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, node *sdk.WorkflowNodeJoin, id int64, u *sdk.User) (*sdk.WorkflowNodeJoinTrigger, error) {
	dbtrigger := JoinTrigger{}
	//Load the trigger
	if err := db.SelectOne(&dbtrigger, "select * from workflow_node_join_trigger where workflow_node_join_id = $1 and id = $2", node.ID, id); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, sdk.WrapError(err, "loadJoinTrigger> Unable to load trigger %d", id)
	}

	t := sdk.WorkflowNodeJoinTrigger(dbtrigger)
	//Load node destination
	if t.WorkflowDestNodeID != 0 {
		dest, err := loadNode(db, store, w, t.WorkflowDestNodeID, u)
		if err != nil {
			return nil, sdk.WrapError(err, "loadJoinTrigger> Unable to load destination node %d", t.WorkflowDestNodeID)
		}
		t.WorkflowDestNode = *dest
	}

	return &t, nil
}

func findNodeByRefInWorkflow(ref string, w *sdk.Workflow) *sdk.WorkflowNode {
	r := findNodeByRef(ref, w.Root)
	if r != nil {
		return r
	}

	for i := range w.Joins {
		j := &w.Joins[i]
		for ti := range j.Triggers {
			t := &j.Triggers[ti]
			r := findNodeByRef(ref, &t.WorkflowDestNode)
			if r != nil {
				return r
			}
		}
	}

	return nil
}

func findNodeByRef(ref string, n *sdk.WorkflowNode) *sdk.WorkflowNode {
	log.Debug("findNodeByRef> finding node %s in node %d (%s) on %s", ref, n.ID, n.Ref, n.Pipeline.Name)
	if n.Ref == ref {
		return n
	}
	for _, t := range n.Triggers {
		n1 := findNodeByRef(ref, &t.WorkflowDestNode)
		if n1 != nil {
			return n1
		}
	}
	return nil
}

func insertJoin(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.WorkflowNodeJoin, u *sdk.User) error {
	log.Debug("insertOrUpdateJoin> %#v", n)
	n.WorkflowID = w.ID
	n.ID = 0
	n.SourceNodeIDs = nil
	dbJoin := Join(*n)

	//Check references to sources
	if len(n.SourceNodeIDs) == 0 {
		if len(n.SourceNodeRefs) == 0 {
			return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Invalid joins references")
		}

		for _, s := range n.SourceNodeRefs {
			//Search references
			var foundRef = findNodeByRefInWorkflow(s, w)
			if foundRef == nil {
				return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Invalid joins references")
			}
			log.Debug("insertOrUpdateJoin> Found reference %s : %d on %s", s, foundRef.ID, foundRef.Pipeline.Name)
			if foundRef.ID == 0 {
				log.Debug("insertOrUpdateJoin> insert or update reference node (%s) %d on %s", s, foundRef.ID, foundRef.Pipeline.Name)
				if err := insertNode(db, w, foundRef, u, true); err != nil {
					return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Unable to insert or update source node")
				}
			}
			n.SourceNodeIDs = append(n.SourceNodeIDs, foundRef.ID)
		}
	}

	//Insert the join
	if err := db.Insert(&dbJoin); err != nil {
		return sdk.WrapError(err, "insertOrUpdateJoin> Unable to insert workflow node join")
	}
	n.ID = dbJoin.ID

	//Setup destination triggers
	for i := range n.Triggers {
		t := &n.Triggers[i]
		if err := insertJoinTrigger(db, w, n, t, u); err != nil {
			return sdk.WrapError(err, "insertOrUpdateJoin> Unable to insert or update join trigger")
		}
	}

	//Insert associations with sources
	query := "insert into workflow_node_join_source(workflow_node_id, workflow_node_join_id) values ($1, $2)"
	for _, source := range n.SourceNodeIDs {
		if _, err := db.Exec(query, source, n.ID); err != nil {
			return sdk.WrapError(err, "insertOrUpdateJoin> Unable to insert associations between node %d and join %d", source, n.ID)
		}
	}

	return nil
}

func insertJoinTrigger(db gorp.SqlExecutor, w *sdk.Workflow, j *sdk.WorkflowNodeJoin, trigger *sdk.WorkflowNodeJoinTrigger, u *sdk.User) error {
	trigger.WorkflowNodeJoinID = j.ID
	trigger.ID = 0

	//Setup destination node
	if err := insertNode(db, w, &trigger.WorkflowDestNode, u, false); err != nil {
		return sdk.WrapError(err, "insertOrUpdateJoinTrigger> Unable to setup destination node")
	}
	trigger.WorkflowDestNodeID = trigger.WorkflowDestNode.ID

	//Insert trigger
	dbt := JoinTrigger(*trigger)
	if err := db.Insert(&dbt); err != nil {
		return sdk.WrapError(err, "insertOrUpdateJoinTrigger> Unable to insert trigger")
	}
	trigger.ID = dbt.ID
	trigger.WorkflowDestNode.TriggerJoinSrcID = trigger.ID

	// Update node trigger ID
	if err := updateWorkflowTriggerJoinSrc(db, &trigger.WorkflowDestNode); err != nil {
		return sdk.WrapError(err, "insertTrigger> Unable to update node %d for trigger %d", trigger.WorkflowDestNode.ID, trigger.ID)
	}

	return nil
}

func deleteJoin(db gorp.SqlExecutor, n sdk.WorkflowNodeJoin) error {
	j := Join(n)
	if _, err := db.Delete(&j); err != nil {
		return sdk.WrapError(err, "deleteJoin> Unable to delete join %d", j.ID)
	}
	return nil
}
<|endoftext|>