<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysql

import (
	"strings"
	"testing"
)

func TestJSON(t *testing.T) {
	testcases := []struct {
		data     []byte
		expected string
	}{{
		data:     []byte{},
		expected: `'null'`,
	}, {
		data:     []byte{0, 1, 0, 14, 0, 11, 0, 1, 0, 12, 12, 0, 97, 1, 98},
		expected: `JSON_OBJECT('a','b')`,
	}, {
		data:     []byte{0, 1, 0, 12, 0, 11, 0, 1, 0, 5, 2, 0, 97},
		expected: `JSON_OBJECT('a',2)`,
	}, {
		data:     []byte{2, 2, 0, 10, 0, 5, 1, 0, 5, 2, 0},
		expected: `JSON_ARRAY(1,2)`,
	}, {
		data:     []byte{0, 4, 0, 60, 0, 32, 0, 1, 0, 33, 0, 1, 0, 34, 0, 2, 0, 36, 0, 2, 0, 12, 38, 0, 12, 40, 0, 12, 42, 0, 2, 46, 0, 97, 99, 97, 98, 98, 99, 1, 98, 1, 100, 3, 97, 98, 99, 2, 0, 14, 0, 12, 10, 0, 12, 12, 0, 1, 120, 1, 121},
		expected: `JSON_OBJECT('a','b','c','d','ab','abc','bc',JSON_ARRAY('x','y'))`,
	}, {
		data:     []byte{2, 3, 0, 37, 0, 12, 13, 0, 2, 18, 0, 12, 33, 0, 4, 104, 101, 114, 101, 2, 0, 15, 0, 12, 10, 0, 12, 12, 0, 1, 73, 2, 97, 109, 3, 33, 33, 33},
		expected: `JSON_ARRAY('here',JSON_ARRAY('I','am'),'!!!')`,
	}, {
		data:     []byte{12, 13, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103},
		expected: `'"scalar string"'`,
	}, {
		data:     []byte{4, 1},
		expected: `'true'`,
	}, {
		data:     []byte{4, 2},
		expected: `'false'`,
	}, {
		data:     []byte{4, 0},
		expected: `'null'`,
	}, {
		data:     []byte{5, 255, 255},
		expected: `'-1'`,
	}, {
		data:     []byte{6, 1, 0},
		expected: `'1'`,
	}, {
		data:     []byte{5, 255, 127},
		expected: `'32767'`,
	}, {
		data:     []byte{7, 0, 128, 0, 0},
		expected: `'32768'`,
	}, {
		data:     []byte{5, 0, 128},
		expected: `'-32768'`,
	}, {
		data:     []byte{7, 255, 127, 255, 255},
		expected: `'-32769'`,
	}, {
		data:     []byte{7, 255, 255, 255, 127},
		expected: `'2147483647'`,
	}, {
		data:     []byte{9, 0, 0, 0, 128, 0, 0, 0, 0},
		expected: `'2147483648'`,
	}, {
		data:     []byte{7, 0, 0, 0, 128},
		expected: `'-2147483648'`,
	}, {
		data:     []byte{9, 255, 255, 255, 127, 255, 255, 255, 255},
		expected: `'-2147483649'`,
	}, {
		data:     []byte{10, 255, 255, 255, 255, 255, 255, 255, 255},
		expected: `'18446744073709551615'`,
	}, {
		data:     []byte{9, 0, 0, 0, 0, 0, 0, 0, 128},
		expected: `'-9223372036854775808'`,
	}, {
		data:     []byte{11, 110, 134, 27, 240, 249, 33, 9, 64},
		expected: `'3.14159E+00'`,
	}, {
		data:     []byte{0, 0, 0, 4, 0},
		expected: `JSON_OBJECT()`,
	}, {
		data:     []byte{2, 0, 0, 4, 0},
		expected: `JSON_ARRAY()`,
	}, {
		// opaque, datetime
		data:     []byte{15, 12, 8, 0, 0, 0, 25, 118, 31, 149, 25},
		expected: `CAST(CAST('2015-01-15 23:24:25' AS DATETIME(6)) AS JSON)`,
	}, {
		// opaque, time
		data:     []byte{15, 11, 8, 0, 0, 0, 25, 118, 1, 0, 0},
		expected: `CAST(CAST('23:24:25' AS TIME(6)) AS JSON)`,
	}, {
		// opaque, time
		data:     []byte{15, 11, 8, 192, 212, 1, 25, 118, 1, 0, 0},
		expected: `CAST(CAST('23:24:25.120000' AS TIME(6)) AS JSON)`,
	}, {
		// opaque, date
		data:     []byte{15, 10, 8, 0, 0, 0, 0, 0, 30, 149, 25},
		expected: `CAST(CAST('2015-01-15' AS DATE) AS JSON)`,
	}, {
		// opaque, decimal
		data:     []byte{15, 246, 8, 13, 4, 135, 91, 205, 21, 4, 210},
		expected: `CAST(CAST('123456789.1234' AS DECIMAL(13,4)) AS JSON)`,
	}, {
		// opaque, bit field. Not yet implemented.
		data:     []byte{15, 16, 2, 202, 254},
		expected: `opaque type 16 is not supported yet, with data [2 202 254]`,
	}}

	for _, tcase := range testcases {
		t.Run(tcase.expected, func(t *testing.T) {
			r, err := printJSONData(tcase.data)
			if err != nil {
				if got := err.Error(); !strings.HasPrefix(got, tcase.expected) {
					t.Errorf("unexpected output for %v: got [%v] expected [%v]", tcase.data, got, tcase.expected)
				}
			} else {
				if got := string(r); got != tcase.expected {
					t.Errorf("unexpected output for %v: got [%v] expected [%v]", tcase.data, got, tcase.expected)
				}
			}

		})
	}
}
<commit_msg>staticcheck: unit test for readVariableInt bugfix<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysql

import (
	"strings"
	"testing"
)

func TestJSON(t *testing.T) {
	testcases := []struct {
		data     []byte
		expected string
	}{{
		data:     []byte{},
		expected: `'null'`,
	}, {
		data:     []byte{0, 1, 0, 14, 0, 11, 0, 1, 0, 12, 12, 0, 97, 1, 98},
		expected: `JSON_OBJECT('a','b')`,
	}, {
		data:     []byte{0, 1, 0, 12, 0, 11, 0, 1, 0, 5, 2, 0, 97},
		expected: `JSON_OBJECT('a',2)`,
	}, {
		data:     []byte{2, 2, 0, 10, 0, 5, 1, 0, 5, 2, 0},
		expected: `JSON_ARRAY(1,2)`,
	}, {
		data:     []byte{0, 4, 0, 60, 0, 32, 0, 1, 0, 33, 0, 1, 0, 34, 0, 2, 0, 36, 0, 2, 0, 12, 38, 0, 12, 40, 0, 12, 42, 0, 2, 46, 0, 97, 99, 97, 98, 98, 99, 1, 98, 1, 100, 3, 97, 98, 99, 2, 0, 14, 0, 12, 10, 0, 12, 12, 0, 1, 120, 1, 121},
		expected: `JSON_OBJECT('a','b','c','d','ab','abc','bc',JSON_ARRAY('x','y'))`,
	}, {
		data:     []byte{2, 3, 0, 37, 0, 12, 13, 0, 2, 18, 0, 12, 33, 0, 4, 104, 101, 114, 101, 2, 0, 15, 0, 12, 10, 0, 12, 12, 0, 1, 73, 2, 97, 109, 3, 33, 33, 33},
		expected: `JSON_ARRAY('here',JSON_ARRAY('I','am'),'!!!')`,
	}, {
		data:     []byte{12, 13, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103},
		expected: `'"scalar string"'`,
	}, {
		// repeat the same string 10 times, to test readVariableInt when length of string
		// requires 2 bytes to store
		data: []byte{12, 129, 2,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103,
			115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103},
		expected: `'"scalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar string"'`,
	}, {
		data:     []byte{4, 1},
		expected: `'true'`,
	}, {
		data:     []byte{4, 2},
		expected: `'false'`,
	}, {
		data:     []byte{4, 0},
		expected: `'null'`,
	}, {
		data:     []byte{5, 255, 255},
		expected: `'-1'`,
	}, {
		data:     []byte{6, 1, 0},
		expected: `'1'`,
	}, {
		data:     []byte{5, 255, 127},
		expected: `'32767'`,
	}, {
		data:     []byte{7, 0, 128, 0, 0},
		expected: `'32768'`,
	}, {
		data:     []byte{5, 0, 128},
		expected: `'-32768'`,
	}, {
		data:     []byte{7, 255, 127, 255, 255},
		expected: `'-32769'`,
	}, {
		data:     []byte{7, 255, 255, 255, 127},
		expected: `'2147483647'`,
	}, {
		data:     []byte{9, 0, 0, 0, 128, 0, 0, 0, 0},
		expected: `'2147483648'`,
	}, {
		data:     []byte{7, 0, 0, 0, 128},
		expected: `'-2147483648'`,
	}, {
		data:     []byte{9, 255, 255, 255, 127, 255, 255, 255, 255},
		expected: `'-2147483649'`,
	}, {
		data:     []byte{10, 255, 255, 255, 255, 255, 255, 255, 255},
		expected: `'18446744073709551615'`,
	}, {
		data:     []byte{9, 0, 0, 0, 0, 0, 0, 0, 128},
		expected: `'-9223372036854775808'`,
	}, {
		data:     []byte{11, 110, 134, 27, 240, 249, 33, 9, 64},
		expected: `'3.14159E+00'`,
	}, {
		data:     []byte{0, 0, 0, 4, 0},
		expected: `JSON_OBJECT()`,
	}, {
		data:     []byte{2, 0, 0, 4, 0},
		expected: `JSON_ARRAY()`,
	}, {
		// opaque, datetime
		data:     []byte{15, 12, 8, 0, 0, 0, 25, 118, 31, 149, 25},
		expected: `CAST(CAST('2015-01-15 23:24:25' AS DATETIME(6)) AS JSON)`,
	}, {
		// opaque, time
		data:     []byte{15, 11, 8, 0, 0, 0, 25, 118, 1, 0, 0},
		expected: `CAST(CAST('23:24:25' AS TIME(6)) AS JSON)`,
	}, {
		// opaque, time
		data:     []byte{15, 11, 8, 192, 212, 1, 25, 118, 1, 0, 0},
		expected: `CAST(CAST('23:24:25.120000' AS TIME(6)) AS JSON)`,
	}, {
		// opaque, date
		data:     []byte{15, 10, 8, 0, 0, 0, 0, 0, 30, 149, 25},
		expected: `CAST(CAST('2015-01-15' AS DATE) AS JSON)`,
	}, {
		// opaque, decimal
		data:     []byte{15, 246, 8, 13, 4, 135, 91, 205, 21, 4, 210},
		expected: `CAST(CAST('123456789.1234' AS DECIMAL(13,4)) AS JSON)`,
	}, {
		// opaque, bit field. Not yet implemented.
		data:     []byte{15, 16, 2, 202, 254},
		expected: `opaque type 16 is not supported yet, with data [2 202 254]`,
	}}

	for _, tcase := range testcases {
		t.Run(tcase.expected, func(t *testing.T) {
			r, err := printJSONData(tcase.data)
			if err != nil {
				if got := err.Error(); !strings.HasPrefix(got, tcase.expected) {
					t.Errorf("unexpected output for %v: got [%v] expected [%v]", tcase.data, got, tcase.expected)
				}
			} else {
				if got := string(r); got != tcase.expected {
					t.Errorf("unexpected output for %v: got [%v] expected [%v]", tcase.data, got, tcase.expected)
				}
			}

		})
	}
}
<|endoftext|>


<commit_before>package models

import (
	"labix.org/v2/mgo/bson"
	"time"
)

type Comment struct {
	Id           bson.ObjectId          `bson:"_id"`
	Body         string                 `bson:"body"`
	OriginType   string                 `bson:"originType"`
	OriginId     bson.ObjectId          `bson:"originId"`
	DeletedAt    time.Time              `bson:"deletedAt,omitempty"`
	Meta         Meta                   `bson:"meta"`
	IsLowQuality bool                   `bson:"isLowQuality,omitempty"`
	DeletedBy    map[string]interface{} `bson:"deletedBy"`
}
<commit_msg>Migration: SocialMessageId field is added to comment collection<commit_after>package models

import (
	"labix.org/v2/mgo/bson"
	"time"
)

type Comment struct {
	Id              bson.ObjectId          `bson:"_id"`
	Body            string                 `bson:"body"`
	OriginType      string                 `bson:"originType"`
	OriginId        bson.ObjectId          `bson:"originId"`
	DeletedAt       time.Time              `bson:"deletedAt,omitempty"`
	Meta            Meta                   `bson:"meta"`
	IsLowQuality    bool                   `bson:"isLowQuality,omitempty"`
	DeletedBy       map[string]interface{} `bson:"deletedBy"`
	SocialMessageId int64                  `bson:"socialMessageId,omitempty"`
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io

import (
	"bytes"
	"strings"
)

// FcnConvertNum is a function to convert number to string
type FcnConvertNum func(row int, x float64) string

// FcnRow is a function that returns the row value as string
type FcnRow func(row int) string

// Report holds data to generate LaTeX and PDF files
type Report struct {

	// configuration
	Title     string // title of pdf
	Author    string // author of pdf
	Landscape bool   // to format paper

	// default options
	TablePos    string  // default table positioning key; e.g. !t (to be written as [!t])
	NumFmt      string  // default number formatting string; e.g. "%v", "%g" or "%.3f"
	TableFontSz string  // default table fontsize string; e.g. \scriptsize
	TableColSep float64 // default table column separation in 'em'; e.g. 0.5 => \setlength{\tabcolsep}{0.5em}
	NotesFmt    string  // default table notes format; e.g. 'c' or 'p{7cm}'
	NotesFontSz string  // default table notes font size; e.g. \scriptsize

	// options
	DoNotAlignTable   bool // align coluns in TeX table (has to loop over rows first...)
	DoNotUseGeomPkg   bool // do not use package geometry for margins
	DoNotGeneratePDF  bool // do not generate pdf when writing tex files
	DoNotShowMessages bool // do not show messages

	// internal
	buffer *bytes.Buffer
}

// Reset clears report
func (o *Report) Reset() {
	o.buffer.Reset()
}

// AddSection adds section and subsections to report
func (o *Report) AddSection(name string, level int) {
	sec := "section"
	for i := 0; i < level; i++ {
		if i < 2 {
			sec = "sub" + sec
		}
	}
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}
	Ff(o.buffer, "\n")
	Ff(o.buffer, "\\%s{%s}\n", sec, name)
}

// AddTex adds TeX commands
func (o *Report) AddTex(commands string) {
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}
	Ff(o.buffer, "\n%s\n", commands)
}

// AddTable adds tex table to report
//   caption -- caption of table
//   label -- label of table
//   keys -- column keys
//   T -- table of values: key => row value
//   key2tex -- maps key to tex formatted text of this key (i.e. equation). may be nil
//   key2convert -- maps key to function to convert numbers to string in that column. may be nil
func (o *Report) AddTable(caption, label, notes string, keys []string, T map[string][]float64, key2tex map[string]string, key2numfmt map[string]FcnConvertNum) {

	// new buffer
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}

	// fix default parameters
	o.fixDefaults()

	// find column widths and set formatting string
	strfmt := make([]string, len(keys)) // for each column
	if !o.DoNotAlignTable {
		widths := make([]int, len(keys)) // column widths
		for j, key := range keys {
			if key2tex == nil {
				widths[j] = imax(widths[j], len(key))
			} else {
				widths[j] = imax(widths[j], len(key2tex[key]))
			}
			for i, v := range T[key] {
				if key2numfmt == nil {
					widths[j] = imax(widths[j], len(Sf(o.NumFmt, v)))
				} else {
					widths[j] = imax(widths[j], len(key2numfmt[key](i, v)))
				}
			}
		}
		for j, width := range widths {
			strfmt[j] = "%" + Sf("%d", width) + "s"
		}
	} else {
		for j := 0; j < len(keys); j++ {
			strfmt[j] = "%s"
		}
	}

	// start table and tabular
	ncols := len(keys)
	o.startTableAndTabular(ncols, caption)

	// header
	for j, key := range keys {
		if j > 0 {
			Ff(o.buffer, " & ")
		}
		if key2tex == nil {
			Ff(o.buffer, strfmt[j], key)
		} else {
			Ff(o.buffer, strfmt[j], key2tex[key])
		}
	}
	Ff(o.buffer, " \\\\ \\hline\n")

	// rows
	nrows := len(T[keys[0]])
	for i := 0; i < nrows; i++ {
		if i > 0 {
			Ff(o.buffer, "\n")
		}
		for j, key := range keys {
			if j > 0 {
				Ff(o.buffer, " & ")
			}
			if key2numfmt == nil {
				Ff(o.buffer, strfmt[j], Sf(o.NumFmt, T[key][i]))
			} else {
				Ff(o.buffer, strfmt[j], key2numfmt[key](i, T[key][i]))
			}
		}
		Ff(o.buffer, " \\\\")
	}

	// end tabular and table
	o.endTableAndTabular(ncols, label, notes)
}

// AddTableF adds tex table to report by using a map of functions to extract row values
//   caption -- caption of table
//   label -- label of table
//   keys -- column keys
//   nrows -- number of rows
//   F -- map of functions: key => function returning formatted row value
//   key2tex -- maps key to tex formatted text of this key (i.e. equation). may be nil
func (o *Report) AddTableF(caption, label, notes string, keys []string, nrows int, F map[string]FcnRow, key2tex map[string]string) {

	// new buffer
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}

	// fix default parameters
	o.fixDefaults()

	// find column widths and set formatting string
	strfmt := make([]string, len(keys)) // for each column
	if !o.DoNotAlignTable {
		widths := make([]int, len(keys)) // column widths
		for j, key := range keys {
			if key2tex == nil {
				widths[j] = imax(widths[j], len(key))
			} else {
				widths[j] = imax(widths[j], len(key2tex[key]))
			}
			for i := 0; i < nrows; i++ {
				widths[j] = imax(widths[j], len(F[key](i)))
			}
		}
		for j, width := range widths {
			strfmt[j] = "%" + Sf("%d", width) + "s"
		}
	} else {
		for j := 0; j < len(keys); j++ {
			strfmt[j] = "%s"
		}
	}

	// start table and tabular
	ncols := len(keys)
	o.startTableAndTabular(ncols, caption)

	// header
	for j, key := range keys {
		if j > 0 {
			Ff(o.buffer, " & ")
		}
		if key2tex == nil {
			Ff(o.buffer, strfmt[j], key)
		} else {
			Ff(o.buffer, strfmt[j], key2tex[key])
		}
	}
	Ff(o.buffer, " \\\\ \\hline\n")

	// rows
	for i := 0; i < nrows; i++ {
		if i > 0 {
			Ff(o.buffer, "\n")
		}
		for j, key := range keys {
			if j > 0 {
				Ff(o.buffer, " & ")
			}
			Ff(o.buffer, strfmt[j], F[key](i))
		}
		Ff(o.buffer, " \\\\")
	}

	// end tabular and table
	o.endTableAndTabular(ncols, label, notes)
}

// WriteTexPdf writes tex file and generates pdf file
//  extra -- extra LaTeX commands; may be nil
func (o *Report) WriteTexPdf(dirout, fnkey string, extra *bytes.Buffer) (err error) {

	// header
	pdf := new(bytes.Buffer)
	if o.Landscape {
		Ff(pdf, "\\documentclass[a4paper,landscape]{article}\n")
	} else {
		Ff(pdf, "\\documentclass[a4paper]{article}\n")
	}
	Ff(pdf, "\\usepackage{amsmath}\n")
	Ff(pdf, "\\usepackage{amssymb}\n")
	Ff(pdf, "\\usepackage{booktabs}\n")
	if !o.DoNotUseGeomPkg {
		Ff(pdf, "\\usepackage[margin=1.5cm,footskip=0.5cm]{geometry}\n")
	}

	// title and author
	hasTitleOrAuthor := false
	if o.Title != "" {
		Ff(pdf, "\n")
		Ff(pdf, "\\title{%s}\n", o.Title)
		hasTitleOrAuthor = true
	}
	if o.Author != "" {
		Ff(pdf, "\\author{%s}\n", o.Author)
		hasTitleOrAuthor = true
	}

	// begin document
	Ff(pdf, "\n")
	Ff(pdf, "\\begin{document}\n")
	if hasTitleOrAuthor {
		Ff(pdf, "\\maketitle\n")
	}

	// buffer
	if o.buffer != nil {
		Ff(pdf, "%v\n", o.buffer)
	}

	// extra LaTeX commands
	if extra != nil {
		Ff(pdf, "\n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% extra commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n\n")
		Ff(pdf, "%v\n", extra)
	}

	// end document
	Ff(pdf, "\n")
	Ff(pdf, "\\end{document}\n")

	// write TeX file
	fn := fnkey + ".tex"
	WriteFileD(dirout, fn, pdf)

	// run pdflatex
	if !o.DoNotGeneratePDF {
		_, err = RunCmd(false, "pdflatex", "-interaction=batchmode", "-halt-on-error", "-output-directory="+dirout, fn)
		if err != nil {
			if !o.DoNotShowMessages {
				PfRed("file <%s/%s> generated\n", dirout, fn)
			}
			return
		}
		if !o.DoNotShowMessages {
			PfBlue("file <%s/%s.pdf> generated\n", dirout, fnkey)
		}
	}
	return
}

// TexNum returns a string representation in TeX format of a real number.
// scientificNotation:
//   peforms the conversion of numbers into scientific notation where
//   the exponent notation with e{+-}{##} is converted into \cdot 10^{{+-}##}
func TexNum(fmt string, num float64, scientificNotation bool) (l string) {
	if fmt == "" {
		fmt = "%g"
	}
	l = Sf(fmt, num)
	if scientificNotation {
		s := strings.Split(l, "e-")
		if len(s) == 2 {
			e := s[1]
			if e == "00" {
				l = s[0]
				return
			}
			if e[0] == '0' {
				e = string(e[1])
			}
			l = s[0] + "\\cdot 10^{-" + e + "}"
		}
		s = strings.Split(l, "e+")
		if len(s) == 2 {
			e := s[1]
			if e == "00" {
				l = s[0]
				return
			}
			if e[0] == '0' {
				e = string(e[1])
			}
			l = s[0] + "\\cdot 10^{" + e + "}"
		}
	}
	return
}

// auxiliary //////////////////////////////////////////////////////////////////////////////////////

// startTableAndTabular starts table and tabular
func (o *Report) startTableAndTabular(ncols int, caption string) {

	// start table
	Ff(o.buffer, "\n")
	Ff(o.buffer, "\\begin{table*} [%s] \\centering\n", o.TablePos)
	Ff(o.buffer, "\\caption{%s}\n", caption)

	// set fontsize and column separation
	Ff(o.buffer, o.TableFontSz)
	Ff(o.buffer, " \\setlength{\\tabcolsep}{%gem}\n", o.TableColSep)

	// start tabular
	cc := ""
	for i := 0; i < ncols; i++ {
		cc += "c"
	}
	Ff(o.buffer, "\\begin{tabular}[c]{%s} \\toprule\n", cc)
}

// endTableAndTabular ends table and tabular
func (o *Report) endTableAndTabular(ncols int, label, notes string) {
	Ff(o.buffer, "\n")
	if notes != "" {
		Ff(o.buffer, "\\hline\n")
		Ff(o.buffer, "\\multicolumn{%d}{%s}{\n", ncols, o.NotesFmt)
		Ff(o.buffer, "%s\n", o.NotesFontSz)
		Ff(o.buffer, "%s\n", notes)
		Ff(o.buffer, "} \\\\\n")
	}
	Ff(o.buffer, "\\bottomrule\n")
	Ff(o.buffer, "\\end{tabular}\n")
	Ff(o.buffer, "\\label{tab:%s}\n", label)
	Ff(o.buffer, "\\end{table*}")
}

// fixDefaults fix default values
func (o *Report) fixDefaults() {

	// default table positioning key
	if o.TablePos == "" {
		o.TablePos = "h"
	}

	// default number formatting string
	if o.NumFmt == "" {
		o.NumFmt = "%g"
	}

	// default table fontsize string
	if o.TableFontSz == "" {
		//o.TableFontSz = "\\scriptsize"
	}

	// default table column separation in 'em'; e.g. 0.5 =>
	if o.TableColSep <= 0 {
		o.TableColSep = 0.5
	}

	// default table notes format
	if o.NotesFmt == "" {
		o.NotesFmt = "l"
	}

	// default table notes font size
	if o.NotesFontSz == "" {
		o.NotesFontSz = "\\scriptsize"
	}
}
<commit_msg>Remvoe space in table generation<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io

import (
	"bytes"
	"strings"
)

// FcnConvertNum is a function to convert number to string
type FcnConvertNum func(row int, x float64) string

// FcnRow is a function that returns the row value as string
type FcnRow func(row int) string

// Report holds data to generate LaTeX and PDF files
type Report struct {

	// configuration
	Title     string // title of pdf
	Author    string // author of pdf
	Landscape bool   // to format paper

	// default options
	TablePos    string  // default table positioning key; e.g. !t (to be written as [!t])
	NumFmt      string  // default number formatting string; e.g. "%v", "%g" or "%.3f"
	TableFontSz string  // default table fontsize string; e.g. \scriptsize
	TableColSep float64 // default table column separation in 'em'; e.g. 0.5 => \setlength{\tabcolsep}{0.5em}
	NotesFmt    string  // default table notes format; e.g. 'c' or 'p{7cm}'
	NotesFontSz string  // default table notes font size; e.g. \scriptsize

	// options
	DoNotAlignTable   bool // align coluns in TeX table (has to loop over rows first...)
	DoNotUseGeomPkg   bool // do not use package geometry for margins
	DoNotGeneratePDF  bool // do not generate pdf when writing tex files
	DoNotShowMessages bool // do not show messages

	// internal
	buffer *bytes.Buffer
}

// Reset clears report
func (o *Report) Reset() {
	o.buffer.Reset()
}

// AddSection adds section and subsections to report
func (o *Report) AddSection(name string, level int) {
	sec := "section"
	for i := 0; i < level; i++ {
		if i < 2 {
			sec = "sub" + sec
		}
	}
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}
	Ff(o.buffer, "\n")
	Ff(o.buffer, "\\%s{%s}\n", sec, name)
}

// AddTex adds TeX commands
func (o *Report) AddTex(commands string) {
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}
	Ff(o.buffer, "\n%s\n", commands)
}

// AddTable adds tex table to report
//   caption -- caption of table
//   label -- label of table
//   keys -- column keys
//   T -- table of values: key => row value
//   key2tex -- maps key to tex formatted text of this key (i.e. equation). may be nil
//   key2convert -- maps key to function to convert numbers to string in that column. may be nil
func (o *Report) AddTable(caption, label, notes string, keys []string, T map[string][]float64, key2tex map[string]string, key2numfmt map[string]FcnConvertNum) {

	// new buffer
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}

	// fix default parameters
	o.fixDefaults()

	// find column widths and set formatting string
	strfmt := make([]string, len(keys)) // for each column
	if !o.DoNotAlignTable {
		widths := make([]int, len(keys)) // column widths
		for j, key := range keys {
			if key2tex == nil {
				widths[j] = imax(widths[j], len(key))
			} else {
				widths[j] = imax(widths[j], len(key2tex[key]))
			}
			for i, v := range T[key] {
				if key2numfmt == nil {
					widths[j] = imax(widths[j], len(Sf(o.NumFmt, v)))
				} else {
					widths[j] = imax(widths[j], len(key2numfmt[key](i, v)))
				}
			}
		}
		for j, width := range widths {
			strfmt[j] = "%" + Sf("%d", width) + "s"
		}
	} else {
		for j := 0; j < len(keys); j++ {
			strfmt[j] = "%s"
		}
	}

	// start table and tabular
	ncols := len(keys)
	o.startTableAndTabular(ncols, caption)

	// header
	for j, key := range keys {
		if j > 0 {
			Ff(o.buffer, " & ")
		}
		if key2tex == nil {
			Ff(o.buffer, strfmt[j], key)
		} else {
			Ff(o.buffer, strfmt[j], key2tex[key])
		}
	}
	Ff(o.buffer, " \\\\ \\hline\n")

	// rows
	nrows := len(T[keys[0]])
	for i := 0; i < nrows; i++ {
		if i > 0 {
			Ff(o.buffer, "\n")
		}
		for j, key := range keys {
			if j > 0 {
				Ff(o.buffer, " & ")
			}
			if key2numfmt == nil {
				Ff(o.buffer, strfmt[j], Sf(o.NumFmt, T[key][i]))
			} else {
				Ff(o.buffer, strfmt[j], key2numfmt[key](i, T[key][i]))
			}
		}
		Ff(o.buffer, " \\\\")
	}

	// end tabular and table
	o.endTableAndTabular(ncols, label, notes)
}

// AddTableF adds tex table to report by using a map of functions to extract row values
//   caption -- caption of table
//   label -- label of table
//   keys -- column keys
//   nrows -- number of rows
//   F -- map of functions: key => function returning formatted row value
//   key2tex -- maps key to tex formatted text of this key (i.e. equation). may be nil
func (o *Report) AddTableF(caption, label, notes string, keys []string, nrows int, F map[string]FcnRow, key2tex map[string]string) {

	// new buffer
	if o.buffer == nil {
		o.buffer = new(bytes.Buffer)
	}

	// fix default parameters
	o.fixDefaults()

	// find column widths and set formatting string
	strfmt := make([]string, len(keys)) // for each column
	if !o.DoNotAlignTable {
		widths := make([]int, len(keys)) // column widths
		for j, key := range keys {
			if key2tex == nil {
				widths[j] = imax(widths[j], len(key))
			} else {
				widths[j] = imax(widths[j], len(key2tex[key]))
			}
			for i := 0; i < nrows; i++ {
				widths[j] = imax(widths[j], len(F[key](i)))
			}
		}
		for j, width := range widths {
			strfmt[j] = "%" + Sf("%d", width) + "s"
		}
	} else {
		for j := 0; j < len(keys); j++ {
			strfmt[j] = "%s"
		}
	}

	// start table and tabular
	ncols := len(keys)
	o.startTableAndTabular(ncols, caption)

	// header
	for j, key := range keys {
		if j > 0 {
			Ff(o.buffer, " & ")
		}
		if key2tex == nil {
			Ff(o.buffer, strfmt[j], key)
		} else {
			Ff(o.buffer, strfmt[j], key2tex[key])
		}
	}
	Ff(o.buffer, " \\\\ \\hline\n")

	// rows
	for i := 0; i < nrows; i++ {
		if i > 0 {
			Ff(o.buffer, "\n")
		}
		for j, key := range keys {
			if j > 0 {
				Ff(o.buffer, " & ")
			}
			Ff(o.buffer, strfmt[j], F[key](i))
		}
		Ff(o.buffer, " \\\\")
	}

	// end tabular and table
	o.endTableAndTabular(ncols, label, notes)
}

// WriteTexPdf writes tex file and generates pdf file
//  extra -- extra LaTeX commands; may be nil
func (o *Report) WriteTexPdf(dirout, fnkey string, extra *bytes.Buffer) (err error) {

	// header
	pdf := new(bytes.Buffer)
	if o.Landscape {
		Ff(pdf, "\\documentclass[a4paper,landscape]{article}\n")
	} else {
		Ff(pdf, "\\documentclass[a4paper]{article}\n")
	}
	Ff(pdf, "\\usepackage{amsmath}\n")
	Ff(pdf, "\\usepackage{amssymb}\n")
	Ff(pdf, "\\usepackage{booktabs}\n")
	if !o.DoNotUseGeomPkg {
		Ff(pdf, "\\usepackage[margin=1.5cm,footskip=0.5cm]{geometry}\n")
	}

	// title and author
	hasTitleOrAuthor := false
	if o.Title != "" {
		Ff(pdf, "\n")
		Ff(pdf, "\\title{%s}\n", o.Title)
		hasTitleOrAuthor = true
	}
	if o.Author != "" {
		Ff(pdf, "\\author{%s}\n", o.Author)
		hasTitleOrAuthor = true
	}

	// begin document
	Ff(pdf, "\n")
	Ff(pdf, "\\begin{document}\n")
	if hasTitleOrAuthor {
		Ff(pdf, "\\maketitle\n")
	}

	// buffer
	if o.buffer != nil {
		Ff(pdf, "%v\n", o.buffer)
	}

	// extra LaTeX commands
	if extra != nil {
		Ff(pdf, "\n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% extra commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n\n")
		Ff(pdf, "%v\n", extra)
	}

	// end document
	Ff(pdf, "\n")
	Ff(pdf, "\\end{document}\n")

	// write TeX file
	fn := fnkey + ".tex"
	WriteFileD(dirout, fn, pdf)

	// run pdflatex
	if !o.DoNotGeneratePDF {
		_, err = RunCmd(false, "pdflatex", "-interaction=batchmode", "-halt-on-error", "-output-directory="+dirout, fn)
		if err != nil {
			if !o.DoNotShowMessages {
				PfRed("file <%s/%s> generated\n", dirout, fn)
			}
			return
		}
		if !o.DoNotShowMessages {
			PfBlue("file <%s/%s.pdf> generated\n", dirout, fnkey)
		}
	}
	return
}

// TexNum returns a string representation in TeX format of a real number.
// scientificNotation:
//   peforms the conversion of numbers into scientific notation where
//   the exponent notation with e{+-}{##} is converted into \cdot 10^{{+-}##}
func TexNum(fmt string, num float64, scientificNotation bool) (l string) {
	if fmt == "" {
		fmt = "%g"
	}
	l = Sf(fmt, num)
	if scientificNotation {
		s := strings.Split(l, "e-")
		if len(s) == 2 {
			e := s[1]
			if e == "00" {
				l = s[0]
				return
			}
			if e[0] == '0' {
				e = string(e[1])
			}
			l = s[0] + "\\cdot 10^{-" + e + "}"
		}
		s = strings.Split(l, "e+")
		if len(s) == 2 {
			e := s[1]
			if e == "00" {
				l = s[0]
				return
			}
			if e[0] == '0' {
				e = string(e[1])
			}
			l = s[0] + "\\cdot 10^{" + e + "}"
		}
	}
	return
}

// auxiliary //////////////////////////////////////////////////////////////////////////////////////

// startTableAndTabular starts table and tabular
func (o *Report) startTableAndTabular(ncols int, caption string) {

	// start table
	Ff(o.buffer, "\n")
	Ff(o.buffer, "\\begin{table*} [%s] \\centering\n", o.TablePos)
	Ff(o.buffer, "\\caption{%s}\n", caption)

	// set fontsize and column separation
	Ff(o.buffer, o.TableFontSz)
	Ff(o.buffer, "\\setlength{\\tabcolsep}{%gem}\n", o.TableColSep)

	// start tabular
	cc := ""
	for i := 0; i < ncols; i++ {
		cc += "c"
	}
	Ff(o.buffer, "\\begin{tabular}[c]{%s} \\toprule\n", cc)
}

// endTableAndTabular ends table and tabular
func (o *Report) endTableAndTabular(ncols int, label, notes string) {
	Ff(o.buffer, "\n")
	if notes != "" {
		Ff(o.buffer, "\\hline\n")
		Ff(o.buffer, "\\multicolumn{%d}{%s}{\n", ncols, o.NotesFmt)
		Ff(o.buffer, "%s\n", o.NotesFontSz)
		Ff(o.buffer, "%s\n", notes)
		Ff(o.buffer, "} \\\\\n")
	}
	Ff(o.buffer, "\\bottomrule\n")
	Ff(o.buffer, "\\end{tabular}\n")
	Ff(o.buffer, "\\label{tab:%s}\n", label)
	Ff(o.buffer, "\\end{table*}")
}

// fixDefaults fix default values
func (o *Report) fixDefaults() {

	// default table positioning key
	if o.TablePos == "" {
		o.TablePos = "h"
	}

	// default number formatting string
	if o.NumFmt == "" {
		o.NumFmt = "%g"
	}

	// default table fontsize string
	if o.TableFontSz == "" {
		//o.TableFontSz = "\\scriptsize"
	}

	// default table column separation in 'em'; e.g. 0.5 =>
	if o.TableColSep <= 0 {
		o.TableColSep = 0.5
	}

	// default table notes format
	if o.NotesFmt == "" {
		o.NotesFmt = "l"
	}

	// default table notes font size
	if o.NotesFontSz == "" {
		o.NotesFontSz = "\\scriptsize"
	}
}
<|endoftext|>


<commit_before>// declaration of the basic alu
package iris2

import (
	"fmt"
)

const (
	aluTagInternal = iota
	aluTagByte
	aluTagSbyte
	aluTagInt16
	aluTagUint16
	aluTagInt32
	aluTagUint32
	aluTagInt64
	aluTagUint64
	aluTagFloat32
	aluTagFloat64
)

type aluValue struct {
	dataType byte
	data     Word
}
type Alu struct {
	input            chan Packet
	output           chan Packet
	terminate        bool
	internalRegister aluValue
}

func NewAlu() *Alu {
	var a Alu
	a.output = make(chan Packet)
	a.input = make(chan Packet)
	go a.parseInput()
	return &a
}

func (this *Alu) Terminate() {
	if !this.terminate {
		close(this.input)
		close(this.output)
		this.terminate = true
	}
}

const (
	aluOpNop = iota
	aluOpAdd
	aluOpSub
	aluOpMul
	aluOpDiv
	aluOpMod
	aluOpShl
	aluOpShr
	aluOpAnd
	aluOpOr
	aluOpNot
	aluOpXor

	maskAluGroup = 0x0F
	maskAluFlags = 0xF0

	aluSaveResultFlag = 0x1
	aluFlag2          = 0x2
	aluFlag3          = 0x4
	aluFlag4          = 0x8
)

func aluGroupMajor(value byte) byte {
	return value & maskAluGroup
}

type aluFlags byte

func (this aluFlags) saveResult() bool {
	return (this & aluSaveResultFlag) != 0
}
func (this aluFlags) flag2() bool {
	return ((this & aluFlag2) >> 1) != 0
}
func (this aluFlags) flag3() bool {
	return ((this & aluFlag3) >> 2) != 0
}
func (this aluFlags) flag4() bool {
	return ((this & aluFlag4) >> 3) != 0
}
func getAluFlags(value byte) aluFlags {
	return aluFlags((value & maskAluFlags) >> 4)
}

type aluOperation func(a, b, ret *aluValue) error

var aluOperations = []aluOperation{
	func(_, _, _ *aluValue) error { return nil }, // nop
	func(a, b, ret *aluValue) error {
		if a.dataType != b.dataType {

		}
		return nil
	}, // add
}

func (this *Alu) parseInput() {
	for !this.terminate {
		var out Packet
		input := <-this.input
		if !input.HasData() {
			out.Error = fmt.Errorf("alu: Command stream is empty")
		} else {
			op := aluGroupMajor(input.First())
			//flags := getAluFlags(input.First())
			switch op {
			case aluOpNop:
				// do nothing
			case aluOpAdd:

			case aluOpSub:
			case aluOpMul:
			case aluOpDiv:
			case aluOpMod:
			case aluOpShl:
			case aluOpShr:
			case aluOpAnd:
			case aluOpOr:
			case aluOpNot:
			case aluOpXor:
			default:
				out.Error = fmt.Errorf("Illegal operation %d", op)
			}
		}
		this.output <- out
	}
}
<commit_msg>migrating alu to a separate package<commit_after>// declaration of the basic alu
package standard

import (
	"fmt"
	"github.com/DrItanium/cores/iris2"
)

const (
	aluTagInternal = iota
	aluTagByte
	aluTagSbyte
	aluTagInt16
	aluTagUint16
	aluTagInt32
	aluTagUint32
	aluTagInt64
	aluTagUint64
)

type aluValue struct {
	dataType byte
	data     iris2.Word
}
type Alu struct {
	input            chan iris2.Packet
	output           chan iris2.Packet
	terminate        bool
	internalRegister aluValue
}

func New() *Alu {
	var a Alu
	a.output = make(chan Packet)
	a.input = make(chan Packet)
	go a.parseInput()
	return &a
}

func (this *Alu) Terminate() {
	if !this.terminate {
		close(this.input)
		close(this.output)
		this.terminate = true
	}
}

const (
	Nop = iota
	Add
	Sub
	Mul
	Div
	Mod
	Shl
	Shr
	And
	Or
	Not
	Xor

	maskAluGroup = 0x0F
	maskAluFlags = 0xF0

	saveResultFlag = 0x1
	flag2          = 0x2
	flag3          = 0x4
	flag4          = 0x8
)

func groupMajor(value byte) byte {
	return value & maskAluGroup
}

type flags byte

func (this aluFlags) saveResult() bool {
	return (this & aluSaveResultFlag) != 0
}
func (this aluFlags) flag2() bool {
	return ((this & aluFlag2) >> 1) != 0
}
func (this aluFlags) flag3() bool {
	return ((this & aluFlag3) >> 2) != 0
}
func (this aluFlags) flag4() bool {
	return ((this & aluFlag4) >> 3) != 0
}
func getAluFlags(value byte) aluFlags {
	return aluFlags((value & maskAluFlags) >> 4)
}

type aluOperation func(a, b, ret *aluValue) error

func aluAdd(x, y, ret *aluValue) error {
	return nil
}

var aluOperations = []aluOperation{
	func(_, _, _ *aluValue) error { return nil }, // nop
	aluAdd,
	aluSub,
	aluMul,
	aluDiv,
	aluMod,
	aluShl,
	aluShr,
	aluAdd,
	aluOr,
}

//func aluAdd8

func (this *Alu) parseInput() {
	for !this.terminate {
		var out Packet
		input := <-this.input
		if !input.HasData() {
			out.Error = fmt.Errorf("alu: Command stream is empty")
		} else {
			op := aluGroupMajor(input.First())
			//flags := getAluFlags(input.First())
			switch op {
			case aluOpNop:
				// do nothing
			case aluOpAdd:

			case aluOpSub:
			case aluOpMul:
			case aluOpDiv:
			case aluOpMod:
			case aluOpShl:
			case aluOpShr:
			case aluOpAnd:
			case aluOpOr:
			case aluOpNot:
			case aluOpXor:
			default:
				out.Error = fmt.Errorf("Illegal operation %d", op)
			}
		}
		this.output <- out
	}
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	log "github.com/Sirupsen/logrus"
	"github.com/davecgh/go-spew/spew"
)

func TestItemAssembly(t *testing.T) {
	var tests = []testpair{
		{
			`>\C[14]…今は使用できません。`,
			`>\C[14]…今は使用できません。`,
		},
		{
			`【\C[14]\N[2]\C[0]】　\{アハァァーーーンッ！！`,
			`【\C[14]\N[2]\C[0]】　\{ アハァァーーーンッ！！`,
		},
		{
			`疾風苦無(消費1)　en(v[25] >= 1)`,
			`疾風苦無 (消費 1)　en(v[25] >= 1)`,
		},
		{
			`PT加入en(!s[484] and v[25] <2)`,
			`PT 加入 en(!s[484] and v[25] <2)`,
		},
		{
			`\i[21]メンタルキュア`,
			`\i[21] メンタルキュア`,
		},
		{
			`"0x#{text}"`,
			`"0x#{text}"`,
		},
		{
			`[レース10]`,
			`[レース 10]`,
		},
		{
			`%sの%sを %s 奪った`,
			`%s の %s を %s 奪った`,
		},
	}

	for _, pair := range tests {
		//pair.input = unescapeText(pair.input)
		items, err := parseText(pair.input)
		if err != nil {
			log.Errorf("%s\ntext: %q", err, pair.input)
			log.Error(spew.Sdump(items))
		} else {
			log.Debug(spew.Sdump(items))
		}

		r := assembleItems(items)

		if r != pair.output {
			t.Errorf("For input:\n%q\nexpected:\n%q\ngot:\n%q\nitems:\n%s", pair.input, pair.output, r, spew.Sdump(items))
		}
	}
}
<commit_msg>Add test case<commit_after>package main

import (
	"testing"

	log "github.com/Sirupsen/logrus"
	"github.com/davecgh/go-spew/spew"
)

func TestItemAssembly(t *testing.T) {
	var tests = []testpair{
		{
			`>\C[14]…今は使用できません。`,
			`>\C[14]…今は使用できません。`,
		},
		{
			`【\C[14]\N[2]\C[0]】　\{アハァァーーーンッ！！`,
			`【\C[14]\N[2]\C[0]】　\{ アハァァーーーンッ！！`,
		},
		{
			`疾風苦無(消費1)　en(v[25] >= 1)`,
			`疾風苦無 (消費 1)　en(v[25] >= 1)`,
		},
		{
			`PT加入en(!s[484] and v[25] <2)`,
			`PT 加入 en(!s[484] and v[25] <2)`,
		},
		{
			`\i[21]メンタルキュア`,
			`\i[21] メンタルキュア`,
		},
		{
			`"0x#{text}"`,
			`"0x#{text}"`,
		},
		{
			`[レース10]`,
			`[レース 10]`,
		},
		{
			`%sの%sを %s 奪った`,
			`%s の %s を %s 奪った`,
		},
		{
			`[武器]攻撃時に\V[19]%の闇属性追加ダメージ`,
			`[武器]攻撃時に \V[19]%の闇属性追加ダメージ`,
		},
	}

	for _, pair := range tests {
		//pair.input = unescapeText(pair.input)
		items, err := parseText(pair.input)
		if err != nil {
			log.Errorf("%s\ntext: %q", err, pair.input)
			log.Error(spew.Sdump(items))
		} else {
			log.Debug(spew.Sdump(items))
		}

		r := assembleItems(items)

		if r != pair.output {
			t.Errorf("For input:\n%q\nexpected:\n%q\ngot:\n%q\nitems:\n%s", pair.input, pair.output, r, spew.Sdump(items))
		}
	}
}
<|endoftext|>


<commit_before>package sortedmap

import (
	"testing"
	"time"
)

func TestIterCh(t *testing.T) {
	if _, _, err := newSortedMapFromRandRecords(1000); err != nil {
		t.Fatal(err)
	}
}

func TestIterChCustom(t *testing.T) {
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}
	reversed := true
	if err := verifyRecords(sm.IterChCustom(reversed, 256), reversed); err != nil {
		t.Fatal(err)
	}
}

func TestIterBetweenCh(t *testing.T) {
	const (
		twoNilBoundValsErr = "accepted two nil bound values"
		generalBoundsErr = "between bounds error"
	)
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}
	reversed := false

	_, ok := sm.IterBetweenCh(nil, nil)
	if ok {
		t.Fatalf("TestIterBetweenCh failed: %v", twoNilBoundValsErr)
	}

	ch, ok := sm.IterBetweenCh(time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), time.Now())
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(time.Now(), time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC))
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(time.Now(), time.Date(5321, 1, 1, 0, 0, 0, 0, time.UTC))
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(time.Date(5321, 1, 1, 0, 0, 0, 0, time.UTC), time.Now())
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}

	ch, ok = sm.IterBetweenCh(time.Date(5321, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(5321, 1, 1, 0, 0, 0, 0, time.UTC))
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}
}

func TestIterBetweenChCustom(t *testing.T) {
	const (
		twoNilBoundValsErr = "accepted two nil bound values"
		generalBoundsErr = "only one bound value"
	)
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}

	reversed := true
	buffSize := 64

	earlierDate := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)
	laterDate := time.Now()

	_, ok := sm.IterBetweenChCustom(reversed, buffSize, nil, nil)
	if ok {
		t.Fatalf("TestIterBetween failed: %v", twoNilBoundValsErr)
	}

	ch, ok := sm.IterBetweenChCustom(reversed, buffSize, earlierDate, laterDate)
	if !ok {
		t.Fatalf("TestIterBetween failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenChCustom(reversed, buffSize, laterDate, earlierDate)
	if !ok {
		t.Fatalf("TestIterBetween failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	reversed = false
	ch, ok = sm.IterBetweenChCustom(reversed, buffSize, laterDate, earlierDate)
	if !ok {
		t.Fatalf("TestIterBetween failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}
}

func TestIterFunc(t *testing.T) {
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}
	sm.IterFunc(false, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatal("TestIterFunc failed: nil record!")
		}
		return true
	})
	sm.IterFunc(true, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatal("TestIterFunc failed: nil record!")
		}
		return true
	})
}

func TestIterBetweenFunc(t *testing.T) {
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}

	earlierDate := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)
	laterDate := time.Now()

	sm.IterBetweenFunc(false, earlierDate, laterDate, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatal("TestIterBetweenFunc failed: nil record!")
		}
		return true
	})
	sm.IterBetweenFunc(true, laterDate, earlierDate, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatal("TestIterBetweenFunc failed: nil record!")
		}
		return true
	})
}<commit_msg>updated iter tests<commit_after>package sortedmap

import (
	"testing"
	"time"
)

func TestIterCh(t *testing.T) {
	if _, _, err := newSortedMapFromRandRecords(1000); err != nil {
		t.Fatal(err)
	}
}

func TestIterChCustom(t *testing.T) {
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}
	reversed := true
	if err := verifyRecords(sm.IterChCustom(reversed, 256), reversed); err != nil {
		t.Fatal(err)
	}
}

func TestIterBetweenCh(t *testing.T) {
	const (
		twoNilBoundValsErr = "accepted two nil bound values"
		generalBoundsErr = "between bounds error"
	)

	var (
		maxTime = time.Unix(1<<63 - 62135596801, 999999999)

		earlierDate = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
		laterDate = time.Date(5321, 1, 1, 0, 0, 0, 0, time.UTC)
	)

	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}
	reversed := false

	_, ok := sm.IterBetweenCh(nil, nil)
	if ok {
		t.Fatalf("TestIterBetweenCh failed: %v", twoNilBoundValsErr)
	}

	ch, ok := sm.IterBetweenCh(time.Time{}, time.Time{})
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(time.Time{}, maxTime)
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(maxTime, time.Time{})
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(earlierDate, time.Now())
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(time.Now(), earlierDate)
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(time.Now(), laterDate)
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenCh(laterDate, time.Now())
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}

	ch, ok = sm.IterBetweenCh(laterDate, laterDate)
	if !ok {
		t.Fatalf("TestIterBetweenCh failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}
}

func TestIterBetweenChCustom(t *testing.T) {
	const (
		twoNilBoundValsErr = "accepted two nil bound values"
		generalBoundsErr = "only one bound value"
	)
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}

	reversed := true
	buffSize := 64

	earlierDate := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)
	laterDate := time.Now()

	_, ok := sm.IterBetweenChCustom(reversed, buffSize, nil, nil)
	if ok {
		t.Fatalf("TestIterBetween failed: %v", twoNilBoundValsErr)
	}

	ch, ok := sm.IterBetweenChCustom(reversed, buffSize, earlierDate, laterDate)
	if !ok {
		t.Fatalf("TestIterBetween failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	ch, ok = sm.IterBetweenChCustom(reversed, buffSize, laterDate, earlierDate)
	if !ok {
		t.Fatalf("TestIterBetween failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}

	reversed = false
	ch, ok = sm.IterBetweenChCustom(reversed, buffSize, laterDate, earlierDate)
	if !ok {
		t.Fatalf("TestIterBetween failed: %v", generalBoundsErr)
	}
	if err := verifyRecords(ch, reversed); err != nil {
		t.Fatal(err)
	}
}

func TestIterFunc(t *testing.T) {
	const (
		nilRecErr = "nil record!"
		runawayIterErr = "runaway iterator!"
	)
	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}
	sm.IterFunc(false, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatalf("TestIterFunc failed: %v", nilRecErr)
		}
		return true
	})
	sm.IterFunc(true, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatalf("TestIterFunc failed: %v", nilRecErr)
		}
		return true
	})
	i := 0
	sm.IterFunc(false, func(rec Record) bool {
		if i > 0 {
			t.Fatalf("TestIterFunc failed: %v", runawayIterErr)
		}
		i++
		return false
	})
}

func TestIterBetweenFunc(t *testing.T) {
	const (
		nilRecErr = "nil record!"
		runawayIterErr = "runaway iterator!"
	)

	sm, _, err := newSortedMapFromRandRecords(1000)
	if err != nil {
		t.Fatal(err)
	}

	earlierDate := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)
	laterDate := time.Now()

	sm.IterBetweenFunc(false, earlierDate, laterDate, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatalf("TestIterBetweenFunc failed: %v", nilRecErr)
		}
		return true
	})
	sm.IterBetweenFunc(true, laterDate, earlierDate, func(rec Record) bool {
		if rec.Key == nil {
			t.Fatalf("TestIterBetweenFunc failed: %v", nilRecErr)
		}
		return true
	})
	i := 0
	sm.IterBetweenFunc(true, laterDate, earlierDate, func(rec Record) bool {
		if i > 0 {
			t.Fatalf("TestIterBetweenFunc failed: %v", runawayIterErr)
		}
		i++
		return false
	})
}<|endoftext|>


<commit_before>package herd

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"os"
	"path"
)

type state struct {
	subInodeToRequiredInode map[uint64]uint64
}

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	fmt.Println("buildUpdateRequest()") // TODO(rgooch): Delete debugging.
	subFS := sub.fileSystem
	requiredImage := sub.herd.getImage(sub.requiredImage)
	requiredFS := requiredImage.FileSystem
	filter := requiredImage.Filter
	request.Triggers = requiredImage.Triggers
	var state state
	state.subInodeToRequiredInode = make(map[uint64]uint64)
	compareDirectories(request, &state,
		&subFS.DirectoryInode, &requiredFS.DirectoryInode,
		"/", filter)
	// TODO(rgooch): Implement this.
}

func compareDirectories(request *subproto.UpdateRequest, state *state,
	subDirectory, requiredDirectory *filesystem.DirectoryInode,
	myPathName string, filter *filter.Filter) {
	// First look for entries that should be deleted.
	makeSubDirectory := false
	if subDirectory == nil {
		makeSubDirectory = true
	} else {
		for name, _ := range subDirectory.EntriesByName {
			pathname := path.Join(myPathName, name)
			if filter.Match(pathname) {
				continue
			}
			if _, ok := requiredDirectory.EntriesByName[name]; !ok {
				request.PathsToDelete = append(request.PathsToDelete, pathname)
				fmt.Printf("Delete: %s\n", pathname) // HACK
			}
		}
		if !filesystem.CompareDirectoriesMetadata(subDirectory,
			requiredDirectory, os.Stdout) {
			fmt.Printf("Different directory: %s...\n", myPathName) // HACK
			makeSubDirectory = true
			// TODO(rgooch): Update metadata.
		}
	}
	if makeSubDirectory {
		var newdir subproto.Directory
		newdir.Name = myPathName
		newdir.Mode = requiredDirectory.Mode
		newdir.Uid = requiredDirectory.Uid
		newdir.Gid = requiredDirectory.Gid
		request.DirectoriesToMake = append(request.DirectoriesToMake, newdir)
	}
	for name, requiredEntry := range requiredDirectory.EntriesByName {
		pathname := path.Join(myPathName, name)
		if filter.Match(pathname) {
			continue
		}
		if subDirectory == nil {
			compareEntries(request, state, nil, requiredEntry, pathname,
				filter)
		} else {
			if subEntry, ok := subDirectory.EntriesByName[name]; ok {
				compareEntries(request, state, subEntry, requiredEntry,
					pathname, filter)
			} else {
				compareEntries(request, state, nil, requiredEntry,
					pathname, filter)
			}
		}
	}
}

func addEntry(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	requiredInode := requiredEntry.Inode()
	if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
		var newdir subproto.Directory
		newdir.Name = myPathName
		newdir.Mode = requiredInode.Mode
		newdir.Uid = requiredInode.Uid
		newdir.Gid = requiredInode.Gid
		request.DirectoriesToMake = append(request.DirectoriesToMake, newdir)
	} else {
		fmt.Printf("Add entry: %s...\n", myPathName) // HACK
	}
}

func compareEntries(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry *filesystem.DirectoryEntry,
	myPathName string, filter *filter.Filter) {
	switch requiredInode := requiredEntry.Inode().(type) {
	case *filesystem.RegularInode:
		compareRegularFile(request, state, subEntry,
			requiredInode, requiredEntry.InodeNumber, myPathName)
		return
	case *filesystem.SymlinkInode:
		compareSymlink(request, state, subEntry,
			requiredInode, requiredEntry.InodeNumber, myPathName)
		return
	case *filesystem.Inode:
		compareFile(request, state, subEntry,
			requiredInode, requiredEntry.InodeNumber, myPathName)
		return
	case *filesystem.DirectoryInode:
		compareDirectory(request, state, subEntry, requiredInode, myPathName,
			filter)
		return
	}
	panic("Unsupported entry type")
}

func compareRegularFile(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.RegularInode, requiredInodeNumber uint64,
	myPathName string) {
	if subEntry == nil {
		fmt.Printf("Add rfile: %s...\n", myPathName) // HACK
		// TODO(rgooch): Add entry.
		return
	}
	if subInode, ok := subEntry.Inode().(*filesystem.RegularInode); ok {
		if requiredInum, ok :=
			state.subInodeToRequiredInode[subEntry.InodeNumber]; ok {
			if requiredInum != requiredInodeNumber {
				//
				fmt.Printf("Different links: %s...\n", myPathName) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subEntry.InodeNumber] =
				requiredInodeNumber
		}
		sameMetadata := filesystem.CompareRegularInodesMetadata(
			subInode, requiredInode, os.Stdout)
		sameData := filesystem.CompareRegularInodesData(subInode,
			requiredInode, os.Stdout)
		if sameMetadata && sameData {
			return
		}
		fmt.Printf("Different rfile: %s...\n", myPathName) // HACK
	} else {
		fmt.Printf("Delete+add rfile: %s...\n", myPathName) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareSymlink(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.SymlinkInode, requiredInodeNumber uint64,
	myPathName string) {
	if subInode, ok := subEntry.Inode().(*filesystem.SymlinkInode); ok {
		if requiredInum, ok :=
			state.subInodeToRequiredInode[subEntry.InodeNumber]; ok {
			if requiredInum != requiredInodeNumber {
				fmt.Printf("Different links: %s...\n", myPathName) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subEntry.InodeNumber] =
				requiredInodeNumber
		}
		if filesystem.CompareSymlinkInodes(subInode, requiredInode, os.Stdout) {
			return
		}
		fmt.Printf("Different symlink: %s...\n", myPathName) // HACK
	} else {
		fmt.Printf("Add symlink: %s...\n", myPathName) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareFile(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.Inode, requiredInodeNumber uint64,
	myPathName string) {
	if subInode, ok := subEntry.Inode().(*filesystem.Inode); ok {
		if requiredInum, ok :=
			state.subInodeToRequiredInode[subEntry.InodeNumber]; ok {
			if requiredInum != requiredInodeNumber {
				fmt.Printf("Different links: %s...\n", myPathName) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subEntry.InodeNumber] =
				requiredInodeNumber
		}
		if filesystem.CompareInodes(subInode, requiredInode, os.Stdout) {
			return
		}
		fmt.Printf("Different file: %s...\n", myPathName) // HACK
	} else {
		fmt.Printf("Add file: %s...\n", myPathName) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareDirectory(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.DirectoryInode,
	myPathName string, filter *filter.Filter) {
	if subEntry == nil {
		compareDirectories(request, state, nil, requiredInode, myPathName,
			filter)
		return
	}
	if subInode, ok := subEntry.Inode().(*filesystem.DirectoryInode); ok {
		compareDirectories(request, state, subInode, requiredInode,
			myPathName, filter)
	} else {
		compareDirectories(request, state, nil, requiredInode, myPathName,
			filter)
	}
}
<commit_msg>Simplify compareDirectories().<commit_after>package herd

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"os"
	"path"
)

type state struct {
	subInodeToRequiredInode map[uint64]uint64
}

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	fmt.Println("buildUpdateRequest()") // TODO(rgooch): Delete debugging.
	subFS := sub.fileSystem
	requiredImage := sub.herd.getImage(sub.requiredImage)
	requiredFS := requiredImage.FileSystem
	filter := requiredImage.Filter
	request.Triggers = requiredImage.Triggers
	var state state
	state.subInodeToRequiredInode = make(map[uint64]uint64)
	compareDirectories(request, &state,
		&subFS.DirectoryInode, &requiredFS.DirectoryInode,
		"/", filter)
	// TODO(rgooch): Implement this.
}

func compareDirectories(request *subproto.UpdateRequest, state *state,
	subDirectory, requiredDirectory *filesystem.DirectoryInode,
	myPathName string, filter *filter.Filter) {
	// First look for entries that should be deleted.
	if subDirectory != nil {
		for name, _ := range subDirectory.EntriesByName {
			pathname := path.Join(myPathName, name)
			if filter.Match(pathname) {
				continue
			}
			if _, ok := requiredDirectory.EntriesByName[name]; !ok {
				request.PathsToDelete = append(request.PathsToDelete, pathname)
				fmt.Printf("Delete: %s\n", pathname) // HACK
			}
		}
	}
	for name, requiredEntry := range requiredDirectory.EntriesByName {
		pathname := path.Join(myPathName, name)
		if filter.Match(pathname) {
			continue
		}
		var subEntry *filesystem.DirectoryEntry
		if subDirectory != nil {
			if se, ok := subDirectory.EntriesByName[name]; ok {
				subEntry = se
			}
		}
		if subEntry == nil {
			addEntry(request, state, requiredEntry, pathname)
		} else {
			compareEntries(request, state, subEntry, requiredEntry, pathname,
				filter)
		}
		// If a directory: descend (possibly with the directory for the sub).
		requiredInode := requiredEntry.Inode()
		if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
			var subInode *filesystem.DirectoryInode
			if si, ok := subEntry.Inode().(*filesystem.DirectoryInode); ok {
				subInode = si
			}
			compareDirectories(request, state, requiredInode, subInode,
				pathname, filter)
		}
	}
}

func addEntry(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	requiredInode := requiredEntry.Inode()
	if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
		makeDirectory(request, requiredInode, myPathName, true)
		fmt.Printf("Add directory: %s...\n", myPathName) // HACK
	} else {
		fmt.Printf("Add entry: %s...\n", myPathName) // HACK
		// TODO(rgooch): Add entry.
	}
}

func makeDirectory(request *subproto.UpdateRequest,
	requiredInode *filesystem.DirectoryInode, pathName string, create bool) {
	var newdir subproto.Directory
	newdir.Name = pathName
	newdir.Mode = requiredInode.Mode
	newdir.Uid = requiredInode.Uid
	newdir.Gid = requiredInode.Gid
	if create {
		request.DirectoriesToMake = append(request.DirectoriesToMake, newdir)
	} else {
		request.DirectoriesToChange = append(request.DirectoriesToMake, newdir)
	}
}

func compareEntries(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry *filesystem.DirectoryEntry,
	myPathName string, filter *filter.Filter) {
	switch requiredInode := requiredEntry.Inode().(type) {
	case *filesystem.RegularInode:
		compareRegularFile(request, state, subEntry,
			requiredInode, requiredEntry.InodeNumber, myPathName)
		return
	case *filesystem.SymlinkInode:
		compareSymlink(request, state, subEntry,
			requiredInode, requiredEntry.InodeNumber, myPathName)
		return
	case *filesystem.Inode:
		compareFile(request, state, subEntry,
			requiredInode, requiredEntry.InodeNumber, myPathName)
		return
	case *filesystem.DirectoryInode:
		compareDirectory(request, state, subEntry, requiredInode, myPathName,
			filter)
		return
	}
	panic("Unsupported entry type")
}

func compareRegularFile(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.RegularInode, requiredInodeNumber uint64,
	myPathName string) {
	if subInode, ok := subEntry.Inode().(*filesystem.RegularInode); ok {
		if requiredInum, ok :=
			state.subInodeToRequiredInode[subEntry.InodeNumber]; ok {
			if requiredInum != requiredInodeNumber {
				//
				fmt.Printf("Different links: %s...\n", myPathName) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subEntry.InodeNumber] =
				requiredInodeNumber
		}
		sameMetadata := filesystem.CompareRegularInodesMetadata(
			subInode, requiredInode, os.Stdout)
		sameData := filesystem.CompareRegularInodesData(subInode,
			requiredInode, os.Stdout)
		if sameMetadata && sameData {
			return
		}
		fmt.Printf("Different rfile: %s...\n", myPathName) // HACK
	} else {
		fmt.Printf("Delete+add rfile: %s...\n", myPathName) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareSymlink(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.SymlinkInode, requiredInodeNumber uint64,
	myPathName string) {
	if subInode, ok := subEntry.Inode().(*filesystem.SymlinkInode); ok {
		if requiredInum, ok :=
			state.subInodeToRequiredInode[subEntry.InodeNumber]; ok {
			if requiredInum != requiredInodeNumber {
				fmt.Printf("Different links: %s...\n", myPathName) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subEntry.InodeNumber] =
				requiredInodeNumber
		}
		if filesystem.CompareSymlinkInodes(subInode, requiredInode, os.Stdout) {
			return
		}
		fmt.Printf("Different symlink: %s...\n", myPathName) // HACK
	} else {
		fmt.Printf("Add symlink: %s...\n", myPathName) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareFile(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.Inode, requiredInodeNumber uint64,
	myPathName string) {
	if subInode, ok := subEntry.Inode().(*filesystem.Inode); ok {
		if requiredInum, ok :=
			state.subInodeToRequiredInode[subEntry.InodeNumber]; ok {
			if requiredInum != requiredInodeNumber {
				fmt.Printf("Different links: %s...\n", myPathName) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subEntry.InodeNumber] =
				requiredInodeNumber
		}
		if filesystem.CompareInodes(subInode, requiredInode, os.Stdout) {
			return
		}
		fmt.Printf("Different file: %s...\n", myPathName) // HACK
	} else {
		fmt.Printf("Add file: %s...\n", myPathName) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareDirectory(request *subproto.UpdateRequest, state *state,
	subEntry *filesystem.DirectoryEntry,
	requiredInode *filesystem.DirectoryInode,
	myPathName string, filter *filter.Filter) {
	if subInode, ok := subEntry.Inode().(*filesystem.DirectoryInode); ok {
		if filesystem.CompareDirectoriesMetadata(subInode, requiredInode,
			os.Stdout) {
			return
		}
		makeDirectory(request, requiredInode, myPathName, false)
		fmt.Printf("Different directory: %s...\n", myPathName) // HACK
	} else {
		request.PathsToDelete = append(request.PathsToDelete, myPathName)
		makeDirectory(request, requiredInode, myPathName, true)
		fmt.Printf("Replace non-directory: %s...\n", myPathName) // HACK
	}
}
<|endoftext|>


<commit_before>package syntax

import (
	"bytes"
	"fmt"
	"reflect"
	"runtime"
)

func Unmarshal(data []byte, v interface{}) (int, error) {
	// Check for well-formedness.
	// Avoids filling out half a data structure
	// before discovering a JSON syntax error.
	d := decodeState{}
	d.Write(data)
	return d.unmarshal(v)
}

// These are the options that can be specified in the struct tag.  Right now,
// all of them apply to variable-length vectors and nothing else
type decOpts struct {
	head uint // length of length in bytes
	min  uint // minimum size in bytes
	max  uint // maximum size in bytes
}

type decodeState struct {
	bytes.Buffer
}

func (d *decodeState) unmarshal(v interface{}) (read int, err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}
			if s, ok := r.(string); ok {
				panic(s)
			}
			err = r.(error)
		}
	}()

	rv := reflect.ValueOf(v)
	if rv.Kind() != reflect.Ptr || rv.IsNil() {
		return 0, fmt.Errorf("Invalid unmarshal target (non-pointer or nil)")
	}

	read = d.value(rv)
	return read, nil
}

func (e *decodeState) value(v reflect.Value) int {
	return valueDecoder(v)(e, v, decOpts{})
}

type decoderFunc func(e *decodeState, v reflect.Value, opts decOpts) int

func valueDecoder(v reflect.Value) decoderFunc {
	return typeDecoder(v.Type().Elem())
}

func typeDecoder(t reflect.Type) decoderFunc {
	// Note: Omits the caching / wait-group things that encoding/json uses
	return newTypeDecoder(t)
}

func newTypeDecoder(t reflect.Type) decoderFunc {
	// Note: Does not support Marshaler, so don't need the allowAddr argument

	switch t.Kind() {
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return uintDecoder
	case reflect.Array:
		return newArrayDecoder(t)
	case reflect.Slice:
		return newSliceDecoder(t)
	case reflect.Struct:
		return newStructDecoder(t)
	default:
		panic(fmt.Errorf("Unsupported type (%s)", t))
	}
}

///// Specific decoders below

func uintDecoder(d *decodeState, v reflect.Value, opts decOpts) int {
	var uintLen int
	switch v.Elem().Kind() {
	case reflect.Uint8:
		uintLen = 1
	case reflect.Uint16:
		uintLen = 2
	case reflect.Uint32:
		uintLen = 4
	case reflect.Uint64:
		uintLen = 8
	}

	buf := make([]byte, uintLen)
	n, err := d.Read(buf)
	if err != nil {
		panic(err)
	}
	if n != uintLen {
		panic(fmt.Errorf("Insufficient data to read uint"))
	}

	val := uint64(0)
	for _, b := range buf {
		val = (val << 8) + uint64(b)
	}

	v.Elem().SetUint(val)
	return uintLen
}

//////////

type arrayDecoder struct {
	elemDec decoderFunc
}

func (ad *arrayDecoder) decode(d *decodeState, v reflect.Value, opts decOpts) int {
	n := v.Elem().Type().Len()
	read := 0
	for i := 0; i < n; i += 1 {
		read += ad.elemDec(d, v.Elem().Index(i).Addr(), opts)
	}
	return read
}

func newArrayDecoder(t reflect.Type) decoderFunc {
	dec := &arrayDecoder{typeDecoder(t.Elem())}
	return dec.decode
}

//////////

type sliceDecoder struct {
	elementType reflect.Type
	elementDec  decoderFunc
}

func (sd *sliceDecoder) decode(d *decodeState, v reflect.Value, opts decOpts) int {
	if opts.head == 0 {
		panic(fmt.Errorf("Cannot decode a slice without a header length"))
	}

	lengthBytes := make([]byte, opts.head)
	n, err := d.Read(lengthBytes)
	if err != nil {
		panic(err)
	}
	if uint(n) != opts.head {
		panic(fmt.Errorf("Not enough data to read header"))
	}

	length := uint(0)
	for _, b := range lengthBytes {
		length = (length << 8) + uint(b)
	}

	if opts.max > 0 && length > opts.max {
		panic(fmt.Errorf("Length of vector exceeds declared max"))
	}
	if length < opts.min {
		panic(fmt.Errorf("Length of vector below declared min"))
	}

	data := make([]byte, length)
	n, err = d.Read(data)
	if err != nil {
		panic(err)
	}
	if uint(n) != length {
		panic(fmt.Errorf("Available data less than declared length"))
	}

	elemBuf := &decodeState{}
	elemBuf.Write(data)
	elems := []reflect.Value{}
	read := int(opts.head)
	for elemBuf.Len() > 0 {
		elem := reflect.New(sd.elementType)
		read += sd.elementDec(elemBuf, elem, opts)
		elems = append(elems, elem)
	}

	v.Elem().Set(reflect.MakeSlice(v.Elem().Type(), len(elems), len(elems)))
	for i := 0; i < len(elems); i += 1 {
		v.Elem().Index(i).Set(elems[i].Elem())
	}
	return read
}

func newSliceDecoder(t reflect.Type) decoderFunc {
	dec := &sliceDecoder{
		elementType: t.Elem(),
		elementDec:  typeDecoder(t.Elem()),
	}
	return dec.decode
}

//////////

type structDecoder struct {
	fieldOpts []decOpts
	fieldDecs []decoderFunc
}

func (sd *structDecoder) decode(d *decodeState, v reflect.Value, opts decOpts) int {
	read := 0
	for i := range sd.fieldDecs {
		read += sd.fieldDecs[i](d, v.Elem().Field(i).Addr(), sd.fieldOpts[i])
	}
	return read
}

func newStructDecoder(t reflect.Type) decoderFunc {
	n := t.NumField()
	sd := structDecoder{
		fieldOpts: make([]decOpts, n),
		fieldDecs: make([]decoderFunc, n),
	}

	for i := 0; i < n; i += 1 {
		f := t.Field(i)

		tag := f.Tag.Get("tls")
		tagOpts := parseTag(tag)

		sd.fieldOpts[i] = decOpts{
			head: tagOpts["head"],
			max:  tagOpts["max"],
			min:  tagOpts["min"],
		}

		sd.fieldDecs[i] = typeDecoder(f.Type)
	}

	return sd.decode
}
<commit_msg>Add more error logging in syntax module<commit_after>package syntax

import (
	"bytes"
	"fmt"
	"reflect"
	"runtime"
)

func Unmarshal(data []byte, v interface{}) (int, error) {
	// Check for well-formedness.
	// Avoids filling out half a data structure
	// before discovering a JSON syntax error.
	d := decodeState{}
	d.Write(data)
	return d.unmarshal(v)
}

// These are the options that can be specified in the struct tag.  Right now,
// all of them apply to variable-length vectors and nothing else
type decOpts struct {
	head uint // length of length in bytes
	min  uint // minimum size in bytes
	max  uint // maximum size in bytes
}

type decodeState struct {
	bytes.Buffer
}

func (d *decodeState) unmarshal(v interface{}) (read int, err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}
			if s, ok := r.(string); ok {
				panic(s)
			}
			err = r.(error)
		}
	}()

	rv := reflect.ValueOf(v)
	if rv.Kind() != reflect.Ptr || rv.IsNil() {
		return 0, fmt.Errorf("Invalid unmarshal target (non-pointer or nil)")
	}

	read = d.value(rv)
	return read, nil
}

func (e *decodeState) value(v reflect.Value) int {
	return valueDecoder(v)(e, v, decOpts{})
}

type decoderFunc func(e *decodeState, v reflect.Value, opts decOpts) int

func valueDecoder(v reflect.Value) decoderFunc {
	return typeDecoder(v.Type().Elem())
}

func typeDecoder(t reflect.Type) decoderFunc {
	// Note: Omits the caching / wait-group things that encoding/json uses
	return newTypeDecoder(t)
}

func newTypeDecoder(t reflect.Type) decoderFunc {
	// Note: Does not support Marshaler, so don't need the allowAddr argument

	switch t.Kind() {
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return uintDecoder
	case reflect.Array:
		return newArrayDecoder(t)
	case reflect.Slice:
		return newSliceDecoder(t)
	case reflect.Struct:
		return newStructDecoder(t)
	default:
		panic(fmt.Errorf("Unsupported type (%s)", t))
	}
}

///// Specific decoders below

func uintDecoder(d *decodeState, v reflect.Value, opts decOpts) int {
	var uintLen int
	switch v.Elem().Kind() {
	case reflect.Uint8:
		uintLen = 1
	case reflect.Uint16:
		uintLen = 2
	case reflect.Uint32:
		uintLen = 4
	case reflect.Uint64:
		uintLen = 8
	}

	buf := make([]byte, uintLen)
	n, err := d.Read(buf)
	if err != nil {
		panic(err)
	}
	if n != uintLen {
		panic(fmt.Errorf("Insufficient data to read uint"))
	}

	val := uint64(0)
	for _, b := range buf {
		val = (val << 8) + uint64(b)
	}

	v.Elem().SetUint(val)
	return uintLen
}

//////////

type arrayDecoder struct {
	elemDec decoderFunc
}

func (ad *arrayDecoder) decode(d *decodeState, v reflect.Value, opts decOpts) int {
	n := v.Elem().Type().Len()
	read := 0
	for i := 0; i < n; i += 1 {
		read += ad.elemDec(d, v.Elem().Index(i).Addr(), opts)
	}
	return read
}

func newArrayDecoder(t reflect.Type) decoderFunc {
	dec := &arrayDecoder{typeDecoder(t.Elem())}
	return dec.decode
}

//////////

type sliceDecoder struct {
	elementType reflect.Type
	elementDec  decoderFunc
}

func (sd *sliceDecoder) decode(d *decodeState, v reflect.Value, opts decOpts) int {
	if opts.head == 0 {
		panic(fmt.Errorf("Cannot decode a slice without a header length"))
	}

	lengthBytes := make([]byte, opts.head)
	n, err := d.Read(lengthBytes)
	if err != nil {
		panic(err)
	}
	if uint(n) != opts.head {
		panic(fmt.Errorf("Not enough data to read header"))
	}

	length := uint(0)
	for _, b := range lengthBytes {
		length = (length << 8) + uint(b)
	}

	if opts.max > 0 && length > opts.max {
		panic(fmt.Errorf("Length of vector exceeds declared max"))
	}
	if length < opts.min {
		panic(fmt.Errorf("Length of vector below declared min"))
	}

	data := make([]byte, length)
	n, err = d.Read(data)
	if err != nil {
		panic(err)
	}
	if uint(n) != length {
		panic(fmt.Errorf("Available data less than declared length [%04x < %04x]", n, length))
	}

	elemBuf := &decodeState{}
	elemBuf.Write(data)
	elems := []reflect.Value{}
	read := int(opts.head)
	for elemBuf.Len() > 0 {
		elem := reflect.New(sd.elementType)
		read += sd.elementDec(elemBuf, elem, opts)
		elems = append(elems, elem)
	}

	v.Elem().Set(reflect.MakeSlice(v.Elem().Type(), len(elems), len(elems)))
	for i := 0; i < len(elems); i += 1 {
		v.Elem().Index(i).Set(elems[i].Elem())
	}
	return read
}

func newSliceDecoder(t reflect.Type) decoderFunc {
	dec := &sliceDecoder{
		elementType: t.Elem(),
		elementDec:  typeDecoder(t.Elem()),
	}
	return dec.decode
}

//////////

type structDecoder struct {
	fieldOpts []decOpts
	fieldDecs []decoderFunc
}

func (sd *structDecoder) decode(d *decodeState, v reflect.Value, opts decOpts) int {
	read := 0
	for i := range sd.fieldDecs {
		read += sd.fieldDecs[i](d, v.Elem().Field(i).Addr(), sd.fieldOpts[i])
	}
	return read
}

func newStructDecoder(t reflect.Type) decoderFunc {
	n := t.NumField()
	sd := structDecoder{
		fieldOpts: make([]decOpts, n),
		fieldDecs: make([]decoderFunc, n),
	}

	for i := 0; i < n; i += 1 {
		f := t.Field(i)

		tag := f.Tag.Get("tls")
		tagOpts := parseTag(tag)

		sd.fieldOpts[i] = decOpts{
			head: tagOpts["head"],
			max:  tagOpts["max"],
			min:  tagOpts["min"],
		}

		sd.fieldDecs[i] = typeDecoder(f.Type)
	}

	return sd.decode
}
<|endoftext|>


<commit_before>package iter

type Callback func(value interface{}) (more bool)

type Func func(Callback)

func All(cb Callback, fs ...Func) bool {
	for _, f := range fs {
		all := true
		f(func(v interface{}) bool {
			all = all && cb(v)
			return all
		})
		if !all {
			return false
		}
	}
	return true
}
<commit_msg>Document iter.{Callback,Func}<commit_after>package iter

// Callback receives a value and returns true if another value should be
// received or false to stop iteration.
type Callback func(value interface{}) (more bool)

// Func iterates by calling Callback for each of its values.
type Func func(Callback)

func All(cb Callback, fs ...Func) bool {
	for _, f := range fs {
		all := true
		f(func(v interface{}) bool {
			all = all && cb(v)
			return all
		})
		if !all {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package html

import (
	"errors"
	"github.com/slyrz/newscat/util"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"io"
	"unicode"
)

// Errors returned during Document parsing.
var (
	ErrNoHTML = errors.New("missing html element")
	ErrNoHead = errors.New("missing head element")
	ErrNoBody = errors.New("missing body element")
)

// Document is a parsed HTML document that extracts the document title and
// holds unexported pointers to the html, head and body nodes.
type Document struct {
	Title  *util.Text // the <title>...</title> text.
	Chunks []*Chunk   // all chunks found in this document.

	// Unexported fields.
	html *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used during parsing.
	ancestors int                // bitmask to track specific ancestor types
	linkText  map[*html.Node]int // length of text inside  tags
	normText  map[*html.Node]int // length of text outside  tags
}

// NewDocument parses the HTML data provided through an io.Reader interface.
func NewDocument(r io.Reader) (*Document, error) {
	root, err := html.Parse(r)
	if err != nil {
		return nil, err
	}

	doc := &Document{
		Title:    util.NewText(),
		Chunks:   make([]*Chunk, 0, 512),
		linkText: make(map[*html.Node]int),
		normText: make(map[*html.Node]int),
	}

	// Assign the fields html, head and body from the HTML page.
	iterateNode(root, func(n *html.Node) int {
		switch n.DataAtom {
		case atom.Html:
			doc.html = n
			return IterNext
		case atom.Body:
			doc.body = n
			return IterSkip
		case atom.Head:
			doc.head = n
			return IterSkip
		}
		// Keep going as long as we're missing some nodes.
		return IterNext
	})

	switch {
	case doc.html == nil:
		return nil, ErrNoHTML
	case doc.head == nil:
		return nil, ErrNoHead
	case doc.body == nil:
		return nil, ErrNoBody
	}

	// Detect the document title: First check if the document provides
	// Open Graph metadata; if so, use the metadata rather than the
	// value of the title element, because the metadata tends to be a tad
	// cleaner.
	title := ""
	iterateNode(doc.head, func(n *html.Node) int {
		if n.Type == html.ElementNode && n.DataAtom == atom.Meta {
			prop, content := "", ""
			for _, attr := range n.Attr {
				switch attr.Key {
				case "property":
					prop = attr.Val
				case "content":
					content = attr.Val
				}
			}
			if prop == "og:title" && content != "" {
				title = content
				return IterStop
			}
		}
		return IterNext
	})
	if title != "" {
		doc.Title.WriteString(title)
	} else {
		iterateNode(doc.head, func(n *html.Node) int {
			if n.Type == html.ElementNode && n.DataAtom == atom.Title {
				iterateText(n, doc.Title.WriteString)
				return IterStop
			}
			return IterNext
		})
	}

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now we link the chunks.
	min, max := 0, len(doc.Chunks)-1
	for i := range doc.Chunks {
		if i > min {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < max {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return doc, nil
}

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

// countText counts the text inside of links and the text outside of links
// per html.Node. Counting is done cumulative, so the numbers of a parent node
// include the numbers of its child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.DataAtom == atom.A {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

var removeElements = map[atom.Atom]bool{
	atom.Address:    true,
	atom.Audio:      true,
	atom.Button:     true,
	atom.Canvas:     true,
	atom.Caption:    true,
	atom.Fieldset:   true,
	atom.Figcaption: true,
	atom.Figure:     true,
	atom.Footer:     true,
	atom.Frame:      true,
	atom.Iframe:     true,
	atom.Map:        true,
	atom.Menu:       true,
	atom.Nav:        true,
	atom.Noscript:   true,
	atom.Object:     true,
	atom.Option:     true,
	atom.Output:     true,
	atom.Script:     true,
	atom.Select:     true,
	atom.Style:      true,
	atom.Svg:        true,
	atom.Textarea:   true,
	atom.Video:      true,
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {
	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		return removeElements[c.DataAtom]
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here because calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

var (
	ignoreNames = util.NewRegexFromWords(
		"breadcrumb",
		"byline",
		"caption",
		"comment",
		"community",
		"credit",
		"description",
		"email",
		"footer",
		"gallery",
		"hide",
		"infotext",
		"photo",
		"related",
		"shares",
		"social",
		"story[-_]?bar",
		"story[-_]?feature",
	)
	ignoreStyle = util.NewRegex(`(?i)display:\s*none`)
)

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprops or if
		// its style attribute contains "display: none".
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignoreNames.In(attr.Val) {
					return
				}
			case "style":
				if ignoreStyle.In(attr.Val) {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.DataAtom {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6, atom.A:
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case atom.Article:
			ancestorMask = AncestorArticle &^ doc.ancestors
		case atom.Aside:
			ancestorMask = AncestorAside &^ doc.ancestors
		case atom.Blockquote:
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case atom.Ul, atom.Ol:
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

// TextStat contains the number of words and sentences found in text.
type TextStat struct {
	Words     int // total number of words
	Sentences int // total number of sentences
	Count     int // number of texts used to calculate this stats
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result. For each chunk pick the best TextStats from its ancestors.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<commit_msg>don't filter body and article elements<commit_after>package html

import (
	"errors"
	"github.com/slyrz/newscat/util"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"io"
	"unicode"
)

// Errors returned during Document parsing.
var (
	ErrNoHTML = errors.New("missing html element")
	ErrNoHead = errors.New("missing head element")
	ErrNoBody = errors.New("missing body element")
)

// Document is a parsed HTML document that extracts the document title and
// holds unexported pointers to the html, head and body nodes.
type Document struct {
	Title  *util.Text // the <title>...</title> text.
	Chunks []*Chunk   // all chunks found in this document.

	// Unexported fields.
	html *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used during parsing.
	ancestors int                // bitmask to track specific ancestor types
	linkText  map[*html.Node]int // length of text inside  tags
	normText  map[*html.Node]int // length of text outside  tags
}

// NewDocument parses the HTML data provided through an io.Reader interface.
func NewDocument(r io.Reader) (*Document, error) {
	root, err := html.Parse(r)
	if err != nil {
		return nil, err
	}

	doc := &Document{
		Title:    util.NewText(),
		Chunks:   make([]*Chunk, 0, 512),
		linkText: make(map[*html.Node]int),
		normText: make(map[*html.Node]int),
	}

	// Assign the fields html, head and body from the HTML page.
	iterateNode(root, func(n *html.Node) int {
		switch n.DataAtom {
		case atom.Html:
			doc.html = n
			return IterNext
		case atom.Body:
			doc.body = n
			return IterSkip
		case atom.Head:
			doc.head = n
			return IterSkip
		}
		// Keep going as long as we're missing some nodes.
		return IterNext
	})

	switch {
	case doc.html == nil:
		return nil, ErrNoHTML
	case doc.head == nil:
		return nil, ErrNoHead
	case doc.body == nil:
		return nil, ErrNoBody
	}

	// Detect the document title: First check if the document provides
	// Open Graph metadata; if so, use the metadata rather than the
	// value of the title element, because the metadata tends to be a tad
	// cleaner.
	title := ""
	iterateNode(doc.head, func(n *html.Node) int {
		if n.Type == html.ElementNode && n.DataAtom == atom.Meta {
			prop, content := "", ""
			for _, attr := range n.Attr {
				switch attr.Key {
				case "property":
					prop = attr.Val
				case "content":
					content = attr.Val
				}
			}
			if prop == "og:title" && content != "" {
				title = content
				return IterStop
			}
		}
		return IterNext
	})
	if title != "" {
		doc.Title.WriteString(title)
	} else {
		iterateNode(doc.head, func(n *html.Node) int {
			if n.Type == html.ElementNode && n.DataAtom == atom.Title {
				iterateText(n, doc.Title.WriteString)
				return IterStop
			}
			return IterNext
		})
	}

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now we link the chunks.
	min, max := 0, len(doc.Chunks)-1
	for i := range doc.Chunks {
		if i > min {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < max {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return doc, nil
}

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

// countText counts the text inside of links and the text outside of links
// per html.Node. Counting is done cumulative, so the numbers of a parent node
// include the numbers of its child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.DataAtom == atom.A {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

var removeElements = map[atom.Atom]bool{
	atom.Address:    true,
	atom.Audio:      true,
	atom.Button:     true,
	atom.Canvas:     true,
	atom.Caption:    true,
	atom.Fieldset:   true,
	atom.Figcaption: true,
	atom.Figure:     true,
	atom.Footer:     true,
	atom.Frame:      true,
	atom.Iframe:     true,
	atom.Map:        true,
	atom.Menu:       true,
	atom.Nav:        true,
	atom.Noscript:   true,
	atom.Object:     true,
	atom.Option:     true,
	atom.Output:     true,
	atom.Script:     true,
	atom.Select:     true,
	atom.Style:      true,
	atom.Svg:        true,
	atom.Textarea:   true,
	atom.Video:      true,
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {
	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		return removeElements[c.DataAtom]
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here because calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

var (
	ignoreNames = util.NewRegexFromWords(
		"breadcrumb",
		"byline",
		"caption",
		"comment",
		"community",
		"credit",
		"description",
		"email",
		"footer",
		"gallery",
		"hide",
		"infotext",
		"photo",
		"related",
		"shares",
		"social",
		"story[-_]?bar",
		"story[-_]?feature",
	)
	ignoreStyle = util.NewRegex(`(?i)display:\s*none`)
)

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprops or if
		// its style attribute contains "display: none".
		if n.DataAtom != atom.Body && n.DataAtom != atom.Article {
			for _, attr := range n.Attr {
				switch attr.Key {
				case "id", "class", "itemprop":
					if ignoreNames.In(attr.Val) {
						return
					}
				case "style":
					if ignoreStyle.In(attr.Val) {
						return
					}
				}
			}
		}

		ancestorMask := 0
		switch n.DataAtom {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6, atom.A:
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case atom.Article:
			ancestorMask = AncestorArticle &^ doc.ancestors
		case atom.Aside:
			ancestorMask = AncestorAside &^ doc.ancestors
		case atom.Blockquote:
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case atom.Ul, atom.Ol:
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

// TextStat contains the number of words and sentences found in text.
type TextStat struct {
	Words     int // total number of words
	Sentences int // total number of sentences
	Count     int // number of texts used to calculate this stats
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result. For each chunk pick the best TextStats from its ancestors.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<|endoftext|>