<commit_before>/* heartthreader: mass, multithreaded testing for servers against Heartbleed (CVE-2014-0160)
 * Copyright (C) 2014, Cyphar All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"os"
	"io"
	"bufio"
	"strings"
	"sync"
	"fmt"
	"flag"
	"time"

	tls "github.com/titanous/heartbleeder/tls"
)

type Server struct {
	Host string
	Vulnerable bool
}

const LICENSE = "heartthreader: mass, multithreaded testing for servers against Heartbleed (CVE-2014-0160)\n" +
	"Copyright (C) 2014, Cyphar All rights reserved.\n" +
	"\n" +
	"Redistribution and use in source and binary forms, with or without\n" +
	"modification, are permitted provided that the following conditions are met:\n" +
	"\n" +
	"1. Redistributions of source code must retain the above copyright notice,\n" +
	"   this list of conditions and the following disclaimer.\n" +
	"\n" +
	"2. Redistributions in binary form must reproduce the above copyright notice,\n" +
	"   this list of conditions and the following disclaimer in the documentation\n" +
	"   and/or other materials provided with the distribution.\n" +
	"\n" +
	"3. Neither the name of the copyright holder nor the names of its contributors\n" +
	"   may be used to endorse or promote products derived from this software without\n" +
	"   specific prior written permission.\n" +
	"\n" +
	"THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n" +
	"AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n" +
	"IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n" +
	"DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\n" +
	"FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n" +
	"DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n" +
	"SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n" +
	"CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n" +
	"OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\n" +
	"USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n"

var Config struct {
	NumThreads int
}

func FixLine(line string) string {
	if strings.HasSuffix(line, "\n") {
		l := len(line)
		line = line[:l-1]
	}

	if !strings.Contains(line, ":") {
		line = line + ":443"
	}

	return line
}

func YieldTargets(done <-chan struct{}, files []string) <-chan string {
	hosts := make(chan string)

	go func() {
		defer close(hosts)
		for _, file := range files {
			f, err := os.Open(file)
			defer f.Close()

			if err != nil {
				fmt.Fprintf(os.Stderr, "E: %s\n", err.Error())
				continue
			}

			bf := bufio.NewReader(f)

			var line string
			for line, err = bf.ReadString('\n'); err == nil; line, err = bf.ReadString('\n') {
				host := FixLine(line)
				select {
					case hosts <- host:
					case <-done:
						fmt.Fprintf(os.Stderr, "I: yielding canceled\n")
						return
				}
			}

			f.Close()
		}
	}()

	return hosts
}

func DigestTarget(done <-chan struct{}, hosts <-chan string, c chan<- Server) {
	for host := range hosts {
		wait_err := func(conn *tls.Conn, ch chan<- error) {
			var err error

			err = conn.WriteHeartbeat(32, nil)
			_, _, err = conn.ReadHeartbeat()

			ch <- err
		}

		conn, err := tls.Dial("tcp", host, &tls.Config{InsecureSkipVerify: true})
		defer conn.Close()

		if err != nil {
			fmt.Fprintf(os.Stderr, "E: %s\n", err.Error())
			continue
		}

		quit := time.After(10 * time.Second)
		ech := make(chan error)

		go wait_err(conn, ech)

		srv := Server{host, false}

		select {
			case err = <-ech:
				switch err {
					case nil:
						// VULNERABLE!!
						srv.Vulnerable = true
					case tls.ErrNoHeartbeat:
					default:
				}
			case <-quit:
			case <-done:
				fmt.Fprintf(os.Stderr, "I: digest of %s canceled\n", host)
				return
		}

		select {
			case c <- srv:
			case <-done:
				fmt.Fprintf(os.Stderr, "I: digest of %s canceled\n", host)
				return
		}

		conn.Close()
	}
}

func DigestAll(files []string, out chan<- Server) {
	servers := make(chan Server)
	done := make(chan struct{})
	defer close(done)

	hosts := YieldTargets(done, files)

	var wg sync.WaitGroup
	wg.Add(Config.NumThreads)

	for i := 0; i < Config.NumThreads; i++ {
		go func() {
			DigestTarget(done, hosts, servers)
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(servers)
	}()

	for server := range servers {
		out <- server
	}
}

func License() {
	fmt.Printf(LICENSE)
}

func Help(out io.Writer) {
	fmt.Fprintf(out, "Usage: %s [opts] hostfile[s]\n\n", os.Args[0])
	fmt.Fprintf(out, "Flags for %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var (
		o_threads = flag.Int("threads", 20, "Specify the number of threads to use when scanning the input files.")

		o_license = flag.Bool("license", false, "Show the license text.")
		o_help    = flag.Bool("help", false, "Show this help page.")
	)

	flag.Usage = func() {
		Help(os.Stderr)
	}

	flag.Parse()

	/* set configs */
	Config.NumThreads = *o_threads

	if *o_license {
		License()
		os.Exit(0)
	}

	if *o_help {
		Help(os.Stdout)
		os.Exit(0)
	}

	if len(flag.Args()) < 1 {
		Help(os.Stderr)
		os.Exit(2)
	}

	servers := make(chan Server)
	go DigestAll(flag.Args(), servers)

	for server := range servers {
		if server.Vulnerable {
			fmt.Printf("V: %s\n", server.Host)
		} else {
			fmt.Printf("N: %s\n", server.Host)
		}
	}
}
<commit_msg>heartthreader: fixed blocking at end of testing<commit_after>/* heartthreader: mass, multithreaded testing for servers against Heartbleed (CVE-2014-0160)
 * Copyright (C) 2014, Cyphar All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"os"
	"io"
	"bufio"
	"strings"
	"sync"
	"fmt"
	"flag"
	"time"

	tls "github.com/titanous/heartbleeder/tls"
)

type Server struct {
	Host string
	Vulnerable bool
}

const LICENSE = "heartthreader: mass, multithreaded testing for servers against Heartbleed (CVE-2014-0160)\n" +
	"Copyright (C) 2014, Cyphar All rights reserved.\n" +
	"\n" +
	"Redistribution and use in source and binary forms, with or without\n" +
	"modification, are permitted provided that the following conditions are met:\n" +
	"\n" +
	"1. Redistributions of source code must retain the above copyright notice,\n" +
	"   this list of conditions and the following disclaimer.\n" +
	"\n" +
	"2. Redistributions in binary form must reproduce the above copyright notice,\n" +
	"   this list of conditions and the following disclaimer in the documentation\n" +
	"   and/or other materials provided with the distribution.\n" +
	"\n" +
	"3. Neither the name of the copyright holder nor the names of its contributors\n" +
	"   may be used to endorse or promote products derived from this software without\n" +
	"   specific prior written permission.\n" +
	"\n" +
	"THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n" +
	"AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n" +
	"IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n" +
	"DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\n" +
	"FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n" +
	"DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n" +
	"SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n" +
	"CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n" +
	"OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\n" +
	"USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n"

var Config struct {
	NumThreads int
}

func FixLine(line string) string {
	if strings.HasSuffix(line, "\n") {
		l := len(line)
		line = line[:l-1]
	}

	if !strings.Contains(line, ":") {
		line = line + ":443"
	}

	return line
}

func YieldTargets(done <-chan struct{}, files []string) <-chan string {
	hosts := make(chan string)

	go func() {
		defer close(hosts)
		for _, file := range files {
			f, err := os.Open(file)
			defer f.Close()

			if err != nil {
				fmt.Fprintf(os.Stderr, "E: %s\n", err.Error())
				continue
			}

			bf := bufio.NewReader(f)

			var line string
			for line, err = bf.ReadString('\n'); err == nil; line, err = bf.ReadString('\n') {
				host := FixLine(line)
				select {
					case hosts <- host:
					case <-done:
						fmt.Fprintf(os.Stderr, "I: yielding canceled\n")
						return
				}
			}

			f.Close()
		}
	}()

	return hosts
}

func DigestTarget(done <-chan struct{}, hosts <-chan string, c chan<- Server) {
	for host := range hosts {
		wait_err := func(conn *tls.Conn, ch chan<- error) {
			var err error

			err = conn.WriteHeartbeat(32, nil)
			_, _, err = conn.ReadHeartbeat()

			ch <- err
		}

		conn, err := tls.Dial("tcp", host, &tls.Config{InsecureSkipVerify: true})
		defer conn.Close()

		if err != nil {
			fmt.Fprintf(os.Stderr, "E: %s\n", err.Error())
			continue
		}

		quit := time.After(10 * time.Second)
		ech := make(chan error)

		go wait_err(conn, ech)

		srv := Server{host, false}

		select {
			case err = <-ech:
				switch err {
					case nil:
						// VULNERABLE!!
						srv.Vulnerable = true
					case tls.ErrNoHeartbeat:
					default:
				}
			case <-quit:
			case <-done:
				fmt.Fprintf(os.Stderr, "I: digest of %s canceled\n", host)
				return
		}

		select {
			case c <- srv:
			case <-done:
				fmt.Fprintf(os.Stderr, "I: digest of %s canceled\n", host)
				return
		}

		conn.Close()
	}
}

func DigestAll(files []string, out chan<- Server) {
	defer close(out)

	servers := make(chan Server)
	done := make(chan struct{})
	defer close(done)

	hosts := YieldTargets(done, files)

	var wg sync.WaitGroup
	wg.Add(Config.NumThreads)

	for i := 0; i < Config.NumThreads; i++ {
		go func() {
			DigestTarget(done, hosts, servers)
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(servers)
	}()

	for server := range servers {
		out <- server
	}
}

func License() {
	fmt.Printf(LICENSE)
}

func Help(out io.Writer) {
	fmt.Fprintf(out, "Usage: %s [opts] hostfile[s]\n\n", os.Args[0])
	fmt.Fprintf(out, "Flags for %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var (
		o_threads = flag.Int("threads", 20, "Specify the number of threads to use when scanning the input files.")

		o_license = flag.Bool("license", false, "Show the license text.")
		o_help    = flag.Bool("help", false, "Show this help page.")
	)

	flag.Usage = func() {
		Help(os.Stderr)
	}

	flag.Parse()

	/* set configs */
	Config.NumThreads = *o_threads

	if *o_license {
		License()
		os.Exit(0)
	}

	if *o_help {
		Help(os.Stdout)
		os.Exit(0)
	}

	if len(flag.Args()) < 1 {
		Help(os.Stderr)
		os.Exit(2)
	}

	servers := make(chan Server)
	go DigestAll(flag.Args(), servers)

	for server := range servers {
		if server.Vulnerable {
			fmt.Printf("V: %s\n", server.Host)
		} else {
			fmt.Printf("N: %s\n", server.Host)
		}
	}
}
<|endoftext|>


<commit_before>package model

import (
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/errors"
)

// TODO: Custom type
const (
	EmailTypeNewLessonNotifier = "new_lesson_notifier"
	EmailTypeFollowReminder    = "follow_reminder"
)

type EventLogEmail struct {
	Datetime   time.Time
	Event      string
	EmailType  string
	UserID     uint32
	UserAgent  string
	TeacherIDs string
	URL        string
}

func (*EventLogEmail) TableName() string {
	return "event_log_email"
}

type EventLogEmailService struct {
	db *gorm.DB
}

func NewEventLogEmailService(db *gorm.DB) *EventLogEmailService {
	return &EventLogEmailService{db}
}

func (s *EventLogEmailService) Create(e *EventLogEmail) error {
	return s.db.Create(e).Error
}

func (s *EventLogEmailService) FindStatsNewLessonNotifierByDate(date time.Time) ([]*StatNewLessonNotifier, error) {
	sql := `
SELECT CAST(datetime AS DATE) AS date, event, COUNT(*) AS count
FROM event_log_email
WHERE
  datetime BETWEEN ? AND ?
  AND email_type = 'new_lesson_notifier'
GROUP BY event;
`
	d := date.Format("2006-01-02")
	values := make([]*StatNewLessonNotifier, 0, 100)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatsNewLessonNotifierUUCountByDate(date time.Time) ([]*StatNewLessonNotifier, error) {
	sql := `
SELECT s.date, s.event, COUNT(*) AS uu_count
FROM (
  SELECT CAST(datetime AS DATE) AS date, event, user_id, COUNT(*) AS count
  FROM event_log_email
  WHERE
    datetime BETWEEN ? AND ?
    AND email_type = 'new_lesson_notifier'
  GROUP BY event, user_id
) AS s
GROUP BY s.event
`
	d := date.Format("2006-01-02")
	values := make([]*StatNewLessonNotifier, 0, 100)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatDailyNotificationEventByDate(date time.Time) ([]*StatDailyNotificationEvent, error) {
	sql := `
SELECT CAST(datetime AS DATE) AS date, event, COUNT(*) AS count
FROM event_log_email
WHERE
  datetime BETWEEN ? AND ?
  AND email_type = 'new_lesson_notifier'
GROUP BY event;
`
	d := date.Format("2006-01-02")
	values := make([]*StatDailyNotificationEvent, 0, 100)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatDailyNotificationEventUUCountByDate(date time.Time) ([]*StatDailyNotificationEvent, error) {
	sql := `
SELECT s.date, s.event, COUNT(*) AS uu_count
FROM (
  SELECT CAST(datetime AS DATE) AS date, event, user_id, COUNT(*) AS count
  FROM event_log_email
  WHERE
    datetime BETWEEN ? AND ?
    AND email_type = 'new_lesson_notifier'
  GROUP BY event, user_id
) AS s
GROUP BY s.event
`
	d := date.Format("2006-01-02")
	values := make([]*StatDailyNotificationEvent, 0, 1000)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatDailyUserNotificationEvent(date time.Time) ([]*StatDailyUserNotificationEvent, error) {
	return nil, nil
}
<commit_msg>Fix CI failure<commit_after>package model

import (
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/errors"
)

// TODO: Custom type
const (
	EmailTypeNewLessonNotifier = "new_lesson_notifier"
	EmailTypeFollowReminder    = "follow_reminder"
)

type EventLogEmail struct {
	Datetime   time.Time
	Event      string
	EmailType  string
	UserID     uint32
	UserAgent  string
	TeacherIDs string
	URL        string
}

func (*EventLogEmail) TableName() string {
	return "event_log_email"
}

type EventLogEmailService struct {
	db *gorm.DB
}

func NewEventLogEmailService(db *gorm.DB) *EventLogEmailService {
	return &EventLogEmailService{db}
}

func (s *EventLogEmailService) Create(e *EventLogEmail) error {
	return s.db.Create(e).Error
}

func (s *EventLogEmailService) FindStatsNewLessonNotifierByDate(date time.Time) ([]*StatNewLessonNotifier, error) {
	sql := `
SELECT CAST(datetime AS DATE) AS date, event, COUNT(*) AS count
FROM event_log_email
WHERE
  datetime BETWEEN ? AND ?
  AND email_type = 'new_lesson_notifier'
GROUP BY event;
`
	d := date.Format("2006-01-02")
	values := make([]*StatNewLessonNotifier, 0, 100)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatsNewLessonNotifierUUCountByDate(date time.Time) ([]*StatNewLessonNotifier, error) {
	sql := `
SELECT s.date, s.event, COUNT(*) AS uu_count
FROM (
  SELECT CAST(datetime AS DATE) AS date, event, user_id, COUNT(*) AS count
  FROM event_log_email
  WHERE
    datetime BETWEEN ? AND ?
    AND email_type = 'new_lesson_notifier'
  GROUP BY event, user_id
) AS s
GROUP BY s.event
`
	d := date.Format("2006-01-02")
	values := make([]*StatNewLessonNotifier, 0, 100)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatDailyNotificationEventByDate(date time.Time) ([]*StatDailyNotificationEvent, error) {
	sql := `
SELECT CAST(datetime AS DATE) AS date, event, COUNT(*) AS count
FROM event_log_email
WHERE
  datetime BETWEEN ? AND ?
  AND email_type = 'new_lesson_notifier'
GROUP BY date, event;
`
	d := date.Format("2006-01-02")
	values := make([]*StatDailyNotificationEvent, 0, 100)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatDailyNotificationEventUUCountByDate(date time.Time) ([]*StatDailyNotificationEvent, error) {
	sql := `
SELECT s.date, s.event, COUNT(*) AS uu_count
FROM (
  SELECT CAST(datetime AS DATE) AS date, event, user_id, COUNT(*) AS count
  FROM event_log_email
  WHERE
    datetime BETWEEN ? AND ?
    AND email_type = 'new_lesson_notifier'
  GROUP BY date, event, user_id
) AS s
GROUP BY s.date, s.event
`
	d := date.Format("2006-01-02")
	values := make([]*StatDailyNotificationEvent, 0, 1000)
	if err := s.db.Raw(strings.TrimSpace(sql), d+" 00:00:00", d+" 23:59:59").Scan(&values).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessagef("select failed"),
			errors.WithResource(errors.NewResource("event_log_email", "date", date.Format("2006-01-02"))),
		)
	}
	return values, nil
}

func (s *EventLogEmailService) FindStatDailyUserNotificationEvent(date time.Time) ([]*StatDailyUserNotificationEvent, error) {
	return nil, nil
}
<|endoftext|>


<commit_before>package cloudca

import (
	"encoding/json"
	"github.com/cloud-ca/go-cloudca/api"
	"github.com/cloud-ca/go-cloudca/services"
)

type NetworkAcl struct {
	Id   string `json:"id,omitempty"`
	Name string `json:"name,omitempty"`
}

type NetworkAclService interface {
	Get(id string) (*NetworkAcl, error)
	List() ([]NetworkAcl, error)
	ListByVpcId(vpcId string) ([]NetworkAcl, error)
	ListWithOptions(options map[string]string) ([]NetworkAcl, error)
}

type NetworkAclApi struct {
	entityService services.EntityService
}

func NewNetworkAclService(apiClient api.ApiClient, serviceCode string, environmentName string) NetworkAclService {
	return &NetworkAclApi{
		entityService: services.NewEntityService(apiClient, serviceCode, environmentName, NETWORK_ACL_ENTITY_TYPE),
	}
}

func parseNetworkAcl(data []byte) *NetworkAcl {
	networkAcl := NetworkAcl{}
	json.Unmarshal(data, &networkAcl)
	return &networkAcl
}

func parseNetworkAclList(data []byte) []NetworkAcl {
	networkAcls := []NetworkAcl{}
	json.Unmarshal(data, &networkAcls)
	return networkAcls
}

//Get network acl with the specified id for the current environment
func (networkAclApi *NetworkAclApi) Get(id string) (*NetworkAcl, error) {
	data, err := networkAclApi.entityService.Get(id, map[string]string{})
	if err != nil {
		return nil, err
	}
	return parseNetworkAcl(data), nil
}

//List all network offerings for the current environment
func (networkAclApi *NetworkAclApi) List() ([]NetworkAcl, error) {
	return networkAclApi.ListWithOptions(map[string]string{})
}

//List all network offerings for the current environment
func (networkAclApi *NetworkAclApi) ListByVpcId(vpcId string) ([]NetworkAcl, error) {
	return networkAclApi.ListWithOptions(map[string]string{"vpc_id": vpcId})
}

//List all network offerings for the current environment. Can use options to do sorting and paging.
func (networkAclApi *NetworkAclApi) ListWithOptions(options map[string]string) ([]NetworkAcl, error) {
	data, err := networkAclApi.entityService.List(options)
	if err != nil {
		return nil, err
	}
	return parseNetworkAclList(data), nil
}
<commit_msg>Added network ACL create and delete operations<commit_after>package cloudca

import (
	"encoding/json"
	"github.com/cloud-ca/go-cloudca/api"
	"github.com/cloud-ca/go-cloudca/services"
)

type NetworkAcl struct {
	Id   string `json:"id,omitempty"`
	Name string `json:"name,omitempty"`
	Description string `json:"description,omitempty"`
	VpcId string `json:"vpcId,omitempty"`
}

type NetworkAclService interface {
	Get(id string) (*NetworkAcl, error)
	List() ([]NetworkAcl, error)
	ListByVpcId(vpcId string) ([]NetworkAcl, error)
	ListWithOptions(options map[string]string) ([]NetworkAcl, error)
}

type NetworkAclApi struct {
	entityService services.EntityService
}

func NewNetworkAclService(apiClient api.ApiClient, serviceCode string, environmentName string) NetworkAclService {
	return &NetworkAclApi{
		entityService: services.NewEntityService(apiClient, serviceCode, environmentName, NETWORK_ACL_ENTITY_TYPE),
	}
}

func parseNetworkAcl(data []byte) *NetworkAcl {
	networkAcl := NetworkAcl{}
	json.Unmarshal(data, &networkAcl)
	return &networkAcl
}

func parseNetworkAclList(data []byte) []NetworkAcl {
	networkAcls := []NetworkAcl{}
	json.Unmarshal(data, &networkAcls)
	return networkAcls
}

//Get network acl with the specified id for the current environment
func (networkAclApi *NetworkAclApi) Get(id string) (*NetworkAcl, error) {
	data, err := networkAclApi.entityService.Get(id, map[string]string{})
	if err != nil {
		return nil, err
	}
	return parseNetworkAcl(data), nil
}

//List all network offerings for the current environment
func (networkAclApi *NetworkAclApi) List() ([]NetworkAcl, error) {
	return networkAclApi.ListWithOptions(map[string]string{})
}

//List all network offerings for the current environment
func (networkAclApi *NetworkAclApi) ListByVpcId(vpcId string) ([]NetworkAcl, error) {
	return networkAclApi.ListWithOptions(map[string]string{"vpc_id": vpcId})
}

//List all network offerings for the current environment. Can use options to do sorting and paging.
func (networkAclApi *NetworkAclApi) ListWithOptions(options map[string]string) ([]NetworkAcl, error) {
	data, err := networkAclApi.entityService.List(options)
	if err != nil {
		return nil, err
	}
	return parseNetworkAclList(data), nil
}

func (networkAclApi *NetworkAclApi) Create(networkAcl NetworkAcl) (*NetworkAcl, error) {
	msg, err := json.Marshal(networkAcl)
	if err != nil {
		return nil, err
	}
	result, err := networkAclApi.entityService.Create(msg, map[string]string{})
	if err != nil {
		return nil, err
	}
	return parseNetworkAcl(result), nil
}

func (networkAclApi *NetworkAclApi) Delete(id string) (bool, error) {
	_, err := networkAclApi.entityService.Delete(id, []byte{}, map[string]string{})
	return err == nil, err
}
<|endoftext|>


<commit_before>package services

import (
	. "github.com/panyam/relay/services/msg/core"
	. "gopkg.in/check.v1"
)

func (s *TestSuite) TestCreateUserService(c *C) {
	svc := s.serviceGroup.UserService
	c.Assert(svc, Not(Equals), nil)
}

func (s *TestSuite) TestSaveUserEmptyId_ShouldCreateId(c *C) {
	svc := s.serviceGroup.UserService
	team, err := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user := NewUser(0, "user1", team)
	err = svc.SaveUser(user, false)
	c.Assert(err, Equals, nil)
	c.Assert(user.Id, Not(Equals), 0)
	c.Assert(user.Username, Equals, "user1")

	fetched_user, err := svc.GetUserById(user.Id)
	c.Assert(fetched_user.Id, Equals, user.Id)
	c.Assert(fetched_user.Team, Not(Equals), nil)
	c.Assert(fetched_user.Team.Id, Equals, user.Team.Id)

	fetched_user, _ = svc.GetUser("user1", team)
	c.Assert(fetched_user.Id, Equals, user.Id)
	c.Assert(fetched_user.Team, Not(Equals), nil)
	c.Assert(fetched_user.Team.Id, Equals, user.Team.Id)
}

func (s *TestSuite) TestGetUserFirstTime(c *C) {
	svc := s.serviceGroup.UserService
	team, _ := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user, _ := svc.GetUserById(1)
	c.Assert(user, Equals, (*User)(nil))
	_, err := svc.GetUser("user1", team)
	c.Assert(err, Not(Equals), nil)
}

func (s *TestSuite) TestSaveUserNormal(c *C) {
	svc := s.serviceGroup.UserService
	team, _ := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user := NewUser(0, "user1", team)
	user.Object = Object{Id: 1}
	err := svc.SaveUser(user, false)
	c.Assert(err, Equals, nil)
}

func (s *TestSuite) TestCreateUserDuplicate(c *C) {
	svc := s.serviceGroup.UserService
	team, _ := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user := User{Username: "user1", Team: team}

	err := svc.SaveUser(&user, false)
	c.Assert(err, Equals, nil)

	// err = svc.SaveUser(&user, false)
	// c.Assert(err, Not(Equals), nil)
}

/*
func (s *TestSuite) TestSaveUserIdOrNameExists(c *C) {
	svc := s.serviceGroup.UserService
	user, err := svc.CreateUser("1", "user1")
	c.Assert(err, Equals, nil)
	user, err = svc.CreateUser("1", "user2")
	c.Assert(err, Not(Equals), nil)
	c.Assert(user, Equals, (*User)(nil))
	user, err = svc.CreateUser("2", "user1")
	c.Assert(err, Not(Equals), nil)
	c.Assert(user, Equals, (*User)(nil))
}
*/
<commit_msg>Using IsNil and Not(IsNil)<commit_after>package services

import (
	. "github.com/panyam/relay/services/msg/core"
	. "gopkg.in/check.v1"
)

func (s *TestSuite) TestCreateUserService(c *C) {
	svc := s.serviceGroup.UserService
	c.Assert(svc, Not(IsNil))
}

func (s *TestSuite) TestSaveUserEmptyId_ShouldCreateId(c *C) {
	svc := s.serviceGroup.UserService
	team, err := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user := NewUser(0, "user1", team)
	err = svc.SaveUser(user, false)
	c.Assert(err, IsNil)
	c.Assert(user.Id, Not(Equals), 0)
	c.Assert(user.Username, Equals, "user1")

	fetched_user, err := svc.GetUserById(user.Id)
	c.Assert(fetched_user.Id, Equals, user.Id)
	c.Assert(fetched_user.Team, Not(IsNil))
	c.Assert(fetched_user.Team.Id, Equals, user.Team.Id)

	fetched_user, _ = svc.GetUser("user1", team)
	c.Assert(fetched_user.Id, Equals, user.Id)
	c.Assert(fetched_user.Team, Not(IsNil))
	c.Assert(fetched_user.Team.Id, Equals, user.Team.Id)
}

func (s *TestSuite) TestGetUserFirstTime(c *C) {
	svc := s.serviceGroup.UserService
	team, _ := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user, _ := svc.GetUserById(1)
	c.Assert(user, IsNil)
	_, err := svc.GetUser("user1", team)
	c.Assert(err, Not(IsNil))
}

func (s *TestSuite) TestSaveUserNormal(c *C) {
	svc := s.serviceGroup.UserService
	team, _ := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user := NewUser(0, "user1", team)
	user.Object = Object{Id: 1}
	err := svc.SaveUser(user, false)
	c.Assert(err, IsNil)
}

func (s *TestSuite) TestCreateUserDuplicate(c *C) {
	svc := s.serviceGroup.UserService
	team, _ := s.serviceGroup.TeamService.CreateTeam(1, "org", "team")
	user := User{Username: "user1", Team: team}

	err := svc.SaveUser(&user, false)
	c.Assert(err, IsNil)

	// err = svc.SaveUser(&user, false)
	// c.Assert(err, Not(IsNil))
}

/*
func (s *TestSuite) TestSaveUserIdOrNameExists(c *C) {
	svc := s.serviceGroup.UserService
	user, err := svc.CreateUser("1", "user1")
	c.Assert(err, IsNil)
	user, err = svc.CreateUser("1", "user2")
	c.Assert(err, Not(IsNil))
	c.Assert(user, IsNil)
	user, err = svc.CreateUser("2", "user1")
	c.Assert(err, Not(IsNil))
	c.Assert(user, IsNil)
}
*/
<|endoftext|>


<commit_before>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"github.com/slyrz/newscat/util"
	"io"
	"regexp"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

var (
	ignorePattern = regexp.MustCompile("(?i)comment|community|gallery|caption|description|credit|foot|story-feature|related")
)

type Document struct {
	Title  *util.Text // the <title>...</title> text.
	Chunks []*Chunk   // list of all chunks found in this document

	// Unexported fields.
	root *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields root, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.root == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// If no title was found, detecting the main heading might fail.
	if doc.Title == nil {
		doc.Title = util.NewText()
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields root, head and body from the HTML tree of node n.
//  doc.root ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.root = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk.Text
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {

	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		switch c.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			return true
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			return level > 5
		}
		return false
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here becase calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignorePattern.FindStringIndex(attr.Val) != nil {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<commit_msg>ignore hide and hidden classes<commit_after>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"github.com/slyrz/newscat/util"
	"io"
	"regexp"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

var (
	ignorePattern = regexp.MustCompile("(?i)comment|community|gallery|caption|description|credit|foot|story-feature|related|hide|hidden")
)

type Document struct {
	Title  *util.Text // the <title>...</title> text.
	Chunks []*Chunk   // list of all chunks found in this document

	// Unexported fields.
	root *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields root, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.root == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// If no title was found, detecting the main heading might fail.
	if doc.Title == nil {
		doc.Title = util.NewText()
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields root, head and body from the HTML tree of node n.
//  doc.root ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.root = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk.Text
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {

	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		switch c.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			return true
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			return level > 5
		}
		return false
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here becase calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignorePattern.FindStringIndex(attr.Val) != nil {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Mitchell Cooper
package main

import (
	wikiclient "github.com/cooper/go-wikiclient"
	"net/http"
	"regexp"
)

var imageRegex = regexp.MustCompile("")

// master handler
func handleRoot(w http.ResponseWriter, r *http.Request) {
	var delayedWiki wikiInfo

	// try each wiki
	for _, wiki := range wikis {

		// wrong root
		wikiRoot := wiki.conf.Get("root.wiki")
		if r.URL.Path != wikiRoot && r.URL.Path != wikiRoot+"/" {
			continue
		}

		// wrong host
		if wiki.host != r.Host {

			// if the wiki host is empty, it is the fallback wiki.
			// delay it until we've checked all other wikis.
			if wiki.host == "" && delayedWiki.name == "" {
				delayedWiki = wiki
			}

			continue
		}

		// host matches
		if handleMainPage(wiki, w, r) {
			return
		}
	}

	// try the delayed wiki
	if delayedWiki.name != "" {
		if handleMainPage(delayedWiki, w, r) {
			return
		}
	}

	// anything else is a 404
	http.NotFound(w, r)
}

func handleMainPage(wiki wikiInfo, w http.ResponseWriter, r *http.Request) bool {

	// no main page configured
	mainPage := wiki.conf.Get("main_page")
	if mainPage == "" {
		return false
	}

	http.Redirect(w, r, wiki.conf.Get("root.page")+"/"+mainPage, http.StatusMovedPermanently)
	return true
}

// page request
func handlePage(wiki wikiInfo, relPath string, w http.ResponseWriter, r *http.Request) {
	res, err := wiki.client.DisplayPage(relPath)

	// wikiclient error or 'not found' response
	if handleError(wiki, err, w, r) || handleError(wiki, res, w, r) {
		return
	}

	// other response
	switch res.Get("type") {

	// page redirect
	case "redirect":
		http.Redirect(w, r, wiki.conf.Get("root.page")+"/"+res.Get("name"), http.StatusMovedPermanently)

	// page content
	case "page":
		renderTemplate(wiki, w, "page", wikiPage{
			Res:        res,
			Title:      res.Get("title"),
			WikiTitle:  wiki.title,
			WikiLogo:   wiki.template.logo,
			WikiRoot:   wiki.conf.Get("root.wiki"),
			StaticRoot: wiki.template.staticRoot,
			navigation: wiki.conf.GetSlice("navigation"),
		})
	}

	http.NotFound(w, r)
}

// image request
func handleImage(wiki wikiInfo, relPath string, w http.ResponseWriter, r *http.Request) {
	res, err := wiki.client.DisplayImage(relPath, 0, 0)
	if handleError(wiki, err, w, r) || handleError(wiki, res, w, r) {
		return
	}
	http.ServeFile(w, r, res.Get("path"))
}

// this is set true when calling handlePage for the error page. this way, if an
// error occurs when trying to display the error page, we don't infinitely loop
// between handleError and handlePage
var useLowLevelError bool

func handleError(wiki wikiInfo, errMaybe interface{}, w http.ResponseWriter, r *http.Request) bool {
	msg := "An unknown error has occurred"
	switch err := errMaybe.(type) {

	// if there's no error, stop
	case nil:
		return false

	// message of type "not found" is an error; otherwise, stop
	case wikiclient.Message:
		if err.Get("type") != "not found" {
			return false
		}
		msg = err.Get("error")

		// string
	case string:
		msg = err

		// error
	case error:
		msg = err.Error()

	}

	// if we have an error page, use it
	errorPage := wiki.conf.Get("error_page")
	if !useLowLevelError && errorPage != "" {
		useLowLevelError = true
		handlePage(wiki, errorPage, w, r)
		useLowLevelError = false
		return true
	}

	// generic error response
	http.Error(w, msg, http.StatusNotFound)
	return true
}

func renderTemplate(wiki wikiInfo, w http.ResponseWriter, templateName string, p wikiPage) {
	err := wiki.template.template.ExecuteTemplate(w, templateName+".tpl", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
<commit_msg>simplify main page checks<commit_after>// Copyright (c) 2017, Mitchell Cooper
package main

import (
	wikiclient "github.com/cooper/go-wikiclient"
	"net/http"
	"regexp"
)

var imageRegex = regexp.MustCompile("")

// master handler
func handleRoot(w http.ResponseWriter, r *http.Request) {
	var delayedWiki wikiInfo

	// try each wiki
	for _, wiki := range wikis {

		// wrong root
		wikiRoot := wiki.conf.Get("root.wiki")
		if r.URL.Path != wikiRoot && r.URL.Path != wikiRoot+"/" {
			continue
		}

		// no main page configured
		mainPage := wiki.conf.Get("main_page")
		if mainPage == "" {
			continue
		}

		// wrong host
		if wiki.host != r.Host {

			// if the wiki host is empty, it is the fallback wiki.
			// delay it until we've checked all other wikis.
			if wiki.host == "" && delayedWiki.name == "" {
				delayedWiki = wiki
			}

			continue
		}

		// host matches
		delayedWiki = wiki
		break
	}

	// try the delayed wiki
	if delayedWiki.name != "" {
		http.Redirect(
			w, r,
			delayedWiki.conf.Get("root.page")+
				"/"+delayedWiki.conf.Get("main_page"),
			http.StatusMovedPermanently,
		)
		return
	}

	// anything else is a 404
	http.NotFound(w, r)
}

// page request
func handlePage(wiki wikiInfo, relPath string, w http.ResponseWriter, r *http.Request) {
	res, err := wiki.client.DisplayPage(relPath)

	// wikiclient error or 'not found' response
	if handleError(wiki, err, w, r) || handleError(wiki, res, w, r) {
		return
	}

	// other response
	switch res.Get("type") {

	// page redirect
	case "redirect":
		http.Redirect(w, r, wiki.conf.Get("root.page")+"/"+res.Get("name"), http.StatusMovedPermanently)

	// page content
	case "page":
		renderTemplate(wiki, w, "page", wikiPage{
			Res:        res,
			Title:      res.Get("title"),
			WikiTitle:  wiki.title,
			WikiLogo:   wiki.template.logo,
			WikiRoot:   wiki.conf.Get("root.wiki"),
			StaticRoot: wiki.template.staticRoot,
			navigation: wiki.conf.GetSlice("navigation"),
		})
	}

	http.NotFound(w, r)
}

// image request
func handleImage(wiki wikiInfo, relPath string, w http.ResponseWriter, r *http.Request) {
	res, err := wiki.client.DisplayImage(relPath, 0, 0)
	if handleError(wiki, err, w, r) || handleError(wiki, res, w, r) {
		return
	}
	http.ServeFile(w, r, res.Get("path"))
}

// this is set true when calling handlePage for the error page. this way, if an
// error occurs when trying to display the error page, we don't infinitely loop
// between handleError and handlePage
var useLowLevelError bool

func handleError(wiki wikiInfo, errMaybe interface{}, w http.ResponseWriter, r *http.Request) bool {
	msg := "An unknown error has occurred"
	switch err := errMaybe.(type) {

	// if there's no error, stop
	case nil:
		return false

	// message of type "not found" is an error; otherwise, stop
	case wikiclient.Message:
		if err.Get("type") != "not found" {
			return false
		}
		msg = err.Get("error")

		// string
	case string:
		msg = err

		// error
	case error:
		msg = err.Error()

	}

	// if we have an error page, use it
	errorPage := wiki.conf.Get("error_page")
	if !useLowLevelError && errorPage != "" {
		useLowLevelError = true
		handlePage(wiki, errorPage, w, r)
		useLowLevelError = false
		return true
	}

	// generic error response
	http.Error(w, msg, http.StatusNotFound)
	return true
}

func renderTemplate(wiki wikiInfo, w http.ResponseWriter, templateName string, p wikiPage) {
	err := wiki.template.template.ExecuteTemplate(w, templateName+".tpl", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"text/template"
)

var (
	TemplateSingleColumnTable = `

		{{range .}}

					{{.message}}

		{{end}}

`

	TemplateDNATable = `

		{{range .}}

					{{.name}}

					{{.dna}}

		{{end}}

`

	TemplateKawaiiTable = `

		{{range .}}

					{{.name}}

					{{.face}}

		{{end}}

`
	TemplateSkeleton = `
<!doctype html>

		Guest:

Kawaiiface

DNA

Joke

Rape

Kill

Factoid

Yomama

Gelbooru Random

Files

Login

		Admin:

File Upload

CreateUser

Plugin Manager

		{{template "content" .}}

`

	TemplatePluginlist = `

 Plugin hub

			File Name

			Functions
			Actions

		{{range .}}

			{{.name}}
			{{if .file}}File Exists{{end}} {{if .loaded}}Loaded{{end}}
			{{range .functions}}{{.}}, {{end}}

				{{if .file}}Edit
				Delete
				(Re-)load{{end}}
				{{if .loaded}}
				Unload
				{{end}}

		{{end}}

Create new:

<form action="/plugins/edit" method="GET">
	<input type="text" name="name" />

	<input type="submit" value="Create" />
</form>
`

	TemplatePluginedit = `

 Edit plugin {{.name}}

<form action="/plugins/edit" method="POST" onsubmit="document.getElementById('contentT').value = ace.edit('content').getSession().getValue();">
	<input type="hidden" value="{{.name}}" name="name" />
	<textarea cols="100" rows="40" name="content" id="contentT">{{.content}}</textarea>

	Save: <input type="text" name="filename" value="{{.name}}" /><input type="submit" value="Submit" />
</form>

 API:

Defining a function:

		use
 Subscribe(string trigger, function(sender, source, line, params) handler)
 to define a function.

		The following trigger types are defined:

!command - These are your bog standard irc commands

privmsg - This will trigger on all messages received

join - Triggers if someone joins the channel

part - Triggers when someone leaves the channel

quit - Triggers when someone disconnects from irc

regex: - Type a regex on which it will trigger after the colon (:)

		The handler function has the following parameters:

sender - This is the nick name of the sender

source - This is either the nick name of the sender on a direct message or the channel the sender posted in

line - Either the complete message or the part after the !command

params - In !commands this is a array of space delimited parameters. Parameters with a space in it can be consolidated with quotes.

!command param param21 "pa sdf asdfj dflk"

				is THREE parameters.

param

param21
and

pa sdf asdfj dflk

				Actual quotes can be escaped using \. Matching quotes will otherwise be removed.

Function definitions

IRC

IRC.Privmsg(destination, message) - sends an IRC message to a channel or a nick

IRC.Action(channel, message) - sends an "action" to a IRC channel

IRC.Notice(nick, message) - sends a Notice towards a user

IRC.Channel - Contains the main channel of the bot

IRC.Server - Contains the current server name

IRC.Nick - Contains the nick of the bot

				DB

DB.Authenticate(user, pwd) - Authenticates against the user database

DB.SaveToDb

DB.ExistsInDB

DB.GetRandomFromDB

DB.GetAndDeleteFirstFromDB

DB.GetNamedFromDB

DB.SaveNamedToDB

`

	TemplateLogin = `
	{{if .}} Error: {{.}} {{end}}
	<form method="POST">
		User: <input type="text" name="username" />

		Password: <input type="password" name="password" />

		<input type="submit" name="submit" value="Login" />
	</form>
`

	TemplateCreateUser = `
	<form method="POST" action="/CreateUser">
		Username: <input type="text" name="username" />

		Password: <input type="text" name="password" />

		<input type="submit" name="submit" value="Create" />
	</form>
`

	TemplateFiles = `
`

	TemplateFileUpload = `

`
)

func Template(subtmpl string) *template.Template {
	templ := template.New("main")
	templ.Parse(TemplateSkeleton)
	contenttmpl := templ.New("content")
	if subtmpl != "" {
		contenttmpl.Parse(subtmpl)
	}
	return templ
}
<commit_msg>Remove useless http handlers; template now.<commit_after>package main

import (
	"text/template"
)

var (
	TemplateSingleColumnTable = `

		{{range .}}

					{{.message}}

		{{end}}

`

	TemplateDNATable = `

		{{range .}}

					{{.name}}

					{{.dna}}

		{{end}}

`

	TemplateKawaiiTable = `

		{{range .}}

					{{.name}}

					{{.face}}

		{{end}}

`
	TemplateSkeleton = `
<!doctype html>

		Guest:

Files

Login

		Admin:

File Upload

CreateUser

Plugin Manager

		{{template "content" .}}

`

	TemplatePluginlist = `

 Plugin hub

			File Name

			Functions
			Actions

		{{range .}}

			{{.name}}
			{{if .file}}File Exists{{end}} {{if .loaded}}Loaded{{end}}
			{{range .functions}}{{.}}, {{end}}

				{{if .file}}Edit
				Delete
				(Re-)load{{end}}
				{{if .loaded}}
				Unload
				{{end}}

		{{end}}

Create new:

<form action="/plugins/edit" method="GET">
	<input type="text" name="name" />

	<input type="submit" value="Create" />
</form>
`

	TemplatePluginedit = `

 Edit plugin {{.name}}

<form action="/plugins/edit" method="POST" onsubmit="document.getElementById('contentT').value = ace.edit('content').getSession().getValue();">
	<input type="hidden" value="{{.name}}" name="name" />
	<textarea cols="100" rows="40" name="content" id="contentT">{{.content}}</textarea>

	Save: <input type="text" name="filename" value="{{.name}}" /><input type="submit" value="Submit" />
</form>

 API:

Defining a function:

		use
 Subscribe(string trigger, function(sender, source, line, params) handler)
 to define a function.

		The following trigger types are defined:

!command - These are your bog standard irc commands

privmsg - This will trigger on all messages received

join - Triggers if someone joins the channel

part - Triggers when someone leaves the channel

quit - Triggers when someone disconnects from irc

regex: - Type a regex on which it will trigger after the colon (:)

		The handler function has the following parameters:

sender - This is the nick name of the sender

source - This is either the nick name of the sender on a direct message or the channel the sender posted in

line - Either the complete message or the part after the !command

params - In !commands this is a array of space delimited parameters. Parameters with a space in it can be consolidated with quotes.

!command param param21 "pa sdf asdfj dflk"

				is THREE parameters.

param

param21
and

pa sdf asdfj dflk

				Actual quotes can be escaped using \. Matching quotes will otherwise be removed.

Function definitions

IRC

IRC.Privmsg(destination, message) - sends an IRC message to a channel or a nick

IRC.Action(channel, message) - sends an "action" to a IRC channel

IRC.Notice(nick, message) - sends a Notice towards a user

IRC.Channel - Contains the main channel of the bot

IRC.Server - Contains the current server name

IRC.Nick - Contains the nick of the bot

				DB

DB.Authenticate(user, pwd) - Authenticates against the user database

DB.SaveToDb

DB.ExistsInDB

DB.GetRandomFromDB

DB.GetAndDeleteFirstFromDB

DB.GetNamedFromDB

DB.SaveNamedToDB

`

	TemplateLogin = `
	{{if .}} Error: {{.}} {{end}}
	<form method="POST">
		User: <input type="text" name="username" />

		Password: <input type="password" name="password" />

		<input type="submit" name="submit" value="Login" />
	</form>
`

	TemplateCreateUser = `
	<form method="POST" action="/CreateUser">
		Username: <input type="text" name="username" />

		Password: <input type="text" name="password" />

		<input type="submit" name="submit" value="Create" />
	</form>
`

	TemplateFiles = `
`

	TemplateFileUpload = `

`
)

func Template(subtmpl string) *template.Template {
	templ := template.New("main")
	templ.Parse(TemplateSkeleton)
	contenttmpl := templ.New("content")
	if subtmpl != "" {
		contenttmpl.Parse(subtmpl)
	}
	return templ
}
<|endoftext|>


<commit_before>package hwaflib

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
)

type VcsPackage struct {
	Type string // type of remote VCS (svn, git, ...)
	Repo string // remote repository URL
	Path string // path under which the package is locally checked out
}

type PackageDb struct {
	db    map[string]VcsPackage
	fname string // file name where the db is backed up
}

func NewPackageDb(fname string) *PackageDb {
	return &PackageDb{
		db:    make(map[string]VcsPackage),
		fname: fname,
	}
}

func (db *PackageDb) Load(fname string) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()
	err = json.NewDecoder(f).Decode(&db.db)
	return err
}

func (db *PackageDb) Add(vcs, pkguri, pkgname string) error {
	_, has := db.db[pkgname]
	if has {
		return fmt.Errorf("hwaf.pkgdb: package [%s] already in db", pkgname)
	}
	db.db[pkgname] = VcsPackage{vcs, pkguri, pkgname}

	err := db.sync()
	if err != nil {
		return err
	}
	// FIXME: should this return an error ?
	exec.Command("git", "add", db.fname).Run()
	exec.Command(
		"git", "commit",
		"-m", fmt.Sprintf("adding package [%s]", pkgname),
	).Run()
	return nil
}

func (db *PackageDb) Remove(pkgname string) error {
	_, ok := db.db[pkgname]
	if !ok {
		return fmt.Errorf("hwaf.pkgdb: package [%s] not in db", pkgname)
	}
	delete(db.db, pkgname)

	err := db.sync()
	if err != nil {
		return err
	}

	// FIXME: should this return an error ?
	exec.Command("git", "add", db.fname).Run()
	exec.Command(
		"git", "commit",
		"-m", fmt.Sprintf("removing package [%s]", pkgname),
	).Run()
	return nil
}

func (db *PackageDb) HasPkg(pkgname string) bool {
	_, ok := db.db[pkgname]
	return ok
}

func (db *PackageDb) GetPkg(pkgname string) (VcsPackage, error) {
	pkg, ok := db.db[pkgname]
	if !ok {
		return pkg, fmt.Errorf("hwaf.pkgdb: package [%s] not in db", pkgname)
	}
	return pkg, nil
}

// Pkgs returns the list of packages this db holds
func (db *PackageDb) Pkgs() []string {
	if db == nil || db.db == nil {
		return []string{}
	}

	pkgs := make([]string, 0, len(db.db))
	for k, _ := range db.db {
		pkgs = append(pkgs, k)
	}
	return pkgs
}

func (db *PackageDb) sync() error {
	if path_exists(db.fname) {
		os.RemoveAll(db.fname)
	}
	f, err := os.Create(db.fname)
	if err != nil {
		return err
	}
	defer f.Close()

	err = json.NewEncoder(f).Encode(&db.db)
	if err != nil {
		return err
	}
	err = f.Sync()
	if err != nil {
		return err
	}
	return f.Close()
}

// EOF
<commit_msg>pkgdb: commit changes to pkgdb.json to git (more explicit log message)<commit_after>package hwaflib

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
)

type VcsPackage struct {
	Type string // type of remote VCS (svn, git, ...)
	Repo string // remote repository URL
	Path string // path under which the package is locally checked out
}

type PackageDb struct {
	db    map[string]VcsPackage
	fname string // file name where the db is backed up
}

func NewPackageDb(fname string) *PackageDb {
	return &PackageDb{
		db:    make(map[string]VcsPackage),
		fname: fname,
	}
}

func (db *PackageDb) Load(fname string) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()
	err = json.NewDecoder(f).Decode(&db.db)
	return err
}

func (db *PackageDb) Add(vcs, pkguri, pkgname string) error {
	_, has := db.db[pkgname]
	if has {
		return fmt.Errorf("hwaf.pkgdb: package [%s] already in db", pkgname)
	}
	db.db[pkgname] = VcsPackage{vcs, pkguri, pkgname}

	err := db.sync()
	if err != nil {
		return err
	}
	// FIXME: should this return an error ?
	exec.Command("git", "add", db.fname).Run()
	exec.Command(
		"git", "commit",
		"-m", fmt.Sprintf("adding package [%s] to %s", pkgname, db.fname),
	).Run()
	return nil
}

func (db *PackageDb) Remove(pkgname string) error {
	_, ok := db.db[pkgname]
	if !ok {
		return fmt.Errorf("hwaf.pkgdb: package [%s] not in db", pkgname)
	}
	delete(db.db, pkgname)

	err := db.sync()
	if err != nil {
		return err
	}

	// FIXME: should this return an error ?
	exec.Command("git", "add", db.fname).Run()
	exec.Command(
		"git", "commit",
		"-m", fmt.Sprintf("removing package [%s] from %s", pkgname, db.fname),
	).Run()
	return nil
}

func (db *PackageDb) HasPkg(pkgname string) bool {
	_, ok := db.db[pkgname]
	return ok
}

func (db *PackageDb) GetPkg(pkgname string) (VcsPackage, error) {
	pkg, ok := db.db[pkgname]
	if !ok {
		return pkg, fmt.Errorf("hwaf.pkgdb: package [%s] not in db", pkgname)
	}
	return pkg, nil
}

// Pkgs returns the list of packages this db holds
func (db *PackageDb) Pkgs() []string {
	if db == nil || db.db == nil {
		return []string{}
	}

	pkgs := make([]string, 0, len(db.db))
	for k, _ := range db.db {
		pkgs = append(pkgs, k)
	}
	return pkgs
}

func (db *PackageDb) sync() error {
	if path_exists(db.fname) {
		os.RemoveAll(db.fname)
	}
	f, err := os.Create(db.fname)
	if err != nil {
		return err
	}
	defer f.Close()

	err = json.NewEncoder(f).Encode(&db.db)
	if err != nil {
		return err
	}
	err = f.Sync()
	if err != nil {
		return err
	}
	return f.Close()
}

// EOF
<|endoftext|>


<commit_before>// FTP工具类
// create by gloomy 2017-4-1 17:32:06
package gutil

import (
	"bytes"
	"fmt"
	"github.com/jlaffaye/ftp"
	"time"
)

// FTP帮助类实体
// create by gloomy 2017-4-1 17:34:16
type FtpHelpStruct struct {
	IpAddr    string        // ip 地址
	Port      int           // 端口
	TimeOut   time.Duration // 超时时间
	UserName  string        // 用户名
	PassWord  string        // 密码
	FilePaths string        // 目标服务器路径
}

const ftpTimeOut = 60 * time.Minute

// FTP文件传输
// create by gloomy 2017-4-1 17:36:11
// FTP配置实体 文件内容 创建目标服务器的文件名
// 错误对象
func FtpFileStor(model *FtpHelpStruct, ftpConntion *ftp.ServerConn, contentByte *[]byte, createFilePath string) error {
	var (
		err error
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return err
		}
	}
	return ftpConntion.Stor(createFilePath, bytes.NewReader(*contentByte))
}

// FTP登录
// create by gloomy 2017-4-1 17:39:59
// 输入参数 FTP配置实体
// 输出参数 FTP连接对象 错误对象
func ftpLogin(model *FtpHelpStruct) (*ftp.ServerConn, error) {
	if model.TimeOut <= 0 {
		model.TimeOut = ftpTimeOut
	}
	c, err := ftp.DialTimeout(fmt.Sprintf("%s:%d", model.IpAddr, model.Port), model.TimeOut)
	if err != nil {
		return nil, err
	}
	err = c.Login(model.UserName, model.PassWord)
	if err != nil {
		return nil, err
	}
	err = c.NoOp()
	if err != nil {
		return nil, err
	}
	return c, err
}

// FTP文件删除
// create by gloomy 2017-04-02 01:08:15
// 文件名 ftp配置对象
// 错误对象
func FtpRemoveFile(filePathStr string, ftpConntion *ftp.ServerConn, model *FtpHelpStruct) error {
	var (
		err error
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return err
		}
	}
	return ftpConntion.Delete(filePathStr)
}

// ftp修正远程服务器文件名称
// create by gloomy 2017-04-04 21:26:48
// 源文件 修正后的文件名称 ftp配置对象
// 错误对象
func FtpRenameFile(from, to string, ftpConntion *ftp.ServerConn, model *FtpHelpStruct) error {
	var (
		err error
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return err
		}
	}
	return ftpConntion.Rename(from, to)
}

// 获取FTP上所有的文件列表
// create by gloomy 2017-04-12 10:43:25
// 目录地址
// 文件列表集 错误对象
func FtpNameList(pathStr string, ftpConntion *ftp.ServerConn, model *FtpHelpStruct) ([]string, error) {
	var (
		err error
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return []string{}, err
		}
	}
	return ftpConntion.NameList(pathStr)
}

// ftp退出
// create by gloomy 2017-4-14 11:25:04
func FtpExit(ftpConntion *ftp.ServerConn) {
	if ftpConntion != nil {
		ftpConntion.Quit()
	}
}
<commit_msg>修正ftp程序运行<commit_after>// FTP工具类
// create by gloomy 2017-4-1 17:32:06
package gutil

import (
	"bytes"
	"fmt"
	"github.com/jlaffaye/ftp"
	"strings"
	"time"
)

// FTP帮助类实体
// create by gloomy 2017-4-1 17:34:16
type FtpHelpStruct struct {
	IpAddr    string        // ip 地址
	Port      int           // 端口
	TimeOut   time.Duration // 超时时间
	UserName  string        // 用户名
	PassWord  string        // 密码
	FilePaths string        // 目标服务器路径
}

const ftpTimeOut = 60 * time.Minute

// FTP文件传输
// create by gloomy 2017-4-1 17:36:11
// FTP配置实体 文件内容 创建目标服务器的文件名
// 错误对象
func FtpFileStor(model *FtpHelpStruct, ftpConntion *ftp.ServerConn, contentByte *[]byte, createFilePath string) error {
	var (
		err error
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return err
		}
	}
	return ftpConntion.Stor(createFilePath, bytes.NewReader(*contentByte))
}

// FTP登录
// create by gloomy 2017-4-1 17:39:59
// 输入参数 FTP配置实体
// 输出参数 FTP连接对象 错误对象
func ftpLogin(model *FtpHelpStruct) (*ftp.ServerConn, error) {
	if model.TimeOut <= 0 {
		model.TimeOut = ftpTimeOut
	}
	c, err := ftp.DialTimeout(fmt.Sprintf("%s:%d", model.IpAddr, model.Port), model.TimeOut)
	if err != nil {
		return nil, err
	}
	err = c.Login(model.UserName, model.PassWord)
	if err != nil {
		return nil, err
	}
	err = c.NoOp()
	if err != nil {
		return nil, err
	}
	return c, err
}

// FTP文件删除
// create by gloomy 2017-04-02 01:08:15
// 文件名 ftp配置对象
// 错误对象
func FtpRemoveFile(filePathStr string, ftpConntion *ftp.ServerConn, model *FtpHelpStruct) error {
	var (
		err error
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return err
		}
	}
	return ftpConntion.Delete(filePathStr)
}

// ftp修正远程服务器文件名称
// create by gloomy 2017-04-04 21:26:48
// 源文件 修正后的文件名称 ftp配置对象
// 错误对象
func FtpRenameFile(from, to string, ftpConntion *ftp.ServerConn, model *FtpHelpStruct) error {
	var (
		err error
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return err
		}
	}
	return ftpConntion.Rename(from, to)
}

// 获取FTP上所有的文件列表
// create by gloomy 2017-04-12 10:43:25
// 目录地址
// 文件列表集 错误对象
func FtpNameList(pathStr string, ftpConntion *ftp.ServerConn, model *FtpHelpStruct) ([]string, error) {
	var (
		nameList []string
	)
	err := ftpList(pathStr, ftpConntion, model, &nameList)
	return nameList, err
}

func ftpList(pathStr string, ftpConntion *ftp.ServerConn, model *FtpHelpStruct, nameListIn *[]string) error {
	var (
		err     error
		pathDir string
	)
	if ftpConntion == nil || ftpConntion.NoOp() != nil {
		ftpConntion, err = ftpLogin(model)
		if err != nil {
			return err
		}
	}
	entryArray, err := ftpConntion.List(pathStr)
	if err != nil {
		return err
	}
	if strings.TrimSpace(pathStr) != "" && !strings.HasSuffix(pathStr, "/") {
		pathStr += "/"
	}
	for _, item := range entryArray {
		pathDir = fmt.Sprintf("%s%s", pathStr, item.Name)
		if item.Type == 0 {
			*nameListIn = append(*nameListIn, pathDir)
		} else {
			err = ftpList(pathDir, ftpConntion, model, nameListIn)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// ftp退出
// create by gloomy 2017-4-14 11:25:04
func FtpExit(ftpConntion *ftp.ServerConn) {
	if ftpConntion != nil {
		ftpConntion.Quit()
	}
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"fmt"
	"os"
	"path"
	"strconv"
	"testing"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	ginkgo_reporters "github.com/onsi/ginkgo/reporters"

	"kubevirt.io/kubevirt/tests/flags"

	"kubevirt.io/kubevirt/tests/reporter"

	"kubevirt.io/kubevirt/tests"
	qe_reporters "kubevirt.io/qe-tools/pkg/ginkgo-reporters"

	_ "kubevirt.io/kubevirt/tests/launchsecurity"
	_ "kubevirt.io/kubevirt/tests/monitoring"
	_ "kubevirt.io/kubevirt/tests/network"
	_ "kubevirt.io/kubevirt/tests/numa"
	_ "kubevirt.io/kubevirt/tests/performance"
	_ "kubevirt.io/kubevirt/tests/realtime"
	_ "kubevirt.io/kubevirt/tests/storage"
	_ "kubevirt.io/kubevirt/tests/virtoperatorbasic"
)

var justAfterEachReporter = []reporter.JustAfterEachReporter{}

func TestTests(t *testing.T) {
	flags.NormalizeFlags()
	tests.CalculateNamespaces()
	maxFails := getMaxFailsFromEnv()
	artifactsPath := path.Join(flags.ArtifactsDir, "k8s-reporter")
	junitOutput := path.Join(flags.ArtifactsDir, "junit.functest.xml")
	if qe_reporters.JunitOutput != "" {
		junitOutput = qe_reporters.JunitOutput
	}
	if config.GinkgoConfig.ParallelTotal > 1 {
		artifactsPath = path.Join(artifactsPath, strconv.Itoa(config.GinkgoConfig.ParallelNode))
		junitOutput = path.Join(flags.ArtifactsDir, fmt.Sprintf("partial.junit.functest.%d.xml", config.GinkgoConfig.ParallelNode))
	}

	outputEnricherReporter := reporter.NewCapturedOutputEnricher(
		ginkgo_reporters.NewJUnitReporter(junitOutput),
	)
	k8sReporter := reporter.NewKubernetesReporter(artifactsPath, maxFails)
	justAfterEachReporter = append(justAfterEachReporter, outputEnricherReporter, k8sReporter)
	reporters := []Reporter{
		outputEnricherReporter,
		k8sReporter,
	}
	if qe_reporters.Polarion.Run {
		reporters = append(reporters, &qe_reporters.Polarion)
	}

	RunSpecsWithDefaultAndCustomReporters(t, "Tests Suite", reporters)
}

var _ = SynchronizedBeforeSuite(tests.SynchronizedBeforeTestSetup, tests.BeforeTestSuitSetup)

var _ = SynchronizedAfterSuite(tests.AfterTestSuitCleanup, tests.SynchronizedAfterTestSuiteCleanup)

func getMaxFailsFromEnv() int {
	maxFailsEnv := os.Getenv("REPORTER_MAX_FAILS")
	if maxFailsEnv == "" {
		return 10
	}

	maxFails, err := strconv.Atoi(maxFailsEnv)
	if err != nil { // if the variable is set with a non int value
		fmt.Println("Invalid REPORTER_MAX_FAILS variable, defaulting to 10")
		return 10
	}

	return maxFails
}

// Collect info directly after each `It` execution with our reporters
// to collect the state directly after the spec.
var _ = JustAfterEach(func() {
	for _, reporter := range justAfterEachReporter {
		reporter.JustAfterEach(CurrentGinkgoTestDescription())
	}
})
<commit_msg>Pass through docker prefix and tag for vms-generator<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"fmt"
	"os"
	"path"
	"strconv"
	"testing"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	ginkgo_reporters "github.com/onsi/ginkgo/reporters"

	"kubevirt.io/kubevirt/tests/flags"

	"kubevirt.io/kubevirt/tests/reporter"

	"kubevirt.io/kubevirt/tests"
	qe_reporters "kubevirt.io/qe-tools/pkg/ginkgo-reporters"

	vmsgeneratorutils "kubevirt.io/kubevirt/tools/vms-generator/utils"

	_ "kubevirt.io/kubevirt/tests/launchsecurity"
	_ "kubevirt.io/kubevirt/tests/monitoring"
	_ "kubevirt.io/kubevirt/tests/network"
	_ "kubevirt.io/kubevirt/tests/numa"
	_ "kubevirt.io/kubevirt/tests/performance"
	_ "kubevirt.io/kubevirt/tests/realtime"
	_ "kubevirt.io/kubevirt/tests/storage"
	_ "kubevirt.io/kubevirt/tests/virtoperatorbasic"
)

var justAfterEachReporter = []reporter.JustAfterEachReporter{}

func TestTests(t *testing.T) {
	flags.NormalizeFlags()
	tests.CalculateNamespaces()
	maxFails := getMaxFailsFromEnv()
	artifactsPath := path.Join(flags.ArtifactsDir, "k8s-reporter")
	junitOutput := path.Join(flags.ArtifactsDir, "junit.functest.xml")
	if qe_reporters.JunitOutput != "" {
		junitOutput = qe_reporters.JunitOutput
	}
	if config.GinkgoConfig.ParallelTotal > 1 {
		artifactsPath = path.Join(artifactsPath, strconv.Itoa(config.GinkgoConfig.ParallelNode))
		junitOutput = path.Join(flags.ArtifactsDir, fmt.Sprintf("partial.junit.functest.%d.xml", config.GinkgoConfig.ParallelNode))
	}

	outputEnricherReporter := reporter.NewCapturedOutputEnricher(
		ginkgo_reporters.NewJUnitReporter(junitOutput),
	)
	k8sReporter := reporter.NewKubernetesReporter(artifactsPath, maxFails)
	justAfterEachReporter = append(justAfterEachReporter, outputEnricherReporter, k8sReporter)
	reporters := []Reporter{
		outputEnricherReporter,
		k8sReporter,
	}
	if qe_reporters.Polarion.Run {
		reporters = append(reporters, &qe_reporters.Polarion)
	}

	vmsgeneratorutils.DockerPrefix = flags.KubeVirtUtilityRepoPrefix
	vmsgeneratorutils.DockerTag = flags.KubeVirtVersionTag

	RunSpecsWithDefaultAndCustomReporters(t, "Tests Suite", reporters)
}

var _ = SynchronizedBeforeSuite(tests.SynchronizedBeforeTestSetup, tests.BeforeTestSuitSetup)

var _ = SynchronizedAfterSuite(tests.AfterTestSuitCleanup, tests.SynchronizedAfterTestSuiteCleanup)

func getMaxFailsFromEnv() int {
	maxFailsEnv := os.Getenv("REPORTER_MAX_FAILS")
	if maxFailsEnv == "" {
		return 10
	}

	maxFails, err := strconv.Atoi(maxFailsEnv)
	if err != nil { // if the variable is set with a non int value
		fmt.Println("Invalid REPORTER_MAX_FAILS variable, defaulting to 10")
		return 10
	}

	return maxFails
}

// Collect info directly after each `It` execution with our reporters
// to collect the state directly after the spec.
var _ = JustAfterEach(func() {
	for _, reporter := range justAfterEachReporter {
		reporter.JustAfterEach(CurrentGinkgoTestDescription())
	}
})
<|endoftext|>