<commit_before>package mit

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/MyHomeworkSpace/api-server/errorlog"

	"github.com/PuerkitoBio/goquery"

	"github.com/thatoddmailbox/touchstone-client/touchstone"
)

const (
	studentDashboardURL                string = "https://student-dashboard.mit.edu"
	studentDashboardAcademicProfileURL string = studentDashboardURL + "/service/apis/students/students/academicProfile"
	studentDashboardRegistrationURL    string = studentDashboardURL + "/service/apis/studentregistration/statusOfRegistration"
)

const (
	peRegistrationURL string = "https://eduapps.mit.edu/mitpe/student/registration/home"
)

type academicProfileInfo struct {
	Name  string `json:"name"`
	Year  string `json:"year"`
	MITID string `json:"mitid"`
}

type subjectSelectionInfo struct {
	SubjectID string `json:"subjectID"`
	SectionID string `json:"sectionID"`
	Title     string `json:"title"`
	Units     int    `json:"units"`
}

type subjectRegistrationInfo struct {
	Selection subjectSelectionInfo `json:"subjectSelectionInfo"`
}

type academicRegistrationInfo struct {
	TermCode         string                    `json:"termCode"`
	TermDescription  string                    `json:"termDescription"`
	Subjects         []subjectRegistrationInfo `json:"regSubjectSelectionInfo"`
	TotalUnits       string                    `json:"totalUnits"`
	RegistrationLoad string                    `json:"registrationLoad"`
}

type peRegistrationInfo struct {
	SectionID        string `json:"sectionID"`
	SectionNumber    int    `json:"sectionNumber"`
	CourseTitle      string `json:"courseTitle"`
	Quarter          string `json:"quarter"`
	QuarterShortCode string `json:"quarterShortCode"`
	PERegTermCode    string `json:"peRegTermCode"`
	LMODTermCode     string `json:"lmodTermCode"`
}

type registrationInfo struct {
	StatusOfRegistration academicRegistrationInfo `json:"statusOfRegistration"`
	PERegistrations      []peRegistrationInfo     `json:"peRegistrations"`
}

func fetchDataWithClient(tsClient *touchstone.Client, username string) (*academicProfileInfo, *registrationInfo, *peInfo, error) {
	// first: authenticate to the fancy beta undergrad dashboard, as it has a nice api
	// we have to load the main page first because the api endpoints won't redirect to touchstone
	_, err := tsClient.AuthenticateToResource(studentDashboardURL)
	if err != nil {
		return nil, nil, nil, err
	}

	// academic profile - name, id, class year
	academicProfileResp, err := tsClient.AuthenticateToResource(studentDashboardAcademicProfileURL)
	if err != nil {
		return nil, nil, nil, err
	}
	defer academicProfileResp.Body.Close()

	academicProfile := academicProfileInfo{}
	err = json.NewDecoder(academicProfileResp.Body).Decode(&academicProfile)
	if err != nil {
		return nil, nil, nil, err
	}

	// status of registration
	registrationResp, err := tsClient.AuthenticateToResource(studentDashboardRegistrationURL)
	if err != nil {
		return nil, nil, nil, err
	}
	defer registrationResp.Body.Close()

	registration := registrationInfo{}
	err = json.NewDecoder(registrationResp.Body).Decode(&registration)
	if err != nil {
		return nil, nil, nil, err
	}

	// we have to get the pe registration separately, as the student dashboard doesn't tell us times of classes
	peRegistrationResp, err := tsClient.AuthenticateToResource(peRegistrationURL)
	if err != nil {
		return nil, nil, nil, err
	}
	defer peRegistrationResp.Body.Close()

	peRegistrationDoc, err := goquery.NewDocumentFromReader(peRegistrationResp.Body)
	if err != nil {
		return nil, nil, nil, err
	}

	peInfo, err := parseDocForPEInfo(peRegistrationDoc)
	if err != nil {
		return nil, nil, nil, err
	}

	return &academicProfile, &registration, peInfo, nil
}

func parsePEDate(peDate string) (string, error) {
	parts := strings.Split(peDate, " ")
	if len(parts) != 2 {
		return "", fmt.Errorf("mit: scraper: couldn't parse pe date '%s'", peDate)
	}

	parsedDate, err := time.Parse("01/02/2006", parts[1])
	if err != nil {
		return "", fmt.Errorf("mit: scraper: couldn't parse pe date '%s'", peDate)
	}

	return parsedDate.Format("2006-01-02"), nil
}

func parsePESchedule(peSchedule string) ([]time.Weekday, int, int, string, error) {
	// return values are: ParsedDaysOfWeek, ParsedStartTime, ParsedEndTime, ParsedLocation, error
	// the input is in the format "Tue, Thu 2:00 PM at Location That Might Have Spaces"
	weekdayMap := map[string]time.Weekday{
		"Sun": time.Sunday,
		"Mon": time.Monday,
		"Tue": time.Tuesday,
		"Wed": time.Wednesday,
		"Thu": time.Thursday,
		"Fri": time.Friday,
		"Sat": time.Saturday,
	}

	atParts := strings.Split(peSchedule, " at ")
	if len(atParts) != 2 {
		return nil, 0, 0, "", fmt.Errorf("mit: scraper: wrong number of atParts, %d", len(atParts))
	}

	// assume the second half of the string is the location
	location := atParts[1]

	// parse the weekdays
	weekdays := []time.Weekday{}
	weekdayStrings := strings.Split(atParts[0], ", ")
	for _, weekdayString := range weekdayStrings {
		cleanWeekdayString := strings.Split(weekdayString, " ")[0]
		weekday, matched := weekdayMap[cleanWeekdayString]
		if !matched {
			return nil, 0, 0, "", fmt.Errorf("mit: scraper: unknown day of week '%s'", weekdayString)
		}

		weekdays = append(weekdays, weekday)
	}

	// parse the start time
	lastWeekdayParts := strings.SplitN(weekdayStrings[len(weekdayStrings)-1], " ", 2)
	if len(lastWeekdayParts) != 2 {
		return nil, 0, 0, "", fmt.Errorf("mit: scraper: wrong number of lastWeekdayParts, %d", len(lastWeekdayParts))
	}

	startTimeString := lastWeekdayParts[1]
	startTime, err := time.Parse("3:04 PM", startTimeString)
	if len(lastWeekdayParts) != 2 {
		return nil, 0, 0, "", err
	}

	// i'm pretty sure all PE classes are an hour long? i think?
	endTime := startTime.Add(time.Hour)

	startTime = startTime.AddDate(1970, 0, 0)
	endTime = endTime.AddDate(1970, 0, 0)

	return weekdays, int(startTime.Unix()), int(endTime.Unix()), location, nil
}

func parseDocForPEInfo(doc *goquery.Document) (*peInfo, error) {
	sectionContainer := doc.Find(".sectionContainer")
	if sectionContainer.Length() == 0 {
		return nil, errors.New("mit: scraper: couldn't find .sectionContainer in PE doc")
	}

	registered := false
	peInfo := peInfo{}

	tableRows := sectionContainer.Find("tr")
	tableRows.Each(func(i int, s *goquery.Selection) {
		key := strings.TrimSpace(s.Find("th").Text())
		value := strings.TrimSpace(s.Find("td").Text())

		if key == "Status" {
			if value == "Registered" {
				registered = true
			}
		} else if key == "Section ID" {
			peInfo.SectionID = value
		} else if key == "Activity" {
			peInfo.Activity = value
		} else if key == "Course Title" {
			peInfo.CourseTitle = value
		} else if key == "Schedule" {
			peInfo.RawSchedule = value
		} else if key == "First Day of Class" {
			peInfo.RawFirstDay = value
		} else if key == "Last Day of Class" {
			peInfo.RawLastDay = value
		} else if key == "Calendar Notes" {
			peInfo.RawCalendarNotes = value
		}
	})

	if !registered {
		return nil, nil
	}

	var err error

	peInfo.ParsedFirstDay, err = parsePEDate(peInfo.RawFirstDay)
	if err != nil {
		return nil, err
	}

	firstDay, err := time.Parse("2006-01-02", peInfo.ParsedFirstDay)
	if err != nil {
		return nil, err
	}

	peInfo.ParsedLastDay, err = parsePEDate(peInfo.RawLastDay)
	if err != nil {
		return nil, err
	}

	peInfo.ParsedSkipDays = []string{}

	if peInfo.RawCalendarNotes != "" {
		// so far, the only format i know for this is "no classes 11/11, 11/26, 11/27"
		parsedNotes := false
		if strings.HasPrefix(peInfo.RawCalendarNotes, "no classes ") {
			noPrefixString := strings.Replace(peInfo.RawCalendarNotes, "no classes ", "", -1)
			parts := strings.Split(noPrefixString, ", ")

			for _, rawSkipDay := range parts {
				// these are in the format "11/26"
				skipDayParts := strings.Split(rawSkipDay, "/")
				if len(skipDayParts) != 2 {
					continue
				}

				month, err := strconv.Atoi(skipDayParts[0])
				if err != nil {
					continue
				}

				date, err := strconv.Atoi(skipDayParts[1])
				if err != nil {
					continue
				}

				// the year not given to us and must be guessed
				year := firstDay.Year()

				// if the month is BEFORE the month that the class starts, assume it's next year
				if time.Month(month) < firstDay.Month() {
					year++
				}

				skipDay := time.Date(year, time.Month(month), date, 0, 0, 0, 0, time.UTC)

				peInfo.ParsedSkipDays = append(peInfo.ParsedSkipDays, skipDay.Format("2006-01-02"))

				parsedNotes = true
			}
		}

		if !parsedNotes {
			// oofie
			// report it back and just hope that there's nothing important...
			errorlog.LogError(
				"parsing PE calendar notes",
				fmt.Errorf(
					"mit: scraper: couldn't parse calendar notes '%s' for '%s'",
					peInfo.RawCalendarNotes,
					peInfo.SectionID,
				),
			)
		}
	}

	// now we have to parse the schedule field
	peInfo.ParsedDaysOfWeek, peInfo.ParsedStartTime, peInfo.ParsedEndTime, peInfo.ParsedLocation, err = parsePESchedule(peInfo.RawSchedule)
	if err != nil {
		return nil, err
	}

	return &peInfo, nil
}
<commit_msg>schools/mit/scraper: working calendar notes parser for q2 2021-2022 pe<commit_after>package mit

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/MyHomeworkSpace/api-server/errorlog"

	"github.com/PuerkitoBio/goquery"

	"github.com/thatoddmailbox/touchstone-client/touchstone"
)

const (
	studentDashboardURL                string = "https://student-dashboard.mit.edu"
	studentDashboardAcademicProfileURL string = studentDashboardURL + "/service/apis/students/students/academicProfile"
	studentDashboardRegistrationURL    string = studentDashboardURL + "/service/apis/studentregistration/statusOfRegistration"
)

const (
	peRegistrationURL string = "https://eduapps.mit.edu/mitpe/student/registration/home"
)

type academicProfileInfo struct {
	Name  string `json:"name"`
	Year  string `json:"year"`
	MITID string `json:"mitid"`
}

type subjectSelectionInfo struct {
	SubjectID string `json:"subjectID"`
	SectionID string `json:"sectionID"`
	Title     string `json:"title"`
	Units     int    `json:"units"`
}

type subjectRegistrationInfo struct {
	Selection subjectSelectionInfo `json:"subjectSelectionInfo"`
}

type academicRegistrationInfo struct {
	TermCode         string                    `json:"termCode"`
	TermDescription  string                    `json:"termDescription"`
	Subjects         []subjectRegistrationInfo `json:"regSubjectSelectionInfo"`
	TotalUnits       string                    `json:"totalUnits"`
	RegistrationLoad string                    `json:"registrationLoad"`
}

type peRegistrationInfo struct {
	SectionID        string `json:"sectionID"`
	SectionNumber    int    `json:"sectionNumber"`
	CourseTitle      string `json:"courseTitle"`
	Quarter          string `json:"quarter"`
	QuarterShortCode string `json:"quarterShortCode"`
	PERegTermCode    string `json:"peRegTermCode"`
	LMODTermCode     string `json:"lmodTermCode"`
}

type registrationInfo struct {
	StatusOfRegistration academicRegistrationInfo `json:"statusOfRegistration"`
	PERegistrations      []peRegistrationInfo     `json:"peRegistrations"`
}

func fetchDataWithClient(tsClient *touchstone.Client, username string) (*academicProfileInfo, *registrationInfo, *peInfo, error) {
	// first: authenticate to the fancy beta undergrad dashboard, as it has a nice api
	// we have to load the main page first because the api endpoints won't redirect to touchstone
	_, err := tsClient.AuthenticateToResource(studentDashboardURL)
	if err != nil {
		return nil, nil, nil, err
	}

	// academic profile - name, id, class year
	academicProfileResp, err := tsClient.AuthenticateToResource(studentDashboardAcademicProfileURL)
	if err != nil {
		return nil, nil, nil, err
	}
	defer academicProfileResp.Body.Close()

	academicProfile := academicProfileInfo{}
	err = json.NewDecoder(academicProfileResp.Body).Decode(&academicProfile)
	if err != nil {
		return nil, nil, nil, err
	}

	// status of registration
	registrationResp, err := tsClient.AuthenticateToResource(studentDashboardRegistrationURL)
	if err != nil {
		return nil, nil, nil, err
	}
	defer registrationResp.Body.Close()

	registration := registrationInfo{}
	err = json.NewDecoder(registrationResp.Body).Decode(&registration)
	if err != nil {
		return nil, nil, nil, err
	}

	// we have to get the pe registration separately, as the student dashboard doesn't tell us times of classes
	peRegistrationResp, err := tsClient.AuthenticateToResource(peRegistrationURL)
	if err != nil {
		return nil, nil, nil, err
	}
	defer peRegistrationResp.Body.Close()

	peRegistrationDoc, err := goquery.NewDocumentFromReader(peRegistrationResp.Body)
	if err != nil {
		return nil, nil, nil, err
	}

	peInfo, err := parseDocForPEInfo(peRegistrationDoc)
	if err != nil {
		return nil, nil, nil, err
	}

	return &academicProfile, &registration, peInfo, nil
}

func parsePEDate(peDate string) (string, error) {
	parts := strings.Split(peDate, " ")
	if len(parts) != 2 {
		return "", fmt.Errorf("mit: scraper: couldn't parse pe date '%s'", peDate)
	}

	parsedDate, err := time.Parse("01/02/2006", parts[1])
	if err != nil {
		return "", fmt.Errorf("mit: scraper: couldn't parse pe date '%s'", peDate)
	}

	return parsedDate.Format("2006-01-02"), nil
}

func parsePESchedule(peSchedule string) ([]time.Weekday, int, int, string, error) {
	// return values are: ParsedDaysOfWeek, ParsedStartTime, ParsedEndTime, ParsedLocation, error
	// the input is in the format "Tue, Thu 2:00 PM at Location That Might Have Spaces"
	weekdayMap := map[string]time.Weekday{
		"Sun": time.Sunday,
		"Mon": time.Monday,
		"Tue": time.Tuesday,
		"Wed": time.Wednesday,
		"Thu": time.Thursday,
		"Fri": time.Friday,
		"Sat": time.Saturday,
	}

	atParts := strings.Split(peSchedule, " at ")
	if len(atParts) != 2 {
		return nil, 0, 0, "", fmt.Errorf("mit: scraper: wrong number of atParts, %d", len(atParts))
	}

	// assume the second half of the string is the location
	location := atParts[1]

	// parse the weekdays
	weekdays := []time.Weekday{}
	weekdayStrings := strings.Split(atParts[0], ", ")
	for _, weekdayString := range weekdayStrings {
		cleanWeekdayString := strings.Split(weekdayString, " ")[0]
		weekday, matched := weekdayMap[cleanWeekdayString]
		if !matched {
			return nil, 0, 0, "", fmt.Errorf("mit: scraper: unknown day of week '%s'", weekdayString)
		}

		weekdays = append(weekdays, weekday)
	}

	// parse the start time
	lastWeekdayParts := strings.SplitN(weekdayStrings[len(weekdayStrings)-1], " ", 2)
	if len(lastWeekdayParts) != 2 {
		return nil, 0, 0, "", fmt.Errorf("mit: scraper: wrong number of lastWeekdayParts, %d", len(lastWeekdayParts))
	}

	startTimeString := lastWeekdayParts[1]
	startTime, err := time.Parse("3:04 PM", startTimeString)
	if len(lastWeekdayParts) != 2 {
		return nil, 0, 0, "", err
	}

	// i'm pretty sure all PE classes are an hour long? i think?
	endTime := startTime.Add(time.Hour)

	startTime = startTime.AddDate(1970, 0, 0)
	endTime = endTime.AddDate(1970, 0, 0)

	return weekdays, int(startTime.Unix()), int(endTime.Unix()), location, nil
}

func parseDocForPEInfo(doc *goquery.Document) (*peInfo, error) {
	sectionContainer := doc.Find(".sectionContainer")
	if sectionContainer.Length() == 0 {
		return nil, errors.New("mit: scraper: couldn't find .sectionContainer in PE doc")
	}

	registered := false
	peInfo := peInfo{}

	tableRows := sectionContainer.Find("tr")
	tableRows.Each(func(i int, s *goquery.Selection) {
		key := strings.TrimSpace(s.Find("th").Text())
		value := strings.TrimSpace(s.Find("td").Text())

		if key == "Status" {
			if value == "Registered" {
				registered = true
			}
		} else if key == "Section ID" {
			peInfo.SectionID = value
		} else if key == "Activity" {
			peInfo.Activity = value
		} else if key == "Course Title" {
			peInfo.CourseTitle = value
		} else if key == "Schedule" {
			peInfo.RawSchedule = value
		} else if key == "First Day of Class" {
			peInfo.RawFirstDay = value
		} else if key == "Last Day of Class" {
			peInfo.RawLastDay = value
		} else if key == "Calendar Notes" {
			peInfo.RawCalendarNotes = value
		}
	})

	if !registered {
		return nil, nil
	}

	var err error

	peInfo.ParsedFirstDay, err = parsePEDate(peInfo.RawFirstDay)
	if err != nil {
		return nil, err
	}

	firstDay, err := time.Parse("2006-01-02", peInfo.ParsedFirstDay)
	if err != nil {
		return nil, err
	}

	peInfo.ParsedLastDay, err = parsePEDate(peInfo.RawLastDay)
	if err != nil {
		return nil, err
	}

	peInfo.ParsedSkipDays = []string{}

	if peInfo.RawCalendarNotes != "" {
		// so far, the only formats I know for this is "no classes 11/11, 11/26, 11/27" and "No Classes 11/11, 22, 23, 24, 25"
		parsedNotes := false
		calendarNotes := strings.ToLower(peInfo.RawCalendarNotes)
		if strings.HasPrefix(calendarNotes, "no classes ") {
			noPrefixString := strings.Replace(calendarNotes, "no classes ", "", -1)
			parts := strings.Split(noPrefixString, ", ")

			lastMonth := 0
			for _, rawSkipDay := range parts {
				// these are in the format "11/26"
				skipDayParts := strings.Split(rawSkipDay, "/")
				if len(skipDayParts) > 2 {
					continue
				}

				month, date := 0, 0
				if len(skipDayParts) == 1 {
					// it's one number
					// this implies that we use the same month as before
					if lastMonth == 0 {
						// there is no last month???
						// not sure what's going on here!!!
						continue
					}

					month = lastMonth
					date, err = strconv.Atoi(skipDayParts[0])
					if err != nil {
						continue
					}
				} else if len(skipDayParts) == 2 {
					// it's two numbers
					// it's a date like 11/26
					month, err = strconv.Atoi(skipDayParts[0])
					if err != nil {
						continue
					}

					date, err = strconv.Atoi(skipDayParts[1])
					if err != nil {
						continue
					}
				}

				// the year not given to us and must be guessed
				year := firstDay.Year()

				// if the month is BEFORE the month that the class starts, assume it's next year
				if time.Month(month) < firstDay.Month() {
					year++
				}

				skipDay := time.Date(year, time.Month(month), date, 0, 0, 0, 0, time.UTC)

				peInfo.ParsedSkipDays = append(peInfo.ParsedSkipDays, skipDay.Format("2006-01-02"))

				parsedNotes = true
			}
		}

		if !parsedNotes {
			// oofie
			// report it back and just hope that there's nothing important...
			errorlog.LogError(
				"parsing PE calendar notes",
				fmt.Errorf(
					"mit: scraper: couldn't parse calendar notes '%s' for '%s'",
					peInfo.RawCalendarNotes,
					peInfo.SectionID,
				),
			)
		}
	}

	// now we have to parse the schedule field
	peInfo.ParsedDaysOfWeek, peInfo.ParsedStartTime, peInfo.ParsedEndTime, peInfo.ParsedLocation, err = parsePESchedule(peInfo.RawSchedule)
	if err != nil {
		return nil, err
	}

	return &peInfo, nil
}
<|endoftext|>


<commit_before>/*
 * (c) 2014, Caoimhe Chaos <caoimhechaos@protonmail.com>,
 *	     Starship Factory. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the name  of the Starship Factory  nor the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package file

import (
	"io"
	"net/url"
	"os"

	"code.google.com/p/go.exp/fsnotify"
	"github.com/caoimhechaos/go-file"
)

// Object for generating watchers for individual files.
type FileWatcherCreator struct {
}

// Create a new watcher object for watching for notifications on the
// given URL.
func (f *FileWatcherCreator) Watch(
	fileid *url.URL, cb func(string, io.ReadCloser)) (
	file.Watcher, error) {
	return NewFileWatcher(fileid.Path, cb)
}

// Object for watching an individual file for changes.
type FileWatcher struct {
	cb       func(string, io.ReadCloser)
	watcher  *fsnotify.Watcher
	path     string
	shutdown bool
}

// Resolve an absolute and a relative path to a new absolute path.
func resolveRelative(orig, relative string) (path string, err error) {
	var origurl, newurl *url.URL

	origurl, err = url.Parse(orig)
	if err != nil {
		return
	}

	newurl, err = origurl.Parse(relative)
	if err != nil {
		return
	}

	path = newurl.String()
	return
}

// Automatically sign us up for file:// URLs.
func init() {
	file.RegisterWatcher("file", &FileWatcherCreator{})
}

// Create a new FileWatcher watching the file at "path".
func NewFileWatcher(path string, cb func(string, io.ReadCloser)) (
	*FileWatcher, error) {
	var fi os.FileInfo
	var ret *FileWatcher
	var watcher *fsnotify.Watcher
	var err error

	watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	err = watcher.Watch(path)
	if err != nil {
		return nil, err
	}

	ret = &FileWatcher{
		cb:      cb,
		watcher: watcher,
		path:    path,
	}

	// Treat the current state of the file as the first change.
	fi, err = os.Stat(path)
	if err != nil {
		return nil, err
	}

	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		var subpath string

		subpath, err = os.Readlink(path)
		if err != nil {
			return nil, err
		}

		path, err = resolveRelative(path, subpath)
		if err != nil {
			return nil, err
		}
	}

	if fi.IsDir() {
		var names []string
		var name string
		var f *os.File

		f, err = os.Open(path)
		if err != nil {
			return nil, err
		}

		names, err = f.Readdirnames(-1)
		if err != nil {
			return nil, err
		}

		for _, name = range names {
			var combined string
			var reader *os.File

			combined, err = resolveRelative(path+"/", name)
			if err != nil {
				return nil, err
			}

			reader, err = os.Open(combined)
			cb(combined, reader)
		}

		f.Close()
	} else {
		var reader *os.File

		reader, err = os.Open(path)
		if err != nil {
			return nil, err
		}

		cb(path, reader)
	}

	go ret.watchForChanges()

	return ret, nil
}

// Read events happening on the file being watched and forward them
// to the relevant callback.
func (f *FileWatcher) watchForChanges() {
	for !f.shutdown {
		var event *fsnotify.FileEvent

		event = <-f.watcher.Event

		if event.IsModify() {
			var fn *os.File
			var err error

			fn, err = os.Open(event.Name)
			if err == nil {
				go f.cb(event.Name, fn)
			} else {
				f.watcher.Error <- err
			}
		}
	}
}

// Stop listening for notifications on the file.
func (f *FileWatcher) Shutdown() error {
	var err error

	f.shutdown = true
	err = f.watcher.RemoveWatch(f.path)
	if err != nil {
		return err
	}

	return f.watcher.Close()
}

// Retrieve the error channel associated with the watcher.
// It will stream a list of all errors created while watching.
func (f *FileWatcher) ErrChan() chan error {
	return f.watcher.Error
}
<commit_msg>Update to a more modern fsnotify API.<commit_after>/*
 * (c) 2014, Caoimhe Chaos <caoimhechaos@protonmail.com>,
 *	     Starship Factory. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the name  of the Starship Factory  nor the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package file

import (
	"io"
	"net/url"
	"os"

	"github.com/caoimhechaos/go-file"
	"gopkg.in/fsnotify.v1"
)

// Object for generating watchers for individual files.
type FileWatcherCreator struct {
}

// Create a new watcher object for watching for notifications on the
// given URL.
func (f *FileWatcherCreator) Watch(
	fileid *url.URL, cb func(string, io.ReadCloser)) (
	file.Watcher, error) {
	return NewFileWatcher(fileid.Path, cb)
}

// Object for watching an individual file for changes.
type FileWatcher struct {
	cb       func(string, io.ReadCloser)
	watcher  *fsnotify.Watcher
	path     string
	shutdown bool
}

// Resolve an absolute and a relative path to a new absolute path.
func resolveRelative(orig, relative string) (path string, err error) {
	var origurl, newurl *url.URL

	origurl, err = url.Parse(orig)
	if err != nil {
		return
	}

	newurl, err = origurl.Parse(relative)
	if err != nil {
		return
	}

	path = newurl.String()
	return
}

// Automatically sign us up for file:// URLs.
func init() {
	file.RegisterWatcher("file", &FileWatcherCreator{})
}

// Create a new FileWatcher watching the file at "path".
func NewFileWatcher(path string, cb func(string, io.ReadCloser)) (
	*FileWatcher, error) {
	var fi os.FileInfo
	var ret *FileWatcher
	var watcher *fsnotify.Watcher
	var err error

	watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	err = watcher.Add(path)
	if err != nil {
		return nil, err
	}

	ret = &FileWatcher{
		cb:      cb,
		watcher: watcher,
		path:    path,
	}

	// Treat the current state of the file as the first change.
	fi, err = os.Stat(path)
	if err != nil {
		return nil, err
	}

	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		var subpath string

		subpath, err = os.Readlink(path)
		if err != nil {
			return nil, err
		}

		path, err = resolveRelative(path, subpath)
		if err != nil {
			return nil, err
		}
	}

	if fi.IsDir() {
		var names []string
		var name string
		var f *os.File

		f, err = os.Open(path)
		if err != nil {
			return nil, err
		}

		names, err = f.Readdirnames(-1)
		if err != nil {
			return nil, err
		}

		for _, name = range names {
			var combined string
			var reader *os.File

			combined, err = resolveRelative(path+"/", name)
			if err != nil {
				return nil, err
			}

			reader, err = os.Open(combined)
			cb(combined, reader)
		}

		f.Close()
	} else {
		var reader *os.File

		reader, err = os.Open(path)
		if err != nil {
			return nil, err
		}

		cb(path, reader)
	}

	go ret.watchForChanges()

	return ret, nil
}

// Read events happening on the file being watched and forward them
// to the relevant callback.
func (f *FileWatcher) watchForChanges() {
	for !f.shutdown {
		var event fsnotify.Event

		event = <-f.watcher.Events

		if event.Op&(fsnotify.Write|fsnotify.Remove|fsnotify.Rename) != 0 {
			var fn *os.File
			var err error

			fn, err = os.Open(event.Name)
			if err == nil {
				go f.cb(event.Name, fn)
			} else {
				f.watcher.Errors <- err
			}
		}
	}
}

// Stop listening for notifications on the file.
func (f *FileWatcher) Shutdown() error {
	var err error

	f.shutdown = true
	err = f.watcher.Remove(f.path)
	if err != nil {
		return err
	}

	return f.watcher.Close()
}

// Retrieve the error channel associated with the watcher.
// It will stream a list of all errors created while watching.
func (f *FileWatcher) ErrChan() chan error {
	return f.watcher.Errors
}
<|endoftext|>


<commit_before>package fin

import (
	"math"
	"testing"
	"time"
)

func TestNetPresentValue(t *testing.T) {
	var tests = []struct {
		rate   float64
		values []float64
		want   float64
	}{
		{0.1, []float64{-10000, 3000, 4200, 6800}, 1188.443412},
	}

	for _, test := range tests {
		if got := NetPresentValue(test.rate, test.values); math.Abs(test.want-got) > Precision {
			t.Errorf("NetPresentValue(%f, %v) = %f", test.rate, test.values, got)
		}
	}
}

func TestInternalRateOfReturn(t *testing.T) {
	var tests = []struct {
		values []float64
		guess  float64
		want   float64
	}{
		{[]float64{-70000, 12000, 15000, 18000, 21000}, 0.1, -0.02124485},
		{[]float64{-70000, 12000, 15000, 18000, 21000, 26000}, 0.1, 0.086630},
		{[]float64{-70000, 12000, 15000}, -0.40, -0.443507},
	}

	for _, test := range tests {
		if got, _ := InternalRateOfReturn(test.values, test.guess); math.Abs(test.want-got) > Precision {
			t.Errorf("InternalRateOfReturn(%v, %f) = %f", test.values, test.guess, got)
		}
	}

	if _, err := InternalRateOfReturn([]float64{70000, 12000, 15000, 18000, 21000}, 0.1); err == nil {
		t.Error("If the cash flow doesn't contain at least one positive value and one negative value, it must return an error")
	}
}

func TestModifiedInternalRateOfReturn(t *testing.T) {
	var tests = []struct {
		values       []float64
		financeRate  float64
		reinvestRate float64
		want         float64
	}{
		{[]float64{-120000, 39000, 30000, 21000, 37000, 46000}, 0.10, 0.12, 0.126094},
		{[]float64{-120000, 39000, 30000, 21000}, 0.10, 0.12, -0.048044},
		{[]float64{-120000, 39000, 30000, 21000, 37000, 46000}, 0.10, 0.14, 0.134759},
	}

	for _, test := range tests {
		if got, _ := ModifiedInternalRateOfReturn(test.values, test.financeRate, test.reinvestRate); math.Abs(test.want-got) > Precision {
			t.Errorf("ModifiedInternalRateOfReturn(%v, %f, %f) = %f", test.values, test.financeRate, test.reinvestRate, got)
		}
	}

	if _, err := ModifiedInternalRateOfReturn([]float64{70000, 12000, 15000, 18000, 21000}, 0.1, 0.1); err == nil {
		t.Error("If the cash flow doesn't contain at least one positive value and one negative value, it must return an error")
	}
}

func TestScheduledNetPresentValue(t *testing.T) {
	var tests = []struct {
		rate   float64
		values []float64
		dates  []time.Time
		want   float64
	}{
		{
			0.09,
			[]float64{
				-10000,
				2750,
				4250,
				3250,
				2750,
			},
			[]time.Time{
				time.Date(2008, time.Month(1), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(3), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(10), 30, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(2), 15, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(4), 1, 0, 0, 0, 0, time.UTC),
			},
			2086.647602,
		},
	}

	for _, test := range tests {
		if got, _ := ScheduledNetPresentValue(test.rate, test.values, test.dates); math.Abs(test.want-got) > Precision {
			t.Errorf("ScheduledNetPresentValue(%f, %v, %v) = %f", test.rate, test.values, test.dates, got)
		}
	}

	if _, err := ScheduledNetPresentValue(0.1, []float64{-10000}, []time.Time{}); err == nil {
		t.Error("If values and dates have different lenghts, it must return an error")
	}
}

func TestScheduledInternalRateOfReturn(t *testing.T) {
	var tests = []struct {
		values []float64
		dates  []time.Time
		guess  float64
		want   float64
	}{
		{
			[]float64{
				-10000,
				2750,
				4250,
				3250,
				2750,
			},
			[]time.Time{
				time.Date(2008, time.Month(1), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(3), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(10), 30, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(2), 15, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(4), 1, 0, 0, 0, 0, time.UTC),
			},
			0.1,
			0.373362535,
		},
	}

	for _, test := range tests {
		if got, _ := ScheduledInternalRateOfReturn(test.values, test.dates, test.guess); math.Abs(test.want-got) > Precision {
			t.Errorf("ScheduledInternalRateOfReturn(%v, %v, %f) = %f", test.values, test.dates, test.guess, got)
		}
	}

	if _, err := ScheduledInternalRateOfReturn([]float64{10000, 2750}, []time.Time{time.Now(), time.Now()}, 0.1); err == nil {
		t.Error("If the cash flow doesn't contain at least one positive value and one negative value, it must return an error")
	}

	if _, err := ScheduledInternalRateOfReturn([]float64{-10000, 2750}, []time.Time{}, 0.1); err == nil {
		t.Error("If values and dates have different lenghts, it must return an error")
	}
}
<commit_msg>Fix misspell reported by Go Report Card<commit_after>package fin

import (
	"math"
	"testing"
	"time"
)

func TestNetPresentValue(t *testing.T) {
	var tests = []struct {
		rate   float64
		values []float64
		want   float64
	}{
		{0.1, []float64{-10000, 3000, 4200, 6800}, 1188.443412},
	}

	for _, test := range tests {
		if got := NetPresentValue(test.rate, test.values); math.Abs(test.want-got) > Precision {
			t.Errorf("NetPresentValue(%f, %v) = %f", test.rate, test.values, got)
		}
	}
}

func TestInternalRateOfReturn(t *testing.T) {
	var tests = []struct {
		values []float64
		guess  float64
		want   float64
	}{
		{[]float64{-70000, 12000, 15000, 18000, 21000}, 0.1, -0.02124485},
		{[]float64{-70000, 12000, 15000, 18000, 21000, 26000}, 0.1, 0.086630},
		{[]float64{-70000, 12000, 15000}, -0.40, -0.443507},
	}

	for _, test := range tests {
		if got, _ := InternalRateOfReturn(test.values, test.guess); math.Abs(test.want-got) > Precision {
			t.Errorf("InternalRateOfReturn(%v, %f) = %f", test.values, test.guess, got)
		}
	}

	if _, err := InternalRateOfReturn([]float64{70000, 12000, 15000, 18000, 21000}, 0.1); err == nil {
		t.Error("If the cash flow doesn't contain at least one positive value and one negative value, it must return an error")
	}
}

func TestModifiedInternalRateOfReturn(t *testing.T) {
	var tests = []struct {
		values       []float64
		financeRate  float64
		reinvestRate float64
		want         float64
	}{
		{[]float64{-120000, 39000, 30000, 21000, 37000, 46000}, 0.10, 0.12, 0.126094},
		{[]float64{-120000, 39000, 30000, 21000}, 0.10, 0.12, -0.048044},
		{[]float64{-120000, 39000, 30000, 21000, 37000, 46000}, 0.10, 0.14, 0.134759},
	}

	for _, test := range tests {
		if got, _ := ModifiedInternalRateOfReturn(test.values, test.financeRate, test.reinvestRate); math.Abs(test.want-got) > Precision {
			t.Errorf("ModifiedInternalRateOfReturn(%v, %f, %f) = %f", test.values, test.financeRate, test.reinvestRate, got)
		}
	}

	if _, err := ModifiedInternalRateOfReturn([]float64{70000, 12000, 15000, 18000, 21000}, 0.1, 0.1); err == nil {
		t.Error("If the cash flow doesn't contain at least one positive value and one negative value, it must return an error")
	}
}

func TestScheduledNetPresentValue(t *testing.T) {
	var tests = []struct {
		rate   float64
		values []float64
		dates  []time.Time
		want   float64
	}{
		{
			0.09,
			[]float64{
				-10000,
				2750,
				4250,
				3250,
				2750,
			},
			[]time.Time{
				time.Date(2008, time.Month(1), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(3), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(10), 30, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(2), 15, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(4), 1, 0, 0, 0, 0, time.UTC),
			},
			2086.647602,
		},
		{
			0.2,
			[]float64{
				-2000,
				1000,
				1000,
				1000,
			},
			[]time.Time{
				time.Date(2020, time.Month(2), 12, 0, 0, 0, 0, time.UTC),
				time.Date(2020, time.Month(3), 20, 0, 0, 0, 0, time.UTC),
				time.Date(2020, time.Month(4), 20, 0, 0, 0, 0, time.UTC),
				time.Date(2020, time.Month(5), 20, 0, 0, 0, 0, time.UTC),
			},
			900.5182206,
		},
	}

	for _, test := range tests {
		if got, _ := ScheduledNetPresentValue(test.rate, test.values, test.dates); math.Abs(test.want-got) > Precision {
			t.Errorf("ScheduledNetPresentValue(%f, %v, %v) = %f", test.rate, test.values, test.dates, got)
		}
	}

	if _, err := ScheduledNetPresentValue(0.1, []float64{-10000}, []time.Time{}); err == nil {
		t.Error("If values and dates have different lengths, it must return an error")
	}
}

func TestScheduledInternalRateOfReturn(t *testing.T) {
	var tests = []struct {
		values []float64
		dates  []time.Time
		guess  float64
		want   float64
	}{
		{
			[]float64{
				-10000,
				2750,
				4250,
				3250,
				2750,
			},
			[]time.Time{
				time.Date(2008, time.Month(1), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(3), 1, 0, 0, 0, 0, time.UTC),
				time.Date(2008, time.Month(10), 30, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(2), 15, 0, 0, 0, 0, time.UTC),
				time.Date(2009, time.Month(4), 1, 0, 0, 0, 0, time.UTC),
			},
			0.1,
			0.373362535,
		},
		{
			[]float64{
				-2000,
				1000,
				1000,
				1000,
			},
			[]time.Time{
				time.Date(2020, time.Month(2), 12, 0, 0, 0, 0, time.UTC),
				time.Date(2020, time.Month(3), 20, 0, 0, 0, 0, time.UTC),
				time.Date(2020, time.Month(4), 20, 0, 0, 0, 0, time.UTC),
				time.Date(2020, time.Month(5), 20, 0, 0, 0, 0, time.UTC),
			},
			0.1,
			8.493343973,
		},
	}

	for _, test := range tests {
		if got, _ := ScheduledInternalRateOfReturn(test.values, test.dates, test.guess); math.Abs(test.want-got) > Precision {
			t.Errorf("ScheduledInternalRateOfReturn(%v, %v, %f) = %f", test.values, test.dates, test.guess, got)
		}
	}

	if _, err := ScheduledInternalRateOfReturn([]float64{10000, 2750}, []time.Time{time.Now(), time.Now()}, 0.1); err == nil {
		t.Error("If the cash flow doesn't contain at least one positive value and one negative value, it must return an error")
	}

	if _, err := ScheduledInternalRateOfReturn([]float64{-10000, 2750}, []time.Time{}, 0.1); err == nil {
		t.Error("If values and dates have different lengths, it must return an error")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
	"github.com/qor/qor"
	"github.com/qor/qor/admin"
	"github.com/qor/qor/worker"
)

var db gorm.DB

func init() {
	var err error
	db, err = gorm.Open("sqlite3", "tmp/worker.db")
	if err != nil {
		panic(err)
	}
	db.LogMode(true)
}

func main() {
	config := qor.Config{DB: &db}
	web := admin.New(&config)
	// web.UseResource(user)

	// if err := db.DropTable(&worker.QorJob{}).Error; err != nil {
	// 	panic(err)
	// }
	if err := db.AutoMigrate(&worker.QorJob{}).Error; err != nil {
		panic(err)
	}
	worker.SetJobDB(&db)

	bq := worker.NewBeanstalkdQueue("beanstalkd", "localhost:11300")
	var counter int
	publishWorker := worker.New("Publish Jobs")

	web.AddResource(publishWorker, nil)

	publish := publishWorker.NewJob(bq, "publish products", "publish products so users could purchase new items", func(job *worker.QorJob) (err error) {
		log, err := job.GetLogger()
		if err != nil {
			return
		}

		_, err = log.Write([]byte(strconv.Itoa(counter) + "\n"))
		counter++
		time.Sleep(time.Minute * 5)
		return
	})

	// job.Meta(&admin.Meta{
	// 	Name: "File",
	// 	Type: "file",
	// 	Valuer: func(interface{}, *qor.Context) interface{} {
	// 		return nil
	// 	},
	// 	Setter: func(resource interface{}, metaValues *resource.MetaValues, context *qor.Context) {
	// 		return
	// 	},
	// })
	publish.Meta(&admin.Meta{
		Name: "Message",
		Type: "string",
	})
	publish.Meta(&admin.Meta{
		Name: "File",
		Type: "file",
	})

	publishWorker.NewJob(bq, "send mail magazines", "send mail magazines to subscribed users", nil)

	// extraInput := admin.NewResource(&Language{})
	// w.ExtraInput(extraInput)

	// worker.Listen()

	// _ = job
	// if _, err := job.NewQorJob(1, time.Now()); err != nil {
	// 	panic(err)
	// }

	fmt.Println("listening on :8080")
	mux := http.NewServeMux()
	web.MountTo("/admin", mux)
	http.ListenAndServe(":8080", mux)
}
<commit_msg>worker: update test pkg<commit_after>package main

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
	"github.com/qor/qor"
	"github.com/qor/qor/admin"
	"github.com/qor/qor/worker"
)

var db gorm.DB

func init() {
	var err error
	db, err = gorm.Open("sqlite3", "tmp/worker.db")
	if err != nil {
		panic(err)
	}
	db.LogMode(true)
}

func main() {
	config := qor.Config{DB: &db}
	web := admin.New(&config)
	// web.UseResource(user)

	// if err := db.DropTable(&worker.QorJob{}).Error; err != nil {
	// 	panic(err)
	// }
	if err := db.AutoMigrate(&worker.QorJob{}).Error; err != nil {
		panic(err)
	}
	worker.SetJobDB(&db)

	bq := worker.NewBeanstalkdQueue("beanstalkd", "localhost:11300")
	var counter int
	publishWorker := worker.New("Publish Jobs")

	web.AddResource(publishWorker, nil)

	publish := publishWorker.NewJob(bq, "publish products", "publish products so users could purchase new items", func(job *worker.QorJob) (err error) {
		log, err := job.GetLogger()
		if err != nil {
			return
		}

		_, err = log.Write([]byte(strconv.Itoa(counter) + "\n"))
		counter++
		time.Sleep(time.Minute * 5)
		return
	})

	// job.Meta(&admin.Meta{
	// 	Name: "File",
	// 	Type: "file",
	// 	Valuer: func(interface{}, *qor.Context) interface{} {
	// 		return nil
	// 	},
	// 	Setter: func(resource interface{}, metaValues *resource.MetaValues, context *qor.Context) {
	// 		return
	// 	},
	// })
	publish.Meta(&admin.Meta{
		Name: "Message",
		Type: "string",
	})
	publish.Meta(&admin.Meta{
		Name: "File",
		Type: "file",
	})

	publishWorker.NewJob(bq, "send mail magazines", "send mail magazines to subscribed users", nil)

	// extraInput := admin.NewResource(&Language{})
	// w.ExtraInput(extraInput)

	worker.Listen()

	// _ = job
	// if _, err := job.NewQorJob(1, time.Now()); err != nil {
	// 	panic(err)
	// }

	fmt.Println("listening on :8080")
	mux := http.NewServeMux()
	web.MountTo("/admin", mux)
	http.ListenAndServe(":8080", mux)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Vulcan Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cassandra

import (
	"sync"
	"time"

	"github.com/digitalocean/vulcan/model"
	"github.com/gocql/gocql"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	namespace = "vulcan"
	subsystem = "cassandra"
)

// Writer implements ingester.Writer to persist TimeSeriesBatch samples
// to Cassandra.
type Writer struct {
	prometheus.Collector

	s  *gocql.Session
	ch chan *writerPayload

	batchWriteDuration  prometheus.Histogram
	sampleWriteDuration prometheus.Histogram
	workerCount         *prometheus.GaugeVec
}

// Describe implements prometheus.Collector.
func (w *Writer) Describe(ch chan<- *prometheus.Desc) {
	w.batchWriteDuration.Describe(ch)
	w.sampleWriteDuration.Describe(ch)
	w.workerCount.Describe(ch)
}

// Collect implements prometheus.Collector.
func (w *Writer) Collect(ch chan<- prometheus.Metric) {
	w.batchWriteDuration.Collect(ch)
	w.sampleWriteDuration.Collect(ch)
	w.workerCount.Collect(ch)
}

type writerPayload struct {
	wg    *sync.WaitGroup
	ts    *model.TimeSeries
	errch chan error
}

// WriterConfig specifies how many goroutines should be used in writing
// TimeSeries to Cassandra. The Session is expected to be already created
// and ready to use.
type WriterConfig struct {
	NumWorkers int
	Session    *gocql.Session
}

// NewWriter creates a Writer and starts the configured number of
// goroutines to write to Cassandra concurrently.
func NewWriter(config *WriterConfig) *Writer {
	w := &Writer{
		s:  config.Session,
		ch: make(chan *writerPayload),

		batchWriteDuration: prometheus.NewHistogram(
			prometheus.HistogramOpts{
				Namespace: namespace,
				Subsystem: subsystem,
				Name:      "batch_write_duration_seconds",
				Help:      "Histogram of seconds elapsed to write a batch.",
				Buckets:   prometheus.DefBuckets,
			},
		),
		sampleWriteDuration: prometheus.NewHistogram(
			prometheus.HistogramOpts{
				Namespace: namespace,
				Subsystem: subsystem,
				Name:      "sample_write_duration_seconds",
				Help:      "Histogram of seconds elapsed to write a sample.",
				Buckets:   prometheus.DefBuckets,
			},
		),
		workerCount: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: namespace,
				Subsystem: subsystem,
				Name:      "worker_count",
				Help:      "Count of workers.",
			},
			[]string{"mode"},
		),
	}
	for n := 0; n < config.NumWorkers; n++ {
		go w.worker()
	}
	return w
}

// Write implements the ingester.Write interface and allows the
// ingester to write TimeSeriesBatch to Cassandra.
func (w *Writer) Write(tsb model.TimeSeriesBatch) error {
	t0 := time.Now()
	defer func() {
		w.batchWriteDuration.Observe(time.Since(t0).Seconds())
	}()
	wg := &sync.WaitGroup{}
	errch := make(chan error, 1) // room for just the first error a worker encounters
	wg.Add(len(tsb))
	for _, ts := range tsb {
		wp := &writerPayload{
			wg:    wg,
			ts:    ts,
			errch: errch,
		}
		w.ch <- wp
	}
	wg.Wait()
	select {
	case err := <-errch:
		return err
	default:
		return nil
	}
}

func (w *Writer) worker() {
	w.workerCount.WithLabelValues("idle").Inc()
	for m := range w.ch {
		w.workerCount.WithLabelValues("idle").Dec()
		w.workerCount.WithLabelValues("active").Inc()
		id := m.ts.ID()
		for _, s := range m.ts.Samples {
			err := w.write(id, s.TimestampMS, s.Value)
			if err != nil {
				// send error back on payload's errch; don't block the worker
				select {
				case m.errch <- err:
				default:
				}
			}
		}
		m.wg.Done()
		w.workerCount.WithLabelValues("idle").Inc()
		w.workerCount.WithLabelValues("active").Dec()
	}
}

func (w *Writer) write(id string, at int64, value float64) error {
	t0 := time.Now()
	defer func() {
		w.sampleWriteDuration.Observe(time.Since(t0).Seconds())
	}()
	return w.s.Query(writeSampleCQL, value, id, at).Exec()
}
<commit_msg>writes TTL<commit_after>// Copyright 2016 The Vulcan Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cassandra

import (
	"sync"
	"time"

	"github.com/digitalocean/vulcan/model"
	"github.com/gocql/gocql"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	namespace = "vulcan"
	subsystem = "cassandra"

	writeTTLSampleCQL = `UPDATE uncompressed USING TTL ? SET value = ? WHERE fqmn = ? AND at = ?`
)

// Writer implements ingester.Writer to persist TimeSeriesBatch samples
// to Cassandra.
type Writer struct {
	prometheus.Collector

	s          *gocql.Session
	ch         chan *writerPayload
	ttlSeconds int64

	batchWriteDuration  prometheus.Histogram
	sampleWriteDuration prometheus.Histogram
	workerCount         *prometheus.GaugeVec
}

// Describe implements prometheus.Collector.
func (w *Writer) Describe(ch chan<- *prometheus.Desc) {
	w.batchWriteDuration.Describe(ch)
	w.sampleWriteDuration.Describe(ch)
	w.workerCount.Describe(ch)
}

// Collect implements prometheus.Collector.
func (w *Writer) Collect(ch chan<- prometheus.Metric) {
	w.batchWriteDuration.Collect(ch)
	w.sampleWriteDuration.Collect(ch)
	w.workerCount.Collect(ch)
}

type writerPayload struct {
	wg    *sync.WaitGroup
	ts    *model.TimeSeries
	errch chan error
}

// WriterConfig specifies how many goroutines should be used in writing
// TimeSeries to Cassandra. The Session is expected to be already created
// and ready to use.
type WriterConfig struct {
	NumWorkers int
	Session    *gocql.Session
	TTL        time.Duration
}

// NewWriter creates a Writer and starts the configured number of
// goroutines to write to Cassandra concurrently.
func NewWriter(config *WriterConfig) *Writer {
	w := &Writer{
		s:          config.Session,
		ch:         make(chan *writerPayload),
		ttlSeconds: config.TTL.Nanoseconds() / int64(time.Second),

		batchWriteDuration: prometheus.NewHistogram(
			prometheus.HistogramOpts{
				Namespace: namespace,
				Subsystem: subsystem,
				Name:      "batch_write_duration_seconds",
				Help:      "Histogram of seconds elapsed to write a batch.",
				Buckets:   prometheus.DefBuckets,
			},
		),
		sampleWriteDuration: prometheus.NewHistogram(
			prometheus.HistogramOpts{
				Namespace: namespace,
				Subsystem: subsystem,
				Name:      "sample_write_duration_seconds",
				Help:      "Histogram of seconds elapsed to write a sample.",
				Buckets:   prometheus.DefBuckets,
			},
		),
		workerCount: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: namespace,
				Subsystem: subsystem,
				Name:      "worker_count",
				Help:      "Count of workers.",
			},
			[]string{"mode"},
		),
	}
	for n := 0; n < config.NumWorkers; n++ {
		go w.worker()
	}
	return w
}

// Write implements the ingester.Write interface and allows the
// ingester to write TimeSeriesBatch to Cassandra.
func (w *Writer) Write(tsb model.TimeSeriesBatch) error {
	t0 := time.Now()
	defer func() {
		w.batchWriteDuration.Observe(time.Since(t0).Seconds())
	}()
	wg := &sync.WaitGroup{}
	errch := make(chan error, 1) // room for just the first error a worker encounters
	wg.Add(len(tsb))
	for _, ts := range tsb {
		wp := &writerPayload{
			wg:    wg,
			ts:    ts,
			errch: errch,
		}
		w.ch <- wp
	}
	wg.Wait()
	select {
	case err := <-errch:
		return err
	default:
		return nil
	}
}

func (w *Writer) worker() {
	w.workerCount.WithLabelValues("idle").Inc()
	for m := range w.ch {
		w.workerCount.WithLabelValues("idle").Dec()
		w.workerCount.WithLabelValues("active").Inc()
		id := m.ts.ID()
		for _, s := range m.ts.Samples {
			err := w.write(id, s.TimestampMS, s.Value)
			if err != nil {
				// send error back on payload's errch; don't block the worker
				select {
				case m.errch <- err:
				default:
				}
			}
		}
		m.wg.Done()
		w.workerCount.WithLabelValues("idle").Inc()
		w.workerCount.WithLabelValues("active").Dec()
	}
}

func (w *Writer) write(id string, at int64, value float64) error {
	t0 := time.Now()
	defer func() {
		w.sampleWriteDuration.Observe(time.Since(t0).Seconds())
	}()
	return w.s.Query(writeTTLSampleCQL, w.ttlSeconds, value, id, at).Exec()
}
<|endoftext|>


<commit_before>package topRanking

import (
	"fmt"
	"time"
	"web_apps/news_crawlers/modules/database"
	"web_apps/news_crawlers/modules/newsCache"
)

var (
	todayTopRank = []string{"index", "news_top_rank"}
)

// GenerateTopRanking aggregate all news categories
// to index by grouping them and limiting
func GenerateTopRanking(loopDelay int) {
	fmt.Println("GenerateTopRanking starting...")

	for t := range time.Tick(time.Duration(loopDelay) * time.Second) {
		fmt.Println(t)
		idSlice := database.TopNewsRanker()

		key := newsCache.RedisKeyGen(todayTopRank...)
		newsCache.PushIDredisObjectID(key, idSlice...)

	}
}
<commit_msg>update some tweak<commit_after>package topRanking

import (
	"fmt"
	"time"
	"web_apps/news_crawlers/modules/database"
	"web_apps/news_crawlers/modules/newsCache"
)

var (
	// main todayTopRank news key
	todayTopRank = []string{"index", "news_top_rank"}
)

// GenerateTopRanking aggregate all news categories
// to index by grouping them and limiting
func GenerateTopRanking(loopDelay int) {
	fmt.Println("GenerateTopRanking starting...")

	for t := range time.Tick(time.Duration(loopDelay) * time.Second) {
		fmt.Println(t)
		idSlice := database.TopNewsRanker()

		key := newsCache.RedisKeyGen(todayTopRank...)
		newsCache.PushIDredisObjectID(key, idSlice...)
	}
}
<|endoftext|>


<commit_before>package helper

import (
	"testing"
)

type User struct {
	ID      uint     `json:"id" form:"id"`
	Jobs    []*Job   `json:"jobs,omitempty" form:"jobs"`
	Name    string   `json:"name" form:"name"`
	Profile *Profile `json:"profile,omitempty" form:"profile"`
}

type Profile struct {
	ID      uint  `json:"id" form:"id"`
	UserID  uint  `json:"user_id" form:"user_id"`
	User    *User `json:"user" form:"user"`
	Engaged bool  `json:"engaged" form:"engaged"`
}

type Job struct {
	ID     uint  `json:"id" form:"id"`
	UserID uint  `json:"user_id" form:"user_id"`
	User   *User `json:"user" form:"user"`
	RoleCd uint  `json:"role_cd" form:"role_cd"`
}

type Company struct {
	ID           uint              `json:"id,omitempty" form:"id"`
	Name         string            `json:"name,omitempty" form:"name"`
	List         bool              `json:"list,omitempty" form:"list"`
	Subsidiary   []*Company        `json:"company,omitempty" form:"company"`
	Organization map[string]string `json:"organization,omitempty" form:"organization"`
	User         *User             `json:"user,omitempty" form:"user"`
}

func TestQueryFields_Wildcard(t *testing.T) {
	fields := map[string]interface{}{"*": nil}
	result := QueryFields(User{}, fields)
	expected := "*"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_Primitive(t *testing.T) {
	fields := map[string]interface{}{"name": nil}
	result := QueryFields(User{}, fields)
	expected := "name"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_Multiple(t *testing.T) {
	fields := map[string]interface{}{"id": nil, "name": nil}
	result := QueryFields(User{}, fields)
	expected := "id,name"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_BelongsTo(t *testing.T) {
	fields := map[string]interface{}{"user": nil}
	result := QueryFields(Profile{}, fields)
	expected := "user_id"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_HasOne(t *testing.T) {
	fields := map[string]interface{}{"profile": nil}
	result := QueryFields(User{}, fields)
	expected := "id"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_HasMany(t *testing.T) {
	fields := map[string]interface{}{"jobs": nil}
	result := QueryFields(User{}, fields)
	expected := "id"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestParseFields_Wildcard(t *testing.T) {
	fields := "*"
	result := ParseFields(fields)

	if _, ok := result["*"]; !ok {
		t.Fatalf("result[*] should exist: %#v", result)
	}

	if result["*"] != nil {
		t.Fatalf("result[*] should be nil: %#v", result)
	}
}

func TestParseFields_Flat(t *testing.T) {
	fields := "profile"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if result["profile"] != nil {
		t.Fatalf("result[profile] should be nil: %#v", result)
	}
}

func TestParseFields_Nested(t *testing.T) {
	fields := "profile.nation"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"]; !ok {
		t.Fatalf("result[profile][nation] should exist: %#v", result)
	}

	if result["profile"].(map[string]interface{})["nation"] != nil {
		t.Fatalf("result[profile][nation] should be nil: %#v", result)
	}
}

func TestParseFields_NestedDeeply(t *testing.T) {
	fields := "profile.nation.name"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"]; !ok {
		t.Fatalf("result[profile][nation] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile][nation] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"]; !ok {
		t.Fatalf("result[profile][nation][name] should exist: %#v", result)
	}

	if result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"] != nil {
		t.Fatalf("result[profile][nation][name] should be nil: %#v", result)
	}
}

func TestParseFields_MultipleFields(t *testing.T) {
	fields := "profile.nation.name,emails"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"]; !ok {
		t.Fatalf("result[profile][nation] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile][nation] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"]; !ok {
		t.Fatalf("result[profile][nation][name] should exist: %#v", result)
	}

	if result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"] != nil {
		t.Fatalf("result[profile][nation][name] should be nil: %#v", result)
	}

	if _, ok := result["emails"]; !ok {
		t.Fatalf("result[emails] should exist: %#v", result)
	}

	if result["emails"] != nil {
		t.Fatalf("result[emails] should be map: %#v", result)
	}
}

func TestParseFields_Included(t *testing.T) {
	fields := "profile.nation.name,profile"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if result["profile"] != nil {
		t.Fatalf("result[profile] should be nil: %#v", result)
	}
}

var profile = Profile{
	ID:      1,
	UserID:  1,
	User:    nil,
	Engaged: true,
}

var job = Job{
	ID:     1,
	UserID: 1,
	User:   nil,
	RoleCd: 1,
}

func TestFieldToMap_Wildcard(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    []*Job{&job},
		Name:    "Taro Yamada",
		Profile: &profile,
	}

	fields := map[string]interface{}{
		"*": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "jobs", "name", "profile"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	if result["jobs"].([]*Job) == nil {
		t.Fatalf("jobs should not be nil. actual: %#v", result["jobs"])
	}

	if result["profile"].(*Profile) == nil {
		t.Fatalf("profile should not be nil. actual: %#v", result["profile"])
	}
}

func TestFieldToMap_OmitEmpty(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    nil,
		Name:    "Taro Yamada",
		Profile: nil,
	}

	fields := map[string]interface{}{
		"*": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"jobs", "profile"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_OmitEmptyWithField(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    nil,
		Name:    "Taro Yamada",
		Profile: nil,
	}

	fields := map[string]interface{}{
		"id":   nil,
		"name": nil,
		"jobs": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name", "jobs"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"profile"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_OmitEmptyAllTypes(t *testing.T) {
	company := Company{
		ID:           0,
		Name:         "",
		List:         false,
		Subsidiary:   []*Company{},
		Organization: make(map[string]string),
		User:         nil,
	}

	fields := map[string]interface{}{
		"*": nil,
	}
	result, err := FieldToMap(company, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name", "list", "subsidiary", "organization", "user"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_SpecifyField(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    nil,
		Name:    "Taro Yamada",
		Profile: nil,
	}

	fields := map[string]interface{}{
		"id":   nil,
		"name": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"jobs", "profile"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_NestedField(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    []*Job{&job},
		Name:    "Taro Yamada",
		Profile: &profile,
	}

	fields := map[string]interface{}{
		"profile": map[string]interface{}{
			"id": nil,
		},
		"name": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"name", "profile"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"id", "jobs"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}

	if result["profile"].(map[string]interface{}) == nil {
		t.Fatalf("profile should not be nil. actual: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["id"]; !ok {
		t.Fatalf("profile.id should exist. actual: %#v", result)
	}

	for _, key := range []string{"id"} {
		if _, ok := result["profile"].(map[string]interface{})[key]; !ok {
			t.Fatalf("profile.%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"user_id", "user", "engaged"} {
		if _, ok := result["profile"].(map[string]interface{})[key]; ok {
			t.Fatalf("profile.%s should not exist. actual: %#v", key, result)
		}
	}
}
<commit_msg>Fix the filter testcase to be regardless of field sequence<commit_after>package helper

import (
	"testing"
)

type User struct {
	ID      uint     `json:"id" form:"id"`
	Jobs    []*Job   `json:"jobs,omitempty" form:"jobs"`
	Name    string   `json:"name" form:"name"`
	Profile *Profile `json:"profile,omitempty" form:"profile"`
}

type Profile struct {
	ID      uint  `json:"id" form:"id"`
	UserID  uint  `json:"user_id" form:"user_id"`
	User    *User `json:"user" form:"user"`
	Engaged bool  `json:"engaged" form:"engaged"`
}

type Job struct {
	ID     uint  `json:"id" form:"id"`
	UserID uint  `json:"user_id" form:"user_id"`
	User   *User `json:"user" form:"user"`
	RoleCd uint  `json:"role_cd" form:"role_cd"`
}

type Company struct {
	ID           uint              `json:"id,omitempty" form:"id"`
	Name         string            `json:"name,omitempty" form:"name"`
	List         bool              `json:"list,omitempty" form:"list"`
	Subsidiary   []*Company        `json:"company,omitempty" form:"company"`
	Organization map[string]string `json:"organization,omitempty" form:"organization"`
	User         *User             `json:"user,omitempty" form:"user"`
}

func TestQueryFields_Wildcard(t *testing.T) {
	fields := map[string]interface{}{"*": nil}
	result := QueryFields(User{}, fields)
	expected := "*"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_Primitive(t *testing.T) {
	fields := map[string]interface{}{"name": nil}
	result := QueryFields(User{}, fields)
	expected := "name"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_Multiple(t *testing.T) {
	fields := map[string]interface{}{"id": nil, "name": nil}
	result := QueryFields(User{}, fields)
	expected1 := "id,name"
	expected2 := "name,id"

	if result != expected1 && result != expected2 {
		t.Fatalf("result should be %s or %s. actual: %s", expected1, expected2, result)
	}
}

func TestQueryFields_BelongsTo(t *testing.T) {
	fields := map[string]interface{}{"user": nil}
	result := QueryFields(Profile{}, fields)
	expected := "user_id"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_HasOne(t *testing.T) {
	fields := map[string]interface{}{"profile": nil}
	result := QueryFields(User{}, fields)
	expected := "id"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestQueryFields_HasMany(t *testing.T) {
	fields := map[string]interface{}{"jobs": nil}
	result := QueryFields(User{}, fields)
	expected := "id"

	if result != expected {
		t.Fatalf("result should be %s. actual: %s", expected, result)
	}
}

func TestParseFields_Wildcard(t *testing.T) {
	fields := "*"
	result := ParseFields(fields)

	if _, ok := result["*"]; !ok {
		t.Fatalf("result[*] should exist: %#v", result)
	}

	if result["*"] != nil {
		t.Fatalf("result[*] should be nil: %#v", result)
	}
}

func TestParseFields_Flat(t *testing.T) {
	fields := "profile"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if result["profile"] != nil {
		t.Fatalf("result[profile] should be nil: %#v", result)
	}
}

func TestParseFields_Nested(t *testing.T) {
	fields := "profile.nation"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"]; !ok {
		t.Fatalf("result[profile][nation] should exist: %#v", result)
	}

	if result["profile"].(map[string]interface{})["nation"] != nil {
		t.Fatalf("result[profile][nation] should be nil: %#v", result)
	}
}

func TestParseFields_NestedDeeply(t *testing.T) {
	fields := "profile.nation.name"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"]; !ok {
		t.Fatalf("result[profile][nation] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile][nation] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"]; !ok {
		t.Fatalf("result[profile][nation][name] should exist: %#v", result)
	}

	if result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"] != nil {
		t.Fatalf("result[profile][nation][name] should be nil: %#v", result)
	}
}

func TestParseFields_MultipleFields(t *testing.T) {
	fields := "profile.nation.name,emails"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"]; !ok {
		t.Fatalf("result[profile][nation] should exist: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{}); !ok {
		t.Fatalf("result[profile][nation] should be map: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"]; !ok {
		t.Fatalf("result[profile][nation][name] should exist: %#v", result)
	}

	if result["profile"].(map[string]interface{})["nation"].(map[string]interface{})["name"] != nil {
		t.Fatalf("result[profile][nation][name] should be nil: %#v", result)
	}

	if _, ok := result["emails"]; !ok {
		t.Fatalf("result[emails] should exist: %#v", result)
	}

	if result["emails"] != nil {
		t.Fatalf("result[emails] should be map: %#v", result)
	}
}

func TestParseFields_Included(t *testing.T) {
	fields := "profile.nation.name,profile"
	result := ParseFields(fields)

	if _, ok := result["profile"]; !ok {
		t.Fatalf("result[profile] should exist: %#v", result)
	}

	if result["profile"] != nil {
		t.Fatalf("result[profile] should be nil: %#v", result)
	}
}

var profile = Profile{
	ID:      1,
	UserID:  1,
	User:    nil,
	Engaged: true,
}

var job = Job{
	ID:     1,
	UserID: 1,
	User:   nil,
	RoleCd: 1,
}

func TestFieldToMap_Wildcard(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    []*Job{&job},
		Name:    "Taro Yamada",
		Profile: &profile,
	}

	fields := map[string]interface{}{
		"*": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "jobs", "name", "profile"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	if result["jobs"].([]*Job) == nil {
		t.Fatalf("jobs should not be nil. actual: %#v", result["jobs"])
	}

	if result["profile"].(*Profile) == nil {
		t.Fatalf("profile should not be nil. actual: %#v", result["profile"])
	}
}

func TestFieldToMap_OmitEmpty(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    nil,
		Name:    "Taro Yamada",
		Profile: nil,
	}

	fields := map[string]interface{}{
		"*": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"jobs", "profile"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_OmitEmptyWithField(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    nil,
		Name:    "Taro Yamada",
		Profile: nil,
	}

	fields := map[string]interface{}{
		"id":   nil,
		"name": nil,
		"jobs": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name", "jobs"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"profile"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_OmitEmptyAllTypes(t *testing.T) {
	company := Company{
		ID:           0,
		Name:         "",
		List:         false,
		Subsidiary:   []*Company{},
		Organization: make(map[string]string),
		User:         nil,
	}

	fields := map[string]interface{}{
		"*": nil,
	}
	result, err := FieldToMap(company, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name", "list", "subsidiary", "organization", "user"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_SpecifyField(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    nil,
		Name:    "Taro Yamada",
		Profile: nil,
	}

	fields := map[string]interface{}{
		"id":   nil,
		"name": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"id", "name"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"jobs", "profile"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}
}

func TestFieldToMap_NestedField(t *testing.T) {
	user := User{
		ID:      1,
		Jobs:    []*Job{&job},
		Name:    "Taro Yamada",
		Profile: &profile,
	}

	fields := map[string]interface{}{
		"profile": map[string]interface{}{
			"id": nil,
		},
		"name": nil,
	}
	result, err := FieldToMap(user, fields)

	if err != nil {
		t.Fatalf("FieldToMap return an error. detail: %#v", err.Error())
	}

	for _, key := range []string{"name", "profile"} {
		if _, ok := result[key]; !ok {
			t.Fatalf("%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"id", "jobs"} {
		if _, ok := result[key]; ok {
			t.Fatalf("%s should not exist. actual: %#v", key, result)
		}
	}

	if result["profile"].(map[string]interface{}) == nil {
		t.Fatalf("profile should not be nil. actual: %#v", result)
	}

	if _, ok := result["profile"].(map[string]interface{})["id"]; !ok {
		t.Fatalf("profile.id should exist. actual: %#v", result)
	}

	for _, key := range []string{"id"} {
		if _, ok := result["profile"].(map[string]interface{})[key]; !ok {
			t.Fatalf("profile.%s should exist. actual: %#v", key, result)
		}
	}

	for _, key := range []string{"user_id", "user", "engaged"} {
		if _, ok := result["profile"].(map[string]interface{})[key]; ok {
			t.Fatalf("profile.%s should not exist. actual: %#v", key, result)
		}
	}
}
<|endoftext|>


<commit_before>package class

import cf "jvmgo/classfile"

type Constant interface{}

type ConstantPool struct {
    consts []Constant
}

func (self *ConstantPool) GetConstant(index uint) (Constant) {
    // todo
    return self.consts[index]
}

func newConstantPool(cfCp * cf.ConstantPool) {
    cpInfos := cfCp.Infos()
    consts := make([]Constant, len(cpInfos))
    for i, cpInfo := range cpInfos {
        if i > 0 {
            switch cpInfo.(type) {
            case *cf.ConstantIntegerInfo:
                cInt := cpInfo.(*cf.ConstantIntegerInfo)
                consts[i] = cInt.Value()
            case *cf.ConstantFloatInfo:
                cFloat := cpInfo.(*cf.ConstantFloatInfo)
                consts[i] = cFloat.Value()
            case *cf.ConstantLongInfo:
                cLong := cpInfo.(*cf.ConstantLongInfo)
                consts[i] = cLong.Value()
            case *cf.ConstantDoubleInfo:
                cDouble := cpInfo.(*cf.ConstantDoubleInfo)
                consts[i] = cDouble.Value()
            // todo
            }
        }
    }
}
<commit_msg>code refactor<commit_after>package class

import cf "jvmgo/classfile"

type Constant interface{}

type ConstantPool struct {
    consts []Constant
}

func (self *ConstantPool) GetConstant(index uint) (Constant) {
    // todo
    return self.consts[index]
}

func newConstantPool(cfCp * cf.ConstantPool) {
    cpInfos := cfCp.Infos()
    consts := make([]Constant, len(cpInfos))
    for i := 1; i < len(cpInfos); i++ {
        cpInfo := cpInfos[i]
        switch cpInfo.(type) {
        case *cf.ConstantIntegerInfo:
            cInt := cpInfo.(*cf.ConstantIntegerInfo)
            consts[i] = cInt.Value()
        case *cf.ConstantFloatInfo:
            cFloat := cpInfo.(*cf.ConstantFloatInfo)
            consts[i] = cFloat.Value()
        case *cf.ConstantLongInfo:
            cLong := cpInfo.(*cf.ConstantLongInfo)
            consts[i] = cLong.Value()
            i++
        case *cf.ConstantDoubleInfo:
            cDouble := cpInfo.(*cf.ConstantDoubleInfo)
            consts[i] = cDouble.Value()
            i++
        // todo
        }
    }
}
<|endoftext|>


<commit_before>package quic

import (
	"context"
	"fmt"
	"io"
	"net"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/qerr"
	"github.com/simia-tech/netx"
	"github.com/simia-tech/netx/value"
)

type conn struct {
	session quic.Session
	stream  quic.Stream
}

func init() {
	netx.RegisterDial("quic", Dial)
}

// Dial opens a connection to the provided address.
func Dial(ctx context.Context, address string, options *value.Options) (net.Conn, error) {
	session, err := quic.DialAddrContext(ctx, address, options.TLSConfig, nil)
	if err != nil {
		return nil, err
	}

	stream, err := session.OpenStreamSync()
	if err != nil {
		return nil, err
	}

	return &conn{
		session: session,
		stream:  stream,
	}, nil
}

func (c *conn) Read(data []byte) (int, error) {
	n, err := c.stream.Read(data)
	if qErr, ok := err.(*qerr.QuicError); ok {
		switch qErr.ErrorCode {
		case qerr.PeerGoingAway:
			err = io.EOF
		case qerr.NetworkIdleTimeout:
			err = fmt.Errorf("read timeout")
		}
	}
	if err != nil && err.Error() == "deadline exceeded" {
		err = fmt.Errorf("read timeout")
	}
	return n, err
}

func (c *conn) Write(data []byte) (int, error) {
	return c.stream.Write(data)
}

func (c *conn) Close() error {
	if c.stream == nil {
		return nil
	}
	if err := c.stream.Close(); err != nil {
		return err
	}
	if c.session == nil {
		return nil
	}
	if err := c.session.Close(nil); err != nil {
		return err
	}
	return nil
}

func (c *conn) LocalAddr() net.Addr {
	if c.session == nil {
		return nil
	}
	return c.session.LocalAddr()
}

func (c *conn) RemoteAddr() net.Addr {
	if c.session == nil {
		return nil
	}
	return c.session.RemoteAddr()
}

func (c *conn) SetDeadline(t time.Time) error {
	return c.stream.SetDeadline(t)
}

func (c *conn) SetReadDeadline(t time.Time) error {
	return c.stream.SetReadDeadline(t)
}

func (c *conn) SetWriteDeadline(t time.Time) error {
	return c.stream.SetWriteDeadline(t)
}

func (c *conn) String() string {
	return fmt.Sprintf("(%s -> %s)", c.LocalAddr(), c.RemoteAddr())
}
<commit_msg>added quic connection close workaround<commit_after>package quic

import (
	"context"
	"fmt"
	"io"
	"net"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/qerr"
	"github.com/simia-tech/netx"
	"github.com/simia-tech/netx/value"
)

type conn struct {
	session quic.Session
	stream  quic.Stream
	conn    *net.UDPConn
}

func init() {
	netx.RegisterDial("quic", Dial)
}

// Dial opens a connection to the provided address.
func Dial(ctx context.Context, address string, options *value.Options) (net.Conn, error) {
	udpAddr, err := net.ResolveUDPAddr("udp", address)
	if err != nil {
		return nil, err
	}

	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return nil, err
	}

	session, err := quic.DialContext(ctx, udpConn, udpAddr, address, options.TLSConfig, nil)
	if err != nil {
		return nil, err
	}

	stream, err := session.OpenStreamSync()
	if err != nil {
		return nil, err
	}

	return &conn{
		session: session,
		stream:  stream,
		conn:    udpConn,
	}, nil
}

func (c *conn) Read(data []byte) (int, error) {
	n, err := c.stream.Read(data)
	if qErr, ok := err.(*qerr.QuicError); ok {
		switch qErr.ErrorCode {
		case qerr.PeerGoingAway:
			err = io.EOF
		case qerr.NetworkIdleTimeout:
			err = fmt.Errorf("read timeout")
		}
	}
	if err != nil && err.Error() == "deadline exceeded" {
		err = fmt.Errorf("read timeout")
	}
	return n, err
}

func (c *conn) Write(data []byte) (int, error) {
	return c.stream.Write(data)
}

func (c *conn) Close() error {
	if c.stream == nil {
		return nil
	}
	if err := c.stream.Close(); err != nil {
		return err
	}
	if c.session == nil {
		return nil
	}
	if err := c.session.Close(nil); err != nil {
		return err
	}
	if c.conn == nil {
		return nil
	}
	if err := c.conn.Close(); err != nil {
		return err
	}
	return nil
}

func (c *conn) LocalAddr() net.Addr {
	if c.session == nil {
		return nil
	}
	return c.session.LocalAddr()
}

func (c *conn) RemoteAddr() net.Addr {
	if c.session == nil {
		return nil
	}
	return c.session.RemoteAddr()
}

func (c *conn) SetDeadline(t time.Time) error {
	return c.stream.SetDeadline(t)
}

func (c *conn) SetReadDeadline(t time.Time) error {
	return c.stream.SetReadDeadline(t)
}

func (c *conn) SetWriteDeadline(t time.Time) error {
	return c.stream.SetWriteDeadline(t)
}

func (c *conn) String() string {
	return fmt.Sprintf("(%s -> %s)", c.LocalAddr(), c.RemoteAddr())
}
<|endoftext|>