<commit_before>package ml

import (
	"github.com/plandem/ooxml/ml"
	"github.com/plandem/xlsx/internal/ml/primitives"
)

//Worksheet is a direct mapping of XSD CT_Worksheet
type Worksheet struct {
	XMLName               ml.Name         `xml:"http://schemas.openxmlformats.org/spreadsheetml/2006/main worksheet"`
	SheetPr               *ml.Reserved    `xml:"sheetPr,omitempty"`
	Dimension             *SheetDimension `xml:"dimension,omitempty"`
	SheetViews            *SheetViews     `xml:"sheetViews,omitempty"`
	SheetFormatPr         *ml.Reserved    `xml:"sheetFormatPr,omitempty"`
	Cols                  *[]*Col         `xml:"cols>col,omitempty"` //we HAVE TO remove 'cols' if there is no any 'col'
	SheetData             []*Row          `xml:"sheetData>row"`
	SheetCalcPr           *ml.Reserved    `xml:"sheetCalcPr,omitempty"`
	SheetProtection       *ml.Reserved    `xml:"sheetProtection,omitempty"`
	ProtectedRanges       *ml.Reserved    `xml:"protectedRanges,omitempty"`
	Scenarios             *ml.Reserved    `xml:"scenarios,omitempty"`
	AutoFilter            *ml.Reserved    `xml:"autoFilter,omitempty"`
	SortState             *ml.Reserved    `xml:"sortState,omitempty"`
	DataConsolidate       *ml.Reserved    `xml:"dataConsolidate,omitempty"`
	CustomSheetViews      *ml.Reserved    `xml:"customSheetViews,omitempty"`
	MergeCells            *[]*MergeCell   `xml:"mergeCells>mergeCell,omitempty"`
	PhoneticPr            *ml.Reserved    `xml:"phoneticPr,omitempty"`
	ConditionalFormatting *ml.Reserved    `xml:"conditionalFormatting,omitempty"`
	DataValidations       *ml.Reserved    `xml:"dataValidations,omitempty"`
	Hyperlinks            *ml.Reserved    `xml:"hyperlinks,omitempty"`
	PrintOptions          *ml.Reserved    `xml:"printOptions,omitempty"`
	PageMargins           *ml.Reserved    `xml:"pageMargins,omitempty"`
	PageSetup             *ml.Reserved    `xml:"pageSetup,omitempty"`
	HeaderFooter          *ml.Reserved    `xml:"headerFooter,omitempty"`
	RowBreaks             *ml.Reserved    `xml:"rowBreaks,omitempty"`
	ColBreaks             *ml.Reserved    `xml:"colBreaks,omitempty"`
	CustomProperties      *ml.Reserved    `xml:"customProperties,omitempty"`
	CellWatches           *ml.Reserved    `xml:"cellWatches,omitempty"`
	IgnoredErrors         *ml.Reserved    `xml:"ignoredErrors,omitempty"`
	SmartTags             *ml.Reserved    `xml:"smartTags,omitempty"`
	Drawing               *ml.Reserved    `xml:"drawing,omitempty"`
	DrawingHF             *ml.Reserved    `xml:"drawingHF,omitempty"`
	Picture               *ml.Reserved    `xml:"picture,omitempty"`
	OleObjects            *ml.Reserved    `xml:"oleObjects,omitempty"`
	Controls              *ml.Reserved    `xml:"controls,omitempty"`
	WebPublishItems       *ml.Reserved    `xml:"webPublishItems,omitempty"`
	TableParts            *ml.Reserved    `xml:"tableParts,omitempty"`
	ExtLst                *ml.Reserved    `xml:"extLst,omitempty"`
}

//SheetDimension is a direct mapping of XSD CT_SheetDimension
type SheetDimension struct {
	Bounds primitives.Bounds `xml:"ref,attr"`
}

//Col is a direct mapping of XSD CT_Col
type Col struct {
	Min          int     `xml:"min,attr"`
	Max          int     `xml:"max,attr"`
	Width        float32 `xml:"width,attr,omitempty"`
	Style        StyleID `xml:"style,attr,omitempty"`
	Hidden       bool    `xml:"hidden,attr,omitempty"`
	BestFit      bool    `xml:"bestFit,attr,omitempty"`
	CustomWidth  bool    `xml:"customWidth,attr,omitempty"`
	Phonetic     bool    `xml:"phonetic,attr,omitempty"`
	OutlineLevel uint8   `xml:"outlineLevel,attr,omitempty"`
	Collapsed    bool    `xml:"collapsed,attr,omitempty"`
	Updated      bool //for internal usage only
}

//Row is a direct mapping of XSD CT_Row
type Row struct {
	Cells        []*Cell      `xml:"c"`
	ExtLst       *ml.Reserved `xml:"extLst,omitempty"`
	Ref          int          `xml:"r,attr,omitempty"` //1-based index
	Spans        string       `xml:"spans,attr,omitempty"`
	Style        StyleID      `xml:"s,attr,omitempty"`
	CustomFormat bool         `xml:"customFormat,attr,omitempty"`
	Height       float32      `xml:"ht,attr,omitempty"`
	Hidden       bool         `xml:"hidden,attr,omitempty"`
	CustomHeight bool         `xml:"customHeight,attr,omitempty"`
	OutlineLevel uint8        `xml:"outlineLevel,attr,omitempty"`
	Collapsed    bool         `xml:"collapsed,attr,omitempty"`
	ThickTop     bool         `xml:"thickTop,attr,omitempty"`
	ThickBot     bool         `xml:"thickBot,attr,omitempty"`
	Phonetic     bool         `xml:"ph,attr,omitempty"`
}

//Cell is a direct mapping of XSD CT_Cell
type Cell struct {
	Formula   *CellFormula        `xml:"f,omitempty"`
	Value     string              `xml:"v,omitempty"`
	InlineStr *StringItem         `xml:"is,omitempty"`
	ExtLst    *ml.Reserved        `xml:"extLst,omitempty"`
	Ref       primitives.CellRef  `xml:"r,attr"`
	Style     StyleID             `xml:"s,attr,omitempty"`
	Type      primitives.CellType `xml:"t,attr,omitempty"`
	Cm        ml.OptionalIndex    `xml:"cm,attr,omitempty"`
	Vm        ml.OptionalIndex    `xml:"vm,attr,omitempty"`
	Ph        bool                `xml:"ph,attr,omitempty"`
}

//CellFormula is a direct mapping of XSD CT_CellFormula
type CellFormula struct {
	Content string                     `xml:",chardata"`
	T       primitives.CellFormulaType `xml:"t,attr,omitempty"` //default 'normal'
	Aca     bool                       `xml:"aca,attr,omitempty"`
	Bounds  primitives.Bounds          `xml:"ref,attr,omitempty"`
	Dt2D    bool                       `xml:"dt2D,attr,omitempty"`
	Dtr     bool                       `xml:"dtr,attr,omitempty"`
	Del1    bool                       `xml:"del1,attr,omitempty"`
	Del2    bool                       `xml:"del2,attr,omitempty"`
	R1      primitives.CellRef         `xml:"r1,attr,omitempty"`
	R2      primitives.CellRef         `xml:"r2,attr,omitempty"`
	Ca      bool                       `xml:"ca,attr,omitempty"`
	Si      ml.OptionalIndex           `xml:"si,attr,omitempty"`
	Bx      bool                       `xml:"bx,attr,omitempty"`
}

//MergeCell is a direct mapping of XSD CT_MergeCell
type MergeCell struct {
	Bounds primitives.Bounds `xml:"ref,attr"`
}

//SheetViews is a direct mapping of XSD CT_SheetViews
type SheetViews struct {
	SheetView []*SheetView `xml:"sheetView,omitempty"`
	ExtLst    *ml.Reserved `xml:"extLst,omitempty"`
}

//SheetView is a direct mapping of XSD CT_SheetView
type SheetView struct {
	Pane                     *ml.Reserved       `xml:"pane,omitempty"`
	Selection                *ml.Reserved       `xml:"selection,omitempty"`
	PivotSelection           *ml.Reserved       `xml:"pivotSelection,omitempty"`
	ExtLst                   *ml.Reserved       `xml:"extLst,omitempty"`
	WindowProtection         bool               `xml:"windowProtection,attr,omitempty"`
	ShowFormulas             bool               `xml:"showFormulas,attr,omitempty"`
	ShowGridLines            bool               `xml:"showGridLines,attr,omitempty"`
	ShowRowColHeaders        bool               `xml:"showRowColHeaders,attr,omitempty"`
	ShowZeros                bool               `xml:"showZeros,attr,omitempty"`
	RightToLeft              bool               `xml:"rightToLeft,attr,omitempty"`
	TabSelected              bool               `xml:"tabSelected,attr,omitempty"`
	ShowRuler                bool               `xml:"showRuler,attr,omitempty"`
	ShowOutlineSymbols       bool               `xml:"showOutlineSymbols,attr,omitempty"`
	DefaultGridColor         bool               `xml:"defaultGridColor,attr,omitempty"`
	ShowWhiteSpace           bool               `xml:"showWhiteSpace,attr,omitempty"`
	View                     string             `xml:"view,attr,omitempty"` //ST_SheetViewType
	TopLeftCell              primitives.CellRef `xml:"topLeftCell,attr,omitempty"`
	ColorId                  uint               `xml:"colorId,attr,omitempty"`
	ZoomScale                uint               `xml:"zoomScale,attr,omitempty"`
	ZoomScaleNormal          uint               `xml:"zoomScaleNormal,attr,omitempty"`
	ZoomScaleSheetLayoutView uint               `xml:"zoomScaleSheetLayoutView,attr,omitempty"`
	ZoomScalePageLayoutView  uint               `xml:"zoomScalePageLayoutView,attr,omitempty"`
	WorkbookViewId           uint               `xml:"workbookViewId,attr"`
}
<commit_msg>fix cols markup<commit_after>package ml

import (
	"github.com/plandem/ooxml/ml"
	"github.com/plandem/xlsx/internal/ml/primitives"
)

//Worksheet is a direct mapping of XSD CT_Worksheet
type Worksheet struct {
	XMLName               ml.Name         `xml:"http://schemas.openxmlformats.org/spreadsheetml/2006/main worksheet"`
	SheetPr               *ml.Reserved    `xml:"sheetPr,omitempty"`
	Dimension             *SheetDimension `xml:"dimension,omitempty"`
	SheetViews            *SheetViews     `xml:"sheetViews,omitempty"`
	SheetFormatPr         *ml.Reserved    `xml:"sheetFormatPr,omitempty"`
	Cols                  *[]*Col         `xml:"cols>col,omitempty"` //we HAVE TO remove 'cols' if there is no any 'col'
	SheetData             []*Row          `xml:"sheetData>row"`
	SheetCalcPr           *ml.Reserved    `xml:"sheetCalcPr,omitempty"`
	SheetProtection       *ml.Reserved    `xml:"sheetProtection,omitempty"`
	ProtectedRanges       *ml.Reserved    `xml:"protectedRanges,omitempty"`
	Scenarios             *ml.Reserved    `xml:"scenarios,omitempty"`
	AutoFilter            *ml.Reserved    `xml:"autoFilter,omitempty"`
	SortState             *ml.Reserved    `xml:"sortState,omitempty"`
	DataConsolidate       *ml.Reserved    `xml:"dataConsolidate,omitempty"`
	CustomSheetViews      *ml.Reserved    `xml:"customSheetViews,omitempty"`
	MergeCells            *[]*MergeCell   `xml:"mergeCells>mergeCell,omitempty"`
	PhoneticPr            *ml.Reserved    `xml:"phoneticPr,omitempty"`
	ConditionalFormatting *ml.Reserved    `xml:"conditionalFormatting,omitempty"`
	DataValidations       *ml.Reserved    `xml:"dataValidations,omitempty"`
	Hyperlinks            *ml.Reserved    `xml:"hyperlinks,omitempty"`
	PrintOptions          *ml.Reserved    `xml:"printOptions,omitempty"`
	PageMargins           *ml.Reserved    `xml:"pageMargins,omitempty"`
	PageSetup             *ml.Reserved    `xml:"pageSetup,omitempty"`
	HeaderFooter          *ml.Reserved    `xml:"headerFooter,omitempty"`
	RowBreaks             *ml.Reserved    `xml:"rowBreaks,omitempty"`
	ColBreaks             *ml.Reserved    `xml:"colBreaks,omitempty"`
	CustomProperties      *ml.Reserved    `xml:"customProperties,omitempty"`
	CellWatches           *ml.Reserved    `xml:"cellWatches,omitempty"`
	IgnoredErrors         *ml.Reserved    `xml:"ignoredErrors,omitempty"`
	SmartTags             *ml.Reserved    `xml:"smartTags,omitempty"`
	Drawing               *ml.Reserved    `xml:"drawing,omitempty"`
	DrawingHF             *ml.Reserved    `xml:"drawingHF,omitempty"`
	Picture               *ml.Reserved    `xml:"picture,omitempty"`
	OleObjects            *ml.Reserved    `xml:"oleObjects,omitempty"`
	Controls              *ml.Reserved    `xml:"controls,omitempty"`
	WebPublishItems       *ml.Reserved    `xml:"webPublishItems,omitempty"`
	TableParts            *ml.Reserved    `xml:"tableParts,omitempty"`
	ExtLst                *ml.Reserved    `xml:"extLst,omitempty"`
}

//SheetDimension is a direct mapping of XSD CT_SheetDimension
type SheetDimension struct {
	Bounds primitives.Bounds `xml:"ref,attr"`
}

//Col is a direct mapping of XSD CT_Col
type Col struct {
	Min          int     `xml:"min,attr"`
	Max          int     `xml:"max,attr"`
	Width        float32 `xml:"width,attr,omitempty"`
	Style        StyleID `xml:"style,attr,omitempty"`
	Hidden       bool    `xml:"hidden,attr,omitempty"`
	BestFit      bool    `xml:"bestFit,attr,omitempty"`
	CustomWidth  bool    `xml:"customWidth,attr,omitempty"`
	Phonetic     bool    `xml:"phonetic,attr,omitempty"`
	OutlineLevel uint8   `xml:"outlineLevel,attr,omitempty"`
	Collapsed    bool    `xml:"collapsed,attr,omitempty"`
	Updated      bool    `xml:"-"` //for internal usage only
}

//Row is a direct mapping of XSD CT_Row
type Row struct {
	Cells        []*Cell      `xml:"c"`
	ExtLst       *ml.Reserved `xml:"extLst,omitempty"`
	Ref          int          `xml:"r,attr,omitempty"` //1-based index
	Spans        string       `xml:"spans,attr,omitempty"`
	Style        StyleID      `xml:"s,attr,omitempty"`
	CustomFormat bool         `xml:"customFormat,attr,omitempty"`
	Height       float32      `xml:"ht,attr,omitempty"`
	Hidden       bool         `xml:"hidden,attr,omitempty"`
	CustomHeight bool         `xml:"customHeight,attr,omitempty"`
	OutlineLevel uint8        `xml:"outlineLevel,attr,omitempty"`
	Collapsed    bool         `xml:"collapsed,attr,omitempty"`
	ThickTop     bool         `xml:"thickTop,attr,omitempty"`
	ThickBot     bool         `xml:"thickBot,attr,omitempty"`
	Phonetic     bool         `xml:"ph,attr,omitempty"`
}

//Cell is a direct mapping of XSD CT_Cell
type Cell struct {
	Formula   *CellFormula        `xml:"f,omitempty"`
	Value     string              `xml:"v,omitempty"`
	InlineStr *StringItem         `xml:"is,omitempty"`
	ExtLst    *ml.Reserved        `xml:"extLst,omitempty"`
	Ref       primitives.CellRef  `xml:"r,attr"`
	Style     StyleID             `xml:"s,attr,omitempty"`
	Type      primitives.CellType `xml:"t,attr,omitempty"`
	Cm        ml.OptionalIndex    `xml:"cm,attr,omitempty"`
	Vm        ml.OptionalIndex    `xml:"vm,attr,omitempty"`
	Ph        bool                `xml:"ph,attr,omitempty"`
}

//CellFormula is a direct mapping of XSD CT_CellFormula
type CellFormula struct {
	Content string                     `xml:",chardata"`
	T       primitives.CellFormulaType `xml:"t,attr,omitempty"` //default 'normal'
	Aca     bool                       `xml:"aca,attr,omitempty"`
	Bounds  primitives.Bounds          `xml:"ref,attr,omitempty"`
	Dt2D    bool                       `xml:"dt2D,attr,omitempty"`
	Dtr     bool                       `xml:"dtr,attr,omitempty"`
	Del1    bool                       `xml:"del1,attr,omitempty"`
	Del2    bool                       `xml:"del2,attr,omitempty"`
	R1      primitives.CellRef         `xml:"r1,attr,omitempty"`
	R2      primitives.CellRef         `xml:"r2,attr,omitempty"`
	Ca      bool                       `xml:"ca,attr,omitempty"`
	Si      ml.OptionalIndex           `xml:"si,attr,omitempty"`
	Bx      bool                       `xml:"bx,attr,omitempty"`
}

//MergeCell is a direct mapping of XSD CT_MergeCell
type MergeCell struct {
	Bounds primitives.Bounds `xml:"ref,attr"`
}

//SheetViews is a direct mapping of XSD CT_SheetViews
type SheetViews struct {
	SheetView []*SheetView `xml:"sheetView,omitempty"`
	ExtLst    *ml.Reserved `xml:"extLst,omitempty"`
}

//SheetView is a direct mapping of XSD CT_SheetView
type SheetView struct {
	Pane                     *ml.Reserved       `xml:"pane,omitempty"`
	Selection                *ml.Reserved       `xml:"selection,omitempty"`
	PivotSelection           *ml.Reserved       `xml:"pivotSelection,omitempty"`
	ExtLst                   *ml.Reserved       `xml:"extLst,omitempty"`
	WindowProtection         bool               `xml:"windowProtection,attr,omitempty"`
	ShowFormulas             bool               `xml:"showFormulas,attr,omitempty"`
	ShowGridLines            bool               `xml:"showGridLines,attr,omitempty"`
	ShowRowColHeaders        bool               `xml:"showRowColHeaders,attr,omitempty"`
	ShowZeros                bool               `xml:"showZeros,attr,omitempty"`
	RightToLeft              bool               `xml:"rightToLeft,attr,omitempty"`
	TabSelected              bool               `xml:"tabSelected,attr,omitempty"`
	ShowRuler                bool               `xml:"showRuler,attr,omitempty"`
	ShowOutlineSymbols       bool               `xml:"showOutlineSymbols,attr,omitempty"`
	DefaultGridColor         bool               `xml:"defaultGridColor,attr,omitempty"`
	ShowWhiteSpace           bool               `xml:"showWhiteSpace,attr,omitempty"`
	View                     string             `xml:"view,attr,omitempty"` //ST_SheetViewType
	TopLeftCell              primitives.CellRef `xml:"topLeftCell,attr,omitempty"`
	ColorId                  uint               `xml:"colorId,attr,omitempty"`
	ZoomScale                uint               `xml:"zoomScale,attr,omitempty"`
	ZoomScaleNormal          uint               `xml:"zoomScaleNormal,attr,omitempty"`
	ZoomScaleSheetLayoutView uint               `xml:"zoomScaleSheetLayoutView,attr,omitempty"`
	ZoomScalePageLayoutView  uint               `xml:"zoomScalePageLayoutView,attr,omitempty"`
	WorkbookViewId           uint               `xml:"workbookViewId,attr"`
}
<|endoftext|>


<commit_before>package s3

import "strings"

func parseExt(a string) string {
	switch {
	case strings.Contains(a, "/json"):
		return ".json"
	case strings.Contains(a, "/html"):
		return ".html"
	case strings.Contains(a, "/jpeg"):
		return ".jpg"
	case strings.Contains(a, "/png"):
		return ".png"
	case strings.Contains(a, "/gif"):
		return ".gif"
	case strings.Contains(a, "/bmp"):
		return ".bmp"
	case strings.Contains(a, "/tiff"):
		return ".tiff"
	case strings.Contains(a, "/plain"):
		return ".txt"
	case strings.Contains(a, "/rtf"):
		return ".rtf"
	case strings.Contains(a, "/msword"):
		return ".doc"
	case strings.Contains(a, "/zip"):
		return ".zip"
	case strings.Contains(a, "/mpeg"):
		return ".mp4"
	case strings.Contains(a, "/pdf"):
		return ".pdf"
	case strings.Contains(a, "/stylesheet"):
		return ".css"
	case strings.Contains(a, "/javascript"):
		return ".js"
	default:
		return ""
	}
}
<commit_msg>corrected css content-type<commit_after>package s3

import "strings"

func parseExt(a string) string {
	switch {
	case strings.Contains(a, "/json"):
		return ".json"
	case strings.Contains(a, "/html"):
		return ".html"
	case strings.Contains(a, "/jpeg"):
		return ".jpg"
	case strings.Contains(a, "/png"):
		return ".png"
	case strings.Contains(a, "/gif"):
		return ".gif"
	case strings.Contains(a, "/bmp"):
		return ".bmp"
	case strings.Contains(a, "/tiff"):
		return ".tiff"
	case strings.Contains(a, "/plain"):
		return ".txt"
	case strings.Contains(a, "/rtf"):
		return ".rtf"
	case strings.Contains(a, "/msword"):
		return ".doc"
	case strings.Contains(a, "/zip"):
		return ".zip"
	case strings.Contains(a, "/mpeg"):
		return ".mp4"
	case strings.Contains(a, "/pdf"):
		return ".pdf"
	case strings.Contains(a, "/css"):
		return ".css"
	case strings.Contains(a, "/javascript"):
		return ".js"
	default:
		return ""
	}
}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/errors"
)

const teacherUrlBase = "http://eikaiwa.dmm.com/teacher/index/%v/"

var (
	idsRegexp        = regexp.MustCompile(`^[\d,]+$`)
	teacherUrlRegexp = regexp.MustCompile(`https?://eikaiwa.dmm.com/teacher/index/([\d]+)`)
)

type Teacher struct {
	ID                uint32
	Name              string
	CountryID         uint16
	Gender            string
	Birthday          time.Time
	YearsOfExperience uint8
	FetchErrorCount   uint8
	CreatedAt         time.Time
	UpdatedAt         time.Time
}

func (*Teacher) TableName() string {
	return "teacher"
}

func NewTeacher(id uint32) *Teacher {
	return &Teacher{ID: id}
}

func NewTeachersFromIDsOrURL(idsOrUrl string) ([]*Teacher, error) {
	if idsRegexp.MatchString(idsOrUrl) {
		ids := strings.Split(idsOrUrl, ",")
		teachers := make([]*Teacher, 0, len(ids))
		for _, sid := range ids {
			if id, err := strconv.ParseUint(sid, 10, 32); err == nil {
				teachers = append(teachers, NewTeacher(uint32(id)))
			} else {
				continue
			}
		}
		return teachers, nil
	} else if group := teacherUrlRegexp.FindStringSubmatch(idsOrUrl); len(group) > 0 {
		id, _ := strconv.ParseUint(group[1], 10, 32)
		return []*Teacher{NewTeacher(uint32(id))}, nil
	}
	return nil, errors.InvalidArgumentf("Failed to parse idsOrUrl: %s", idsOrUrl)
}

func (t *Teacher) URL() string {
	return fmt.Sprintf(teacherUrlBase, t.ID)
}

type TeacherService struct {
	db *gorm.DB
}

func NewTeacherService(db *gorm.DB) *TeacherService {
	return &TeacherService{db: db}
}

func (s *TeacherService) CreateOrUpdate(t *Teacher) error {
	sql := fmt.Sprintf("INSERT INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", t.TableName())
	sql += " ON DUPLICATE KEY UPDATE"
	sql += " country_id=?, gender=?, years_of_experience=?, birthday=?"
	now := time.Now()
	values := []interface{}{
		t.ID,
		t.Name,
		t.CountryID,
		t.Gender,
		t.Birthday.Format("2006-01-02"),
		t.YearsOfExperience,
		t.FetchErrorCount,
		now.Format("2006-01-02 15:04:05"),
		now.Format("2006-01-02 15:04:05"),
		// update
		t.CountryID,
		t.Gender,
		t.YearsOfExperience,
		t.Birthday.Format("2006-01-02"),
	}

	if err := s.db.Exec(sql, values...).Error; err != nil {
		return errors.InternalWrapf(err, "Failed to INSERT or UPDATE teacher: id=%v", t.ID)
	}
	return nil
}

func (s *TeacherService) FindByPK(id uint32) (*Teacher, error) {
	teacher := &Teacher{}
	if err := s.db.First(teacher, &Teacher{ID: id}).Error; err != nil {
		return nil, errors.InternalWrapf(err, "Failed to FindByPK")
	}
	return teacher, nil
}

func (s *TeacherService) IncrementFetchErrorCount(id uint32, value int) error {
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = fetch_error_count + ?, updatd_at = NOW() WHERE id = ?`,
		new(Teacher).TableName(),
	)
	if err := s.db.Exec(sql, value, id).Error; err != nil {
		return errors.InternalWrapf(err, "Failed to UPDATE teacher: id=%v", id)
	}
	return nil
}

func (s *TeacherService) ResetFetchErrorCount(id uint32) error {
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = 0, updated_at = NOW() WHERE id = ?`,
		new(Teacher).TableName(),
	)
	if err := s.db.Exec(sql, id).Error; err != nil {
		return errors.InternalWrapf(err, "Failed to UPDATE teacher: id=%v", id)
	}
	return nil
}

func (s *TeacherService) FindByFetchErrorCountGt(count uint32) ([]*Teacher, error) {
	values := make([]*Teacher, 0, 3000)
	sql := fmt.Sprintf(`SELECT * FROM teacher WHERE fetch_error_count > ? ORDER BY id LIMIT 3000`)
	if result := s.db.Raw(sql, count).Scan(&values); result.Error != nil {
		return nil, errors.InternalWrapf(result.Error, "Failed to FindByFetchErrorCountGt")
	}
	return values, nil
}
<commit_msg>Fix typo<commit_after>package model

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/errors"
)

const teacherUrlBase = "http://eikaiwa.dmm.com/teacher/index/%v/"

var (
	idsRegexp        = regexp.MustCompile(`^[\d,]+$`)
	teacherUrlRegexp = regexp.MustCompile(`https?://eikaiwa.dmm.com/teacher/index/([\d]+)`)
)

type Teacher struct {
	ID                uint32
	Name              string
	CountryID         uint16
	Gender            string
	Birthday          time.Time
	YearsOfExperience uint8
	FetchErrorCount   uint8
	CreatedAt         time.Time
	UpdatedAt         time.Time
}

func (*Teacher) TableName() string {
	return "teacher"
}

func NewTeacher(id uint32) *Teacher {
	return &Teacher{ID: id}
}

func NewTeachersFromIDsOrURL(idsOrUrl string) ([]*Teacher, error) {
	if idsRegexp.MatchString(idsOrUrl) {
		ids := strings.Split(idsOrUrl, ",")
		teachers := make([]*Teacher, 0, len(ids))
		for _, sid := range ids {
			if id, err := strconv.ParseUint(sid, 10, 32); err == nil {
				teachers = append(teachers, NewTeacher(uint32(id)))
			} else {
				continue
			}
		}
		return teachers, nil
	} else if group := teacherUrlRegexp.FindStringSubmatch(idsOrUrl); len(group) > 0 {
		id, _ := strconv.ParseUint(group[1], 10, 32)
		return []*Teacher{NewTeacher(uint32(id))}, nil
	}
	return nil, errors.InvalidArgumentf("Failed to parse idsOrUrl: %s", idsOrUrl)
}

func (t *Teacher) URL() string {
	return fmt.Sprintf(teacherUrlBase, t.ID)
}

type TeacherService struct {
	db *gorm.DB
}

func NewTeacherService(db *gorm.DB) *TeacherService {
	return &TeacherService{db: db}
}

func (s *TeacherService) CreateOrUpdate(t *Teacher) error {
	sql := fmt.Sprintf("INSERT INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", t.TableName())
	sql += " ON DUPLICATE KEY UPDATE"
	sql += " country_id=?, gender=?, years_of_experience=?, birthday=?"
	now := time.Now()
	values := []interface{}{
		t.ID,
		t.Name,
		t.CountryID,
		t.Gender,
		t.Birthday.Format("2006-01-02"),
		t.YearsOfExperience,
		t.FetchErrorCount,
		now.Format("2006-01-02 15:04:05"),
		now.Format("2006-01-02 15:04:05"),
		// update
		t.CountryID,
		t.Gender,
		t.YearsOfExperience,
		t.Birthday.Format("2006-01-02"),
	}

	if err := s.db.Exec(sql, values...).Error; err != nil {
		return errors.InternalWrapf(err, "Failed to INSERT or UPDATE teacher: id=%v", t.ID)
	}
	return nil
}

func (s *TeacherService) FindByPK(id uint32) (*Teacher, error) {
	teacher := &Teacher{}
	if err := s.db.First(teacher, &Teacher{ID: id}).Error; err != nil {
		return nil, errors.InternalWrapf(err, "Failed to FindByPK")
	}
	return teacher, nil
}

func (s *TeacherService) IncrementFetchErrorCount(id uint32, value int) error {
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = fetch_error_count + ?, updated_at = NOW() WHERE id = ?`,
		new(Teacher).TableName(),
	)
	if err := s.db.Exec(sql, value, id).Error; err != nil {
		return errors.InternalWrapf(err, "Failed to UPDATE teacher: id=%v", id)
	}
	return nil
}

func (s *TeacherService) ResetFetchErrorCount(id uint32) error {
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = 0, updated_at = NOW() WHERE id = ?`,
		new(Teacher).TableName(),
	)
	if err := s.db.Exec(sql, id).Error; err != nil {
		return errors.InternalWrapf(err, "Failed to UPDATE teacher: id=%v", id)
	}
	return nil
}

func (s *TeacherService) FindByFetchErrorCountGt(count uint32) ([]*Teacher, error) {
	values := make([]*Teacher, 0, 3000)
	sql := fmt.Sprintf(`SELECT * FROM teacher WHERE fetch_error_count > ? ORDER BY id LIMIT 3000`)
	if result := s.db.Raw(sql, count).Scan(&values); result.Error != nil {
		return nil, errors.InternalWrapf(result.Error, "Failed to FindByFetchErrorCountGt")
	}
	return values, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"

	"github.com/gorilla/mux"
	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/domain/infosync"
	"github.com/pingcap/tidb/parser/terror"
	"github.com/pingcap/tidb/util/logutil"
	"go.uber.org/zap"
)

// PlanReplayerHandler is the handler for dumping plan replayer file.
type PlanReplayerHandler struct {
	infoGetter *infosync.InfoSyncer
	address    string
	statusPort uint
}

func (s *Server) newPlanReplayerHandler() *PlanReplayerHandler {
	cfg := config.GetGlobalConfig()
	prh := &PlanReplayerHandler{
		address:    cfg.AdvertiseAddress,
		statusPort: cfg.Status.StatusPort,
	}
	if s.dom != nil && s.dom.InfoSyncer() != nil {
		prh.infoGetter = s.dom.InfoSyncer()
	}
	return prh
}

func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	name := params[pFileName]
	handler := downloadFileHandler{
		filePath:           filepath.Join(domain.GetPlanReplayerDirName(), name),
		fileName:           name,
		infoGetter:         prh.infoGetter,
		address:            prh.address,
		statusPort:         prh.statusPort,
		urlPath:            fmt.Sprintf("plan_replyaer/dump/%s", name),
		downloadedFilename: "plan_replayer",
	}
	handleDownloadFile(handler, w, req)
}

func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	name := params[pFileName]
	path := handler.filePath
	if isExists(path) {
		w.Header().Set("Content-Type", "application/zip")
		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", handler.downloadedFilename))
		file, err := os.Open(path)
		if err != nil {
			writeError(w, err)
			return
		}
		_, err = io.Copy(w, file)
		if err != nil {
			writeError(w, err)
			return
		}
		err = file.Close()
		if err != nil {
			writeError(w, err)
			return
		}
		err = os.Remove(path)
		if err != nil {
			writeError(w, err)
			return
		}
		w.WriteHeader(http.StatusOK)
		return
	}
	if handler.infoGetter == nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	// we didn't find file for forward request, return 404
	forwarded := req.URL.Query().Get("forward")
	if len(forwarded) > 0 {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	// If we didn't find file in origin request, try to broadcast the request to all remote tidb-servers
	topos, err := handler.infoGetter.GetAllTiDBTopology(req.Context())
	if err != nil {
		writeError(w, err)
		return
	}
	// transfer each remote tidb-server and try to find dump file
	for _, topo := range topos {
		if topo.IP == handler.address && topo.StatusPort == handler.statusPort {
			continue
		}
		url := fmt.Sprintf("http://%s:%v/%s?forward=true", topo.IP, topo.StatusPort, handler.urlPath)
		resp, err := http.Get(url) // #nosec G107
		if err != nil {
			terror.Log(errors.Trace(err))
			logutil.BgLogger().Error("forward request failed", zap.String("addr", topo.IP), zap.Uint("port", topo.StatusPort), zap.Error(err))
			continue
		}
		if resp.StatusCode != http.StatusOK {
			continue
		}
		// find dump file in one remote tidb-server, return file directly
		w.Header().Set("Content-Type", "application/zip")
		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", handler.downloadedFilename))
		_, err = io.Copy(w, resp.Body)
		if err != nil {
			writeError(w, err)
			return
		}
		err = resp.Body.Close()
		if err != nil {
			writeError(w, err)
			return
		}
		w.WriteHeader(http.StatusOK)
		return
	}
	// we can't find dump file in any tidb-server, return 404 directly
	logutil.BgLogger().Info("can't find dump file in any remote server", zap.String("filename", name))
	w.WriteHeader(http.StatusNotFound)
}

type downloadFileHandler struct {
	filePath           string
	fileName           string
	infoGetter         *infosync.InfoSyncer
	address            string
	statusPort         uint
	urlPath            string
	downloadedFilename string
}

func isExists(path string) bool {
	_, err := os.Stat(path)
	if err != nil && !os.IsExist(err) {
		return false
	}
	return true
}
<commit_msg>server: refine code logic in handleDownloadFile (#30422)<commit_after>// Copyright 2021 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"

	"github.com/gorilla/mux"
	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/domain/infosync"
	"github.com/pingcap/tidb/parser/terror"
	"github.com/pingcap/tidb/util/logutil"
	"go.uber.org/zap"
)

// PlanReplayerHandler is the handler for dumping plan replayer file.
type PlanReplayerHandler struct {
	infoGetter *infosync.InfoSyncer
	address    string
	statusPort uint
}

func (s *Server) newPlanReplayerHandler() *PlanReplayerHandler {
	cfg := config.GetGlobalConfig()
	prh := &PlanReplayerHandler{
		address:    cfg.AdvertiseAddress,
		statusPort: cfg.Status.StatusPort,
	}
	if s.dom != nil && s.dom.InfoSyncer() != nil {
		prh.infoGetter = s.dom.InfoSyncer()
	}
	return prh
}

func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	name := params[pFileName]
	handler := downloadFileHandler{
		filePath:           filepath.Join(domain.GetPlanReplayerDirName(), name),
		fileName:           name,
		infoGetter:         prh.infoGetter,
		address:            prh.address,
		statusPort:         prh.statusPort,
		urlPath:            fmt.Sprintf("plan_replyaer/dump/%s", name),
		downloadedFilename: "plan_replayer",
	}
	handleDownloadFile(handler, w, req)
}

func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	name := params[pFileName]
	path := handler.filePath
	exist, err := isExists(path)
	if err != nil {
		writeError(w, err)
		return
	}
	if exist {
		file, err := os.Open(path)
		if err != nil {
			writeError(w, err)
			return
		}
		content, err := ioutil.ReadAll(file)
		if err != nil {
			writeError(w, err)
			return
		}
		err = file.Close()
		if err != nil {
			writeError(w, err)
			return
		}
		err = os.Remove(path)
		if err != nil {
			writeError(w, err)
			return
		}
		_, err = w.Write(content)
		if err != nil {
			writeError(w, err)
			return
		}
		w.Header().Set("Content-Type", "application/zip")
		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", handler.downloadedFilename))
		return
	}
	if handler.infoGetter == nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	// we didn't find file for forward request, return 404
	forwarded := req.URL.Query().Get("forward")
	if len(forwarded) > 0 {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	// If we didn't find file in origin request, try to broadcast the request to all remote tidb-servers
	topos, err := handler.infoGetter.GetAllTiDBTopology(req.Context())
	if err != nil {
		writeError(w, err)
		return
	}
	// transfer each remote tidb-server and try to find dump file
	for _, topo := range topos {
		if topo.IP == handler.address && topo.StatusPort == handler.statusPort {
			continue
		}
		url := fmt.Sprintf("http://%s:%v/%s?forward=true", topo.IP, topo.StatusPort, handler.urlPath)
		resp, err := http.Get(url) // #nosec G107
		if err != nil {
			terror.Log(errors.Trace(err))
			logutil.BgLogger().Error("forward request failed", zap.String("addr", topo.IP), zap.Uint("port", topo.StatusPort), zap.Error(err))
			continue
		}
		if resp.StatusCode != http.StatusOK {
			continue
		}
		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			writeError(w, err)
			return
		}
		err = resp.Body.Close()
		if err != nil {
			writeError(w, err)
			return
		}
		_, err = w.Write(content)
		if err != nil {
			writeError(w, err)
			return
		}
		// find dump file in one remote tidb-server, return file directly
		w.Header().Set("Content-Type", "application/zip")
		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", handler.downloadedFilename))
		return
	}
	// we can't find dump file in any tidb-server, return 404 directly
	logutil.BgLogger().Error("can't find dump file in any remote server", zap.String("filename", name))
	w.WriteHeader(http.StatusNotFound)
}

type downloadFileHandler struct {
	filePath           string
	fileName           string
	infoGetter         *infosync.InfoSyncer
	address            string
	statusPort         uint
	urlPath            string
	downloadedFilename string
}

func isExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}
<|endoftext|>


<commit_before>package db

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	chars      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "."
)

var (
	ErrFileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	Hash    string
	Mode    uint32
}

type FileData struct {
	Name string
	Data []byte
	Mode uint32
}

type Label uint8

type FileTag struct {
	Label Label
	Mode  uint32
}

type TagList struct {
	Files map[string]FileTag
	Dirs  map[string]FileTag
}

const (
	File = Label(iota)
	Conflict
	Directory
	Deleted
)

// Parse parses a TraDb structure.
// Fails if the given string is not in the correct format.
func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version hash mode
			if len(fields) != 7 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return nil, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return nil, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return nil, err
			}

			sum := fields[5]

			mode, err := strconv.ParseUint(fields[6], 10, 32)
			if err != nil {
				return nil, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId, sum, uint32(mode)}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return nil, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

// ParseFile parses a TraDb from the db file in the current directory.
func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(ErrFileNotFound.Error())
		tradb, err = New()
		if err == nil {
			return tradb, ErrFileNotFound
		} else {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	defer func() { err = dbfile.Close() }()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return nil, err
	}

	tradb, err = Parse(string(bs))
	return tradb, err
}

// New creates a new TraDb.
//
// The replica ID is a random string, and the version
// number is set to 1. Checks for files in the current
// directory and stores relevant file state in a map.
func New() (*TraDb, error) {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = chars[rand.Intn(len(chars))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := readDir(currentDir)
	if err != nil {
		return nil, err
	}

	filemap := make(map[string]FileState)

	for filename, file := range files {
		var err error
		var hashString string

		if !file.IsDir() {
			hashString, err = hash(filename)
			if err != nil {
				return nil, err
			}
		} else {
			hashString = "[dir]"
		}

		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
			Hash:    hashString,
			Mode:    uint32(file.Mode()),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}, nil
}

func readDir(dir string) (map[string]os.FileInfo, error) {
	files := make(map[string]os.FileInfo)

	if err := readDirRecursive(dir, files); err != nil {
		return nil, err
	}
	delete(files, TRADB)

	return files, nil
}

func readDirRecursive(dir string, filemap map[string]os.FileInfo) error {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, fileinfo := range files {
		name := filepath.Join(dir, fileinfo.Name())

		filemap[name] = fileinfo
		if fileinfo.IsDir() {
			err := readDirRecursive(fileinfo.Name(), filemap)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Write writes a TraDb to the db file .trago.db.
func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		var err error
		var hashString string

		if mode := os.FileMode(info.Mode); mode.IsDir() {
			hashString = "[dir]"
		} else {
			hashString, err = hash(filename)
			if err != nil {
				return err
			}
		}

		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d %s %d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
			hashString,
			info.Mode,
		)
		i++
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	return ioutil.WriteFile(TRADB, dataToWrite, 0644)
}

// Update looks for modified files in the current directory
// and updates the filemap accordingly.
func (db *TraDb) Update() error {
	files, err := readDir(currentDir)
	if err != nil {
		return err
	}

	visitedFiles := make(map[string]bool)
	ourVersion := db.VersionVec[db.ReplicaId]

	for filename, file := range files {
		var err error
		var hashString string

		if file.IsDir() {
			hashString = "[dir]"
		} else {
			hashString, err = hash(filename)
			if err != nil {
				return err
			}
		}

		dbRecord := db.Files[filename]
		if dbRecord.Version == 0 {
			log.Printf("found a new file: %s\n", filename)

			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
				Hash:    hashString,
				Mode:    uint32(file.Mode()),
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
			dbRecord.Mode = uint32(file.Mode())
			dbRecord.Replica = db.ReplicaId
			dbRecord.Hash = hashString
			db.Files[filename] = dbRecord
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
		visitedFiles[filename] = true
	}

	// Check for deleted files.
	for filename, _ := range db.Files {
		if !visitedFiles[filename] {
			log.Printf("update: deleting entry for %s\n", filename)
			delete(db.Files, filename)
		}
	}

	return nil
}

// Compare compares two TraDbs.
// Returns a map which gives the FileTag for each changed file.
func (local *TraDb) Compare(remote *TraDb) (TagList, error) {
	var tags TagList

	tags.Dirs = make(map[string]FileTag)
	tags.Files = make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if mode := os.FileMode(state.Mode); mode.IsDir() {
			continue
		}

		if remoteState.Version == 0 { // file not on server
			if state.Version <= remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags.Files[file] = FileTag{Deleted, 0}
				continue
			}
		}

		changed, err := isFileChanged(state, remoteState)
		if err != nil {
			return tags, err
		}

		if changed {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags.Files[file] = FileTag{File, remoteFiles[file].Mode}

				dir := filepath.Dir(file)
				tags.Dirs[dir] = FileTag{Directory, remoteFiles[dir].Mode}
			} else {
				log.Printf("conflict: %s\n", file)
				tags.Files[file] = FileTag{Conflict, 0}
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if mode := os.FileMode(state.Mode); mode.IsDir() {
			continue
		}

		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags.Files[file] = FileTag{File, remoteFiles[file].Mode}

			dir := filepath.Dir(file)
			tags.Dirs[dir] = FileTag{Directory, remoteFiles[dir].Mode}
		}
	}

	return tags, nil
}

func (db *TraDb) UpdateMTimes() error {
	files, err := readDir(currentDir)
	if err != nil {
		return err
	}

	for filename, file := range files {
		if file.IsDir() {
			continue
		}

		dbRecord := db.Files[filename]
		if mtime := file.ModTime().UTC().UnixNano(); mtime > dbRecord.MTime {
			log.Printf("updating mtime: %s\n", filename)
			dbRecord.MTime = mtime
			db.Files[filename] = dbRecord
		}
	}

	return nil
}

func CombineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func hash(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return filename, err
	}
	defer func() { err = f.Close() }()

	h := md5.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return filename, err
	}

	return hex.EncodeToString(h.Sum(nil)), err
}

func isFileChanged(fs1 FileState, fs2 FileState) (bool, error) {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		h1, err := hex.DecodeString(fs1.Hash)
		if err != nil {
			return false, err
		}

		h2, err := hex.DecodeString(fs2.Hash)
		if err != nil {
			return false, err
		}

		return !bytes.Equal(h1, h2), nil
	}
	return false, nil
}
<commit_msg>fix bug. use absolute directory name when listing files in subdirectories<commit_after>package db

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	chars      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "."
)

var (
	ErrFileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	Hash    string
	Mode    uint32
}

type FileData struct {
	Name string
	Data []byte
	Mode uint32
}

type Label uint8

type FileTag struct {
	Label Label
	Mode  uint32
}

type TagList struct {
	Files map[string]FileTag
	Dirs  map[string]FileTag
}

const (
	File = Label(iota)
	Conflict
	Directory
	Deleted
)

// Parse parses a TraDb structure.
// Fails if the given string is not in the correct format.
func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version hash mode
			if len(fields) != 7 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return nil, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return nil, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return nil, err
			}

			sum := fields[5]

			mode, err := strconv.ParseUint(fields[6], 10, 32)
			if err != nil {
				return nil, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId, sum, uint32(mode)}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return nil, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

// ParseFile parses a TraDb from the db file in the current directory.
func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(ErrFileNotFound.Error())
		tradb, err = New()
		if err == nil {
			return tradb, ErrFileNotFound
		} else {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	defer func() { err = dbfile.Close() }()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return nil, err
	}

	tradb, err = Parse(string(bs))
	return tradb, err
}

// New creates a new TraDb.
//
// The replica ID is a random string, and the version
// number is set to 1. Checks for files in the current
// directory and stores relevant file state in a map.
func New() (*TraDb, error) {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = chars[rand.Intn(len(chars))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := readDir(currentDir)
	if err != nil {
		return nil, err
	}

	filemap := make(map[string]FileState)

	for filename, file := range files {
		var err error
		var hashString string

		if !file.IsDir() {
			hashString, err = hash(filename)
			if err != nil {
				return nil, err
			}
		} else {
			hashString = "[dir]"
		}

		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
			Hash:    hashString,
			Mode:    uint32(file.Mode()),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}, nil
}

func readDir(dir string) (map[string]os.FileInfo, error) {
	files := make(map[string]os.FileInfo)

	if err := readDirRecursive(dir, files); err != nil {
		return nil, err
	}
	delete(files, TRADB)

	return files, nil
}

func readDirRecursive(dir string, filemap map[string]os.FileInfo) error {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, fileinfo := range files {
		name := filepath.Join(dir, fileinfo.Name())

		filemap[name] = fileinfo
		if fileinfo.IsDir() {
			err := readDirRecursive(name, filemap)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Write writes a TraDb to the db file .trago.db.
func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		var err error
		var hashString string

		if mode := os.FileMode(info.Mode); mode.IsDir() {
			hashString = "[dir]"
		} else {
			hashString, err = hash(filename)
			if err != nil {
				return err
			}
		}

		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d %s %d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
			hashString,
			info.Mode,
		)
		i++
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	return ioutil.WriteFile(TRADB, dataToWrite, 0644)
}

// Update looks for modified files in the current directory
// and updates the filemap accordingly.
func (db *TraDb) Update() error {
	files, err := readDir(currentDir)
	if err != nil {
		return err
	}

	visitedFiles := make(map[string]bool)
	ourVersion := db.VersionVec[db.ReplicaId]

	for filename, file := range files {
		var err error
		var hashString string

		if file.IsDir() {
			hashString = "[dir]"
		} else {
			hashString, err = hash(filename)
			if err != nil {
				return err
			}
		}

		dbRecord := db.Files[filename]
		if dbRecord.Version == 0 {
			log.Printf("found a new file: %s\n", filename)

			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
				Hash:    hashString,
				Mode:    uint32(file.Mode()),
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
			dbRecord.Mode = uint32(file.Mode())
			dbRecord.Replica = db.ReplicaId
			dbRecord.Hash = hashString
			db.Files[filename] = dbRecord
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
		visitedFiles[filename] = true
	}

	// Check for deleted files.
	for filename, _ := range db.Files {
		if !visitedFiles[filename] {
			log.Printf("update: deleting entry for %s\n", filename)
			delete(db.Files, filename)
		}
	}

	return nil
}

// Compare compares two TraDbs.
// Returns a map which gives the FileTag for each changed file.
func (local *TraDb) Compare(remote *TraDb) (TagList, error) {
	var tags TagList

	tags.Dirs = make(map[string]FileTag)
	tags.Files = make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if mode := os.FileMode(state.Mode); mode.IsDir() {
			continue
		}

		if remoteState.Version == 0 { // file not on server
			if state.Version <= remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags.Files[file] = FileTag{Deleted, 0}
				continue
			}
		}

		changed, err := isFileChanged(state, remoteState)
		if err != nil {
			return tags, err
		}

		if changed {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags.Files[file] = FileTag{File, remoteFiles[file].Mode}

				dir := filepath.Dir(file)
				tags.Dirs[dir] = FileTag{Directory, remoteFiles[dir].Mode}
			} else {
				log.Printf("conflict: %s\n", file)
				tags.Files[file] = FileTag{Conflict, 0}
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if mode := os.FileMode(state.Mode); mode.IsDir() {
			continue
		}

		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags.Files[file] = FileTag{File, remoteFiles[file].Mode}

			dir := filepath.Dir(file)
			tags.Dirs[dir] = FileTag{Directory, remoteFiles[dir].Mode}
		}
	}

	return tags, nil
}

func (db *TraDb) UpdateMTimes() error {
	files, err := readDir(currentDir)
	if err != nil {
		return err
	}

	for filename, file := range files {
		if file.IsDir() {
			continue
		}

		dbRecord := db.Files[filename]
		if mtime := file.ModTime().UTC().UnixNano(); mtime > dbRecord.MTime {
			log.Printf("updating mtime: %s\n", filename)
			dbRecord.MTime = mtime
			db.Files[filename] = dbRecord
		}
	}

	return nil
}

func CombineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func hash(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return filename, err
	}
	defer func() { err = f.Close() }()

	h := md5.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return filename, err
	}

	return hex.EncodeToString(h.Sum(nil)), err
}

func isFileChanged(fs1 FileState, fs2 FileState) (bool, error) {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		h1, err := hex.DecodeString(fs1.Hash)
		if err != nil {
			return false, err
		}

		h2, err := hex.DecodeString(fs2.Hash)
		if err != nil {
			return false, err
		}

		return !bytes.Equal(h1, h2), nil
	}
	return false, nil
}
<|endoftext|>


<commit_before>// Package dburl provides a standardized way of processing database connection
// strings in the form of a URL.
//
// Standard URLs are of the form
// protocol+transport://user:pass@host/dbname?opt1=a&opt2=b
//
// For example, the following are URLs that can be processed using Parse or
// Open:
//
//     postgres://user:pass@localhost/dbname
//     pg://user:pass@localhost/dbname?sslmode=disable
//     mysql://user:pass@localhost/dbname
//     sqlserver://user:pass@remote-host.com/dbname
//     oracle://user:pass@somehost.com/oracledb
//     sqlite:/path/to/file.db
// 	   file:myfile.sqlite3?loc=auto
//
// Protocol aliases:
//
// The following protocol aliases are available, and will be parsed according
// to the rules for their respective driver.
//
//     Database (driver)            | Aliases
//     ------------------------------------------------------------------
//     Microsoft SQL Server (mssql) | ms, sqlserver
//     MySQL (mysql)                | my, mariadb, maria, percona, aurora
//     Oracle (ora)                 | or, oracle, oci8, oci
//     PostgreSQL (postgres)        | pg, postgresql, pgsql
//     SQLite3 (sqlite3)            | sq, sqlite, file
//
package dburl

import (
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"path"
	"strconv"
	"strings"
)

// URL wraps the standard net/url.URL type, adding a Proto string.
type URL struct {
	url.URL
	OriginalScheme, Proto, Driver, DSN string
}

// String satisfies the stringer interface.
func (u *URL) String() string {
	p := &url.URL{
		Scheme:   u.OriginalScheme,
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return p.String()
}

// Short provides a short description of the user, host, and database.
func (u *URL) Short() string {
	s := u.Driver[:2]
	if s == "po" {
		s = "pg"
	}

	s += ":"

	if u.User != nil {
		if username := u.User.Username(); username != "" {
			s += username + "@"
		}
	}
	if u.Host != "" {
		s += u.Host
	}
	if u.Path != "" && u.Path != "/" {
		s += u.Path
	}
	if u.Opaque != "" {
		s += u.Opaque
	}

	return s
}

// Parse parses a rawurl string and normalizes the scheme.
func Parse(rawurl string) (*URL, error) {
	// parse url
	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if u.Scheme == "" {
		return nil, errors.New("invalid database scheme")
	}

	// create url
	v := &URL{URL: *u, OriginalScheme: u.Scheme, Proto: "tcp"}
	v.Scheme = strings.ToLower(v.Scheme)

	// check if +unix or whatever is in the scheme
	if strings.Contains(v.Scheme, "+") {
		p := strings.SplitN(v.Scheme, "+", 2)
		v.Scheme = p[0]
		v.Proto = p[1]
	}

	// check protocol
	if v.Proto != "tcp" && v.Proto != "udp" && v.Proto != "unix" {
		return nil, errors.New("invalid transport protocol")
	}

	// get loader
	loader, ok := loaders[v.Scheme]
	if !ok {
		return nil, errors.New("unknown database type")
	}

	// process
	v.Driver, v.DSN, err = loader(v)
	if err != nil {
		return nil, err
	}

	if v.Driver != "sqlite3" && v.Opaque != "" {
		return Parse(v.Scheme + "://" + v.Opaque)
	}

	return v, nil
}

// OpenURL opens a sql.DB connection to the provided URL.
func OpenURL(u *URL) (*sql.DB, error) {
	return sql.Open(u.Driver, u.DSN)
}

// Open takes a rawurl like
// "protocol+transport://user:pass@host/dbname?option1=a&option2=b" and creates a
// standard sql.DB connection.
//
// Supports mysql, postgresql, mssql, sqlite, and oracle databases.
func Open(rawurl string) (*sql.DB, error) {
	u, err := Parse(rawurl)
	if err != nil {
		return nil, err
	}

	return OpenURL(u)
}

// mssqlProcess processes a mssql url and protocol.
func mssqlProcess(u *URL) (string, string, error) {
	var err error

	// build host or domain socket
	host := u.Host
	port := 1433
	var dbname string

	// grab dbname
	if u.Path != "" {
		dbname = u.Path[1:]
	}
	if dbname == "" {
		return "", "", errors.New("no database name specified")
	}

	// extract port if present
	pos := strings.Index(host, ":")
	if pos != -1 {
		port, err = strconv.Atoi(host[pos+1:])
		if err != nil {
			return "", "", errors.New("invalid port")
		}
		host = host[:pos]
	}

	// format dsn
	dsn := fmt.Sprintf("server=%s;port=%d;database=%s", host, port, dbname)

	// add user/pass
	if u.User != nil {
		if user := u.User.Username(); len(user) > 0 {
			dsn = dsn + ";user id=" + user
		}
		if pass, ok := u.User.Password(); ok {
			dsn = dsn + ";password=" + pass
		}
	}

	// add params
	for k, v := range u.Query() {
		dsn = dsn + ";" + k + "=" + v[0]
	}

	return "mssql", dsn, nil
}

// mysqlProcess processes a mssql url and protocol.
func mysqlProcess(u *URL) (string, string, error) {
	// build host or domain socket
	host := u.Host
	dbname := u.Path

	if strings.HasPrefix(dbname, "/") {
		dbname = dbname[1:]
	}

	if u.Proto == "unix" {
		if u.Opaque != "" {
			host = path.Dir(u.Opaque)
			dbname = path.Base(u.Opaque)
		} else {
			host = path.Join(u.Host, path.Dir(u.Path))
			dbname = path.Base(u.Path)
		}
	} else if !strings.Contains(host, ":") {
		// append default port
		host = host + ":3306"
	}

	// create dsn
	dsn := fmt.Sprintf("%s(%s)", u.Proto, host)

	// build user/pass
	userinfo := ""
	if u.User != nil {
		if un := u.User.Username(); len(un) > 0 {
			userinfo = un
			if up, ok := u.User.Password(); ok {
				userinfo = userinfo + ":" + up
			}
		}
	}
	if userinfo != "" {
		dsn = userinfo + "@" + dsn
	}

	// add database name
	dsn += "/" + dbname

	// add params
	params := u.Query().Encode()
	if len(params) > 0 {
		dsn = dsn + "?" + params
	}

	// format
	return "mysql", dsn, nil
}

// oracleProcess processes a mssql url and protocol.
func oracleProcess(u *URL) (string, string, error) {
	if u.User == nil {
		return "", "", errors.New("must provide username and password in oracle dsn")
	}

	// build host or domain socket
	host := u.Host
	dbname := u.Path[1:]

	// build user/pass
	userinfo := ""
	if un := u.User.Username(); len(un) > 0 {
		userinfo = un
		if up, ok := u.User.Password(); ok {
			userinfo = userinfo + "/" + up
		}
	}

	// format
	return "ora", fmt.Sprintf(
		"%s@%s/%s",
		userinfo,
		host,
		dbname,
	), nil
}

// postgresProcess processes a mssql url and protocol.
func postgresProcess(u *URL) (string, string, error) {
	p := &url.URL{
		Scheme:   "postgres",
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return "postgres", p.String(), nil
}

// sqliteProcess processes a mssql url and protocol.
func sqliteProcess(u *URL) (string, string, error) {
	p := u.Opaque
	if u.Path != "" {
		p = u.Path
	}

	if u.Host != "" && u.Host != "localhost" {
		p = path.Join(u.Host, p)
	}

	return "sqlite3", p + u.Query().Encode(), nil
}

var loaders = map[string]func(*URL) (string, string, error){
	// mssql
	"mssql":     mssqlProcess,
	"sqlserver": mssqlProcess,
	"ms":        mssqlProcess,

	// mysql
	"mysql":   mysqlProcess,
	"mariadb": mysqlProcess,
	"maria":   mysqlProcess,
	"percona": mysqlProcess,
	"aurora":  mysqlProcess,
	"my":      mysqlProcess,

	// oracle
	"ora":    oracleProcess,
	"oracle": oracleProcess,
	"oci8":   oracleProcess,
	"oci":    oracleProcess,
	"or":     oracleProcess,

	// postgresql
	"postgres":   postgresProcess,
	"postgresql": postgresProcess,
	"pgsql":      postgresProcess,
	"pg":         postgresProcess,

	// sqlite
	"sqlite3": sqliteProcess,
	"sqlite":  sqliteProcess,
	"file":    sqliteProcess,
	"sq":      sqliteProcess,
}
<commit_msg>Fixing issue with mssql databases<commit_after>// Package dburl provides a standardized way of processing database connection
// strings in the form of a URL.
//
// Standard URLs are of the form
// protocol+transport://user:pass@host/dbname?opt1=a&opt2=b
//
// For example, the following are URLs that can be processed using Parse or
// Open:
//
//     postgres://user:pass@localhost/dbname
//     pg://user:pass@localhost/dbname?sslmode=disable
//     mysql://user:pass@localhost/dbname
//     sqlserver://user:pass@remote-host.com/dbname
//     oracle://user:pass@somehost.com/oracledb
//     sqlite:/path/to/file.db
// 	   file:myfile.sqlite3?loc=auto
//
// Protocol aliases:
//
// The following protocol aliases are available, and will be parsed according
// to the rules for their respective driver.
//
//     Database (driver)            | Aliases
//     ------------------------------------------------------------------
//     Microsoft SQL Server (mssql) | ms, sqlserver
//     MySQL (mysql)                | my, mariadb, maria, percona, aurora
//     Oracle (ora)                 | or, oracle, oci8, oci
//     PostgreSQL (postgres)        | pg, postgresql, pgsql
//     SQLite3 (sqlite3)            | sq, sqlite, file
//
package dburl

import (
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"path"
	"strconv"
	"strings"
)

// URL wraps the standard net/url.URL type, adding OriginalScheme, Proto,
// Driver, and DSN strings.
type URL struct {
	url.URL
	OriginalScheme, Proto, Driver, DSN string
}

// String satisfies the stringer interface.
func (u *URL) String() string {
	p := &url.URL{
		Scheme:   u.OriginalScheme,
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return p.String()
}

// Short provides a short description of the user, host, and database.
func (u *URL) Short() string {
	s := u.Driver[:2]
	if s == "po" {
		s = "pg"
	}

	s += ":"

	if u.User != nil {
		if un := u.User.Username(); un != "" {
			s += un + "@"
		}
	}

	if u.Host != "" {
		s += u.Host
	}

	if u.Path != "" && u.Path != "/" {
		s += u.Path
	}

	if u.Opaque != "" {
		s += u.Opaque
	}

	return s
}

// Parse parses a rawurl string and normalizes the scheme.
func Parse(rawurl string) (*URL, error) {
	// parse url
	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if u.Scheme == "" {
		return nil, errors.New("invalid database scheme")
	}

	// create url
	v := &URL{URL: *u, OriginalScheme: u.Scheme, Proto: "tcp"}
	v.Scheme = strings.ToLower(v.Scheme)

	// check if +unix or whatever is in the scheme
	if strings.Contains(v.Scheme, "+") {
		p := strings.SplitN(v.Scheme, "+", 2)
		v.Scheme = p[0]
		v.Proto = p[1]
	}

	// check protocol
	if v.Proto != "tcp" && v.Proto != "udp" && v.Proto != "unix" {
		return nil, errors.New("invalid transport protocol")
	}

	// get loader
	loader, ok := loaders[v.Scheme]
	if !ok {
		return nil, errors.New("unknown database type")
	}

	// process
	v.Driver, v.DSN, err = loader(v)
	if err != nil {
		return nil, err
	}

	if v.Driver != "sqlite3" && v.Opaque != "" {
		return Parse(v.OriginalScheme + "://" + v.Opaque)
	}

	return v, nil
}

// Open takes a rawurl like "protocol+transport://user:pass@host/dbname?option1=a&option2=b"
// and creates a standard sql.DB connection.
//
// See Parse for information on formatting URLs to work properly with Open.
func Open(rawurl string) (*sql.DB, error) {
	u, err := Parse(rawurl)
	if err != nil {
		return nil, err
	}

	return sql.Open(u.Driver, u.DSN)
}

// mssqlProcess processes a mssql url and protocol.
func mssqlProcess(u *URL) (string, string, error) {
	var err error

	// build host or domain socket
	host := u.Host
	port := 1433

	// grab dbname
	var dbname string
	if u.Path != "" {
		dbname = u.Path[1:]
	}

	// extract port if present
	pos := strings.Index(host, ":")
	if pos != -1 {
		port, err = strconv.Atoi(host[pos+1:])
		if err != nil {
			return "", "", errors.New("invalid port")
		}
		host = host[:pos]
	}

	// format dsn
	dsn := fmt.Sprintf("server=%s;port=%d", host, port)
	if dbname != "" {
		dsn += ";database=" + dbname
	}

	// add user/pass
	if u.User != nil {
		if user := u.User.Username(); len(user) > 0 {
			dsn += ";user id=" + user
		}
		if pass, ok := u.User.Password(); ok {
			dsn += ";password=" + pass
		}
	}

	// add params
	for k, v := range u.Query() {
		dsn += ";" + k + "=" + v[0]
	}

	return "mssql", dsn, nil
}

// mysqlProcess processes a mssql url and protocol.
func mysqlProcess(u *URL) (string, string, error) {
	// build host or domain socket
	host := u.Host
	dbname := u.Path

	if strings.HasPrefix(dbname, "/") {
		dbname = dbname[1:]
	}

	if u.Proto == "unix" {
		if u.Opaque != "" {
			host = path.Dir(u.Opaque)
			dbname = path.Base(u.Opaque)
		} else {
			host = path.Join(u.Host, path.Dir(u.Path))
			dbname = path.Base(u.Path)
		}
	} else if !strings.Contains(host, ":") {
		// append default port
		host = host + ":3306"
	}

	// create dsn
	dsn := fmt.Sprintf("%s(%s)", u.Proto, host)

	// build user/pass
	if u.User != nil {
		if un := u.User.Username(); len(un) > 0 {
			if up, ok := u.User.Password(); ok {
				un += ":" + up
			}
			dsn = un + "@" + dsn
		}
	}

	// add database name
	dsn += "/" + dbname

	// add params
	params := u.Query().Encode()
	if len(params) > 0 {
		dsn = dsn + "?" + params
	}

	// format
	return "mysql", dsn, nil
}

// oracleProcess processes a mssql url and protocol.
func oracleProcess(u *URL) (string, string, error) {
	if u.User == nil {
		return "", "", errors.New("must provide username and password in oracle dsn")
	}

	// build host or domain socket
	host := u.Host
	dbname := u.Path[1:]

	// build user/pass
	userinfo := ""
	if un := u.User.Username(); len(un) > 0 {
		userinfo = un
		if up, ok := u.User.Password(); ok {
			userinfo = userinfo + "/" + up
		}
	}

	// format
	return "ora", fmt.Sprintf(
		"%s@%s/%s",
		userinfo,
		host,
		dbname,
	), nil
}

// postgresProcess processes a mssql url and protocol.
func postgresProcess(u *URL) (string, string, error) {
	p := &url.URL{
		Scheme:   "postgres",
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return "postgres", p.String(), nil
}

// sqliteProcess processes a mssql url and protocol.
func sqliteProcess(u *URL) (string, string, error) {
	p := u.Opaque
	if u.Path != "" {
		p = u.Path
	}

	if u.Host != "" && u.Host != "localhost" {
		p = path.Join(u.Host, p)
	}

	return "sqlite3", p + u.Query().Encode(), nil
}

var loaders = map[string]func(*URL) (string, string, error){
	// mssql
	"mssql":     mssqlProcess,
	"sqlserver": mssqlProcess,
	"ms":        mssqlProcess,

	// mysql
	"mysql":   mysqlProcess,
	"mariadb": mysqlProcess,
	"maria":   mysqlProcess,
	"percona": mysqlProcess,
	"aurora":  mysqlProcess,
	"my":      mysqlProcess,

	// oracle
	"ora":    oracleProcess,
	"oracle": oracleProcess,
	"oci8":   oracleProcess,
	"oci":    oracleProcess,
	"or":     oracleProcess,

	// postgresql
	"postgres":   postgresProcess,
	"postgresql": postgresProcess,
	"pgsql":      postgresProcess,
	"pg":         postgresProcess,

	// sqlite
	"sqlite3": sqliteProcess,
	"sqlite":  sqliteProcess,
	"file":    sqliteProcess,
	"sq":      sqliteProcess,
}
<|endoftext|>


<commit_before>package debug

import (
    "fmt"
    "os"
    "path/filepath"
    "strings"
    "github.com/wsxiaoys/terminal/color"
    "crypto/md5"
    "io"
)

const COLORS = "rgbcmykw"

var (
    match_cache  = map[string]bool{}
    prefix_cache = map[string]string{}
)

func match(namespace string) (match bool) {
    var ok bool
    if match, ok = match_cache[namespace]; !ok {
        selectors := strings.Split(os.Getenv("DEBUG"), " ")
        for selector := range selectors {
            if ok, _ = filepath.Match(selectors[selector], namespace); ok  {
                match = true
                match_cache[namespace] = match
                return
            }
        }
        match = false
        match_cache[namespace] = match
    }
    return
}

func getcolor(namespace string) string {
    h := md5.New()
    io.WriteString(h, namespace)
    var sum int
    sumbytes := h.Sum(nil)
    for i := range sumbytes {
        sum += int(sumbytes[i])
    }
    return fmt.Sprintf("%c", COLORS[sum % len(COLORS)])

}

func printns(namespace string) {
    var ok bool
    var prefix string

    if prefix, ok = prefix_cache[namespace]; !ok {
        prefix = fmt.Sprintf("@%s%s@| ", getcolor(namespace), namespace)
        prefix_cache[namespace] = prefix
    }

    color.Print(prefix)
}

func Log(namespace, msg string, args ...interface{}) {
    if match(namespace) {
        printns(namespace)
        fmt.Printf(msg, args...)
    }
}

func Logger(namespace string) func (string, ...interface{}) {
    return func (msg string, args ...interface{}) {
        Log(namespace, msg, args...)
    }
}
<commit_msg>Make map writes thread safe<commit_after>package debug

import (
    "fmt"
    "os"
    "path/filepath"
    "strings"
    "github.com/wsxiaoys/terminal/color"
    "crypto/md5"
    "io"
    "sync"
)

const COLORS = "rgbcmykw"

var (
    match_cache  = map[string]bool{}
    prefix_cache = map[string]string{}
    match_mutex = new(sync.Mutex)
    prefix_mutex = new(sync.Mutex)
)

func match(namespace string) (match bool) {
    var ok bool
    if match, ok = match_cache[namespace]; !ok {
        selectors := strings.Split(os.Getenv("DEBUG"), " ")
        for selector := range selectors {
            if ok, _ = filepath.Match(selectors[selector], namespace); ok  {
                match = true
                match_mutex.Lock()
                defer match_mutex.Unlock()
                match_cache[namespace] = match
                return
            }
        }
        match = false
        match_cache[namespace] = match
    }
    return
}

func getcolor(namespace string) string {
    h := md5.New()
    io.WriteString(h, namespace)
    var sum int
    sumbytes := h.Sum(nil)
    for i := range sumbytes {
        sum += int(sumbytes[i])
    }
    return fmt.Sprintf("%c", COLORS[sum % len(COLORS)])

}

func printns(namespace string) {
    var ok bool
    var prefix string

    if prefix, ok = prefix_cache[namespace]; !ok {
        prefix = fmt.Sprintf("@%s%s@| ", getcolor(namespace), namespace)
        prefix_mutex.Lock()
        defer prefix_mutex.Unlock()
        prefix_cache[namespace] = prefix
    }

    color.Print(prefix)
}

func Log(namespace, msg string, args ...interface{}) {
    if match(namespace) {
        printns(namespace)
        fmt.Printf(msg, args...)
    }
}

func Logger(namespace string) func (string, ...interface{}) {
    return func (msg string, args ...interface{}) {
        Log(namespace, msg, args...)
    }
}
<|endoftext|>


<commit_before>package py

/*
#cgo darwin pkg-config: python-2.7
#include "Python.h"
#include "datetime.h"

void init_PyDateTime() {
  PyDateTime_IMPORT;
}

int PyDateTimeCheckExact(PyObject* o) {
  return PyDateTime_CheckExact(o);
}

PyObject* GetPyDateTime(int year, int month, int day, int hour, int minute,
                        int second, int us) {
  return PyDateTime_FromDateAndTime(year, month, day, hour, minute, second, us);
}

int PyTimeDeltaCheckExact(PyObject* o) {
  return PyDelta_CheckExact(o);
}
*/
import "C"
import (
	"time"
)

func init() {
	C.init_PyDateTime()
}

func pyDateTimeCheckExact(o *C.PyObject) bool {
	return C.PyDateTimeCheckExact(o) > 0
}

func getPyDateTime(t time.Time) *C.PyObject {
	us := int(t.Nanosecond() / 1e3)
	return C.GetPyDateTime(C.int(t.Year()), C.int(t.Month()), C.int(t.Day()),
		C.int(t.Hour()), C.int(t.Minute()), C.int(t.Second()), C.int(us))
}

func pyTimeDeltaCheckExact(o *C.PyObject) bool {
	return C.PyTimeDeltaCheckExact(o) > 0
}
<commit_msg>Refactor function definition order<commit_after>package py

/*
#cgo darwin pkg-config: python-2.7
#include "Python.h"
#include "datetime.h"

void init_PyDateTime() {
  PyDateTime_IMPORT;
}

int PyDateTimeCheckExact(PyObject* o) {
  return PyDateTime_CheckExact(o);
}

int PyTimeDeltaCheckExact(PyObject* o) {
  return PyDelta_CheckExact(o);
}

PyObject* GetPyDateTime(int year, int month, int day, int hour, int minute,
                        int second, int us) {
  return PyDateTime_FromDateAndTime(year, month, day, hour, minute, second, us);
}
*/
import "C"
import (
	"time"
)

func init() {
	C.init_PyDateTime()
}

func pyDateTimeCheckExact(o *C.PyObject) bool {
	return C.PyDateTimeCheckExact(o) > 0
}

func pyTimeDeltaCheckExact(o *C.PyObject) bool {
	return C.PyTimeDeltaCheckExact(o) > 0
}

func getPyDateTime(t time.Time) *C.PyObject {
	us := int(t.Nanosecond() / 1e3)
	return C.GetPyDateTime(C.int(t.Year()), C.int(t.Month()), C.int(t.Day()),
		C.int(t.Hour()), C.int(t.Minute()), C.int(t.Second()), C.int(us))
}
<|endoftext|>


<commit_before>package server

import (
	"bufio"
	"firempq/factory/queue_factory"
	"firempq/proto/text_proto"
	"firempq/queue_facade"
	"fmt"
	"io"
	"log"
	"net"
	"strings"
	"time"
	"unicode"
)

const (
	ENDL          = "\n"
    ENDL_BYTE     = '\n'
	SIMPLE_SERVER = "simple"
)

type SimpleServer struct {
	address     string
	queueFacade *queue_facade.QFacade
}

func NewSimpleServer(address string) IQueueServer {
	return &SimpleServer{address: address,
		queueFacade: queue_facade.NewPQFacade()}
}

func (this *SimpleServer) Run() {

	listener, err := net.Listen("tcp", this.address)
	if err != nil {
		log.Fatalln("Can't listen to %s: %s", this.address, err.Error())
	}

	log.Println("Listening at %s", this.address)
	for {
		conn, err := listener.Accept()
		if err == nil {
			go this.handleConnection(conn)
		} else {
			log.Println("Could not accept incoming request: %s", err.Error())
		}
	}
}

func (this *SimpleServer) handleConnection(conn net.Conn) {
	defer conn.Close()

	r := bufio.NewReader(conn)
	w := bufio.NewWriter(conn)

	rw_conn := bufio.NewReadWriter(r, w)

	for {
		err := this.readCommand(rw_conn)
		if err != nil {
			if err == io.EOF {
				log.Print("Client disconnected")
				break
			}
			log.Print(err.Error())
		}
	}
}

func (this *SimpleServer) readCommand(rw *bufio.ReadWriter) error {

	data, err := rw.ReadString(ENDL_BYTE)

	if err != nil {
		return err
	}

	data = strings.TrimRightFunc(data, unicode.IsSpace)
	splits := strings.Split(data, " ")
	var tokens []string
	for _, s := range splits {
		if len(s) > 0 {
			tokens = append(tokens, s)
		}
	}

	if len(splits) == 0 {
		return nil
	}

	switch strings.ToUpper(string(splits[0])) {
	case text_proto.CMD_PING:
		this.writeResponse(rw, text_proto.RESP_PONG)

	case text_proto.CMD_QUIT:
		this.writeResponse(rw, text_proto.RESP_BYE)
		return io.EOF

	case text_proto.CMD_UNIX_TS:
		stamp := fmt.Sprint(time.Now().Unix())
		return this.writeResponse(rw, stamp)

	case text_proto.CMD_CREATE_QUEUE:
		return this.createQueue(rw, tokens[1:])

	case text_proto.CMD_DROP_QUEUE:
		return this.dropQueue(rw, tokens[1:])

	default:
		return this.writeResponse(rw, text_proto.RESP_ERROR)
	}
	return nil
}

func (this *SimpleServer) writeResponse(rw *bufio.ReadWriter, line string) error {
	_, err := rw.WriteString(line)
	if err != nil {
		return err
	}
	_, err = rw.WriteString(ENDL)
	if err != nil {
		return err
	}

	err = rw.Flush()
	if err != nil {
		return err
	}
	return nil
}

type QueueFunc func(req []string) error

func (this *SimpleServer) queueOp(rw *bufio.ReadWriter, args []string, queueFunc QueueFunc) error {
	if len(args) < 1 {
		return this.writeResponse(rw, text_proto.RESP_ERROR+" Not enough parameters")
	}

	if err := queueFunc(args); err == nil {
		return this.writeResponse(rw, text_proto.RESP_OK)
	} else {
		return this.writeResponse(rw, text_proto.RESP_ERROR+" "+err.Error())
	}
}

func (this *SimpleServer) createQueue(rw *bufio.ReadWriter, req []string) error {
	return this.queueOp(rw, req,
		func(args []string) error {
			return this.queueFacade.AddQueue(args[0], queue_factory.GetPQueue(args[0]))
		})
}

func (this *SimpleServer) dropQueue(rw *bufio.ReadWriter, req []string) error {
	return this.queueOp(rw, req,
		func(args []string) error {
			return this.queueFacade.DropQueue(args[0])
		})
}
<commit_msg>Minor fix.<commit_after>package server

import (
	"bufio"
	"firempq/factory/queue_factory"
	"firempq/proto/text_proto"
	"firempq/queue_facade"
	"fmt"
	"io"
	"log"
	"net"
	"strings"
	"time"
	"unicode"
)

const (
	ENDL          = "\n"
    ENDL_BYTE     = '\n'
	SIMPLE_SERVER = "simple"
)

type QueueOpFunc func(req []string) error

type SimpleServer struct {
	address     string
	queueFacade *queue_facade.QFacade
}

func NewSimpleServer(address string) IQueueServer {
	return &SimpleServer{address: address,
		queueFacade: queue_facade.NewPQFacade()}
}

func (this *SimpleServer) Run() {

	listener, err := net.Listen("tcp", this.address)
	if err != nil {
		log.Fatalln("Can't listen to %s: %s", this.address, err.Error())
	}

	log.Println("Listening at %s", this.address)
	for {
		conn, err := listener.Accept()
		if err == nil {
			go this.handleConnection(conn)
		} else {
			log.Println("Could not accept incoming request: %s", err.Error())
		}
	}
}

func (this *SimpleServer) handleConnection(conn net.Conn) {
	defer conn.Close()

	r := bufio.NewReader(conn)
	w := bufio.NewWriter(conn)

	rw_conn := bufio.NewReadWriter(r, w)

	for {
		err := this.readCommand(rw_conn)
		if err != nil {
			if err == io.EOF {
				log.Print("Client disconnected")
				break
			}
			log.Print(err.Error())
		}
	}
}

func (this *SimpleServer) readCommand(rw *bufio.ReadWriter) error {

	data, err := rw.ReadString(ENDL_BYTE)

	if err != nil {
		return err
	}

	data = strings.TrimRightFunc(data, unicode.IsSpace)
	splits := strings.Split(data, " ")
	var tokens []string
	for _, s := range splits {
		if len(s) > 0 {
			tokens = append(tokens, s)
		}
	}

	if len(splits) == 0 {
		return nil
	}

	switch strings.ToUpper(string(splits[0])) {
	case text_proto.CMD_PING:
		this.writeResponse(rw, text_proto.RESP_PONG)

	case text_proto.CMD_QUIT:
		this.writeResponse(rw, text_proto.RESP_BYE)
		return io.EOF

	case text_proto.CMD_UNIX_TS:
		stamp := fmt.Sprint(time.Now().Unix())
		return this.writeResponse(rw, stamp)

	case text_proto.CMD_CREATE_QUEUE:
		return this.createQueue(rw, tokens[1:])

	case text_proto.CMD_DROP_QUEUE:
		return this.dropQueue(rw, tokens[1:])

	default:
		return this.writeResponse(rw, text_proto.RESP_ERROR)
	}
	return nil
}

func (this *SimpleServer) writeResponse(rw *bufio.ReadWriter, line string) error {
	_, err := rw.WriteString(line)
	if err != nil {
		return err
	}
	_, err = rw.WriteString(ENDL)
	if err != nil {
		return err
	}

	err = rw.Flush()
	if err != nil {
		return err
	}
	return nil
}

func (this *SimpleServer) queueOp(rw *bufio.ReadWriter, args []string, queueFunc QueueOpFunc) error {
	if len(args) < 1 {
		return this.writeResponse(rw, text_proto.RESP_ERROR+" Not enough parameters")
	}

	if err := queueFunc(args); err == nil {
		return this.writeResponse(rw, text_proto.RESP_OK)
	} else {
		return this.writeResponse(rw, text_proto.RESP_ERROR+" "+err.Error())
	}
}

func (this *SimpleServer) createQueue(rw *bufio.ReadWriter, req []string) error {
	return this.queueOp(rw, req,
		func(args []string) error {
			return this.queueFacade.AddQueue(args[0], queue_factory.GetPQueue(args[0]))
		})
}

func (this *SimpleServer) dropQueue(rw *bufio.ReadWriter, req []string) error {
	return this.queueOp(rw, req,
		func(args []string) error {
			return this.queueFacade.DropQueue(args[0])
		})
}
<|endoftext|>