<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"unicode/utf16"
)

type File struct {
	stringPool     *ResStringPool
	resourceMap    []uint32
	notPrecessedNS map[ResStringPoolRef]ResStringPoolRef
	namespaces     map[ResStringPoolRef]ResStringPoolRef
	XMLBuffer      bytes.Buffer
}

const (
	RES_NULL_TYPE        = 0x0000
	RES_STRING_POOL_TYPE = 0x0001
	RES_TABLE_TYPE       = 0x0002
	RES_XML_TYPE         = 0x0003

	// Chunk types in RES_XML_TYPE
	RES_XML_FIRST_CHUNK_TYPE     = 0x0100
	RES_XML_START_NAMESPACE_TYPE = 0x0100
	RES_XML_END_NAMESPACE_TYPE   = 0x0101
	RES_XML_START_ELEMENT_TYPE   = 0x0102
	RES_XML_END_ELEMENT_TYPE     = 0x0103
	RES_XML_CDATA_TYPE           = 0x0104
	RES_XML_LAST_CHUNK_TYPE      = 0x017f

	// This contains a uint32_t array mapping strings in the string
	// pool back to resource identifiers.  It is optional.
	RES_XML_RESOURCE_MAP_TYPE = 0x0180

	// Chunk types in RES_TABLE_TYPE
	RES_TABLE_PACKAGE_TYPE   = 0x0200
	RES_TABLE_TYPE_TYPE      = 0x0201
	RES_TABLE_TYPE_SPEC_TYPE = 0x0202
)

type ResChunkHeader struct {
	Type       uint16
	HeaderSize uint16
	Size       uint32
}

const SORTED_FLAG = 1 << 0
const UTF8_FLAG = 1 << 8

type ResStringPoolRef uint32

const NilResStringPoolRef = ResStringPoolRef(0xFFFFFFFF)

type ResStringPoolHeader struct {
	Header      ResChunkHeader
	StringCount uint32
	StyleCount  uint32
	Flags       uint32
	StringStart uint32
	StylesStart uint32
}

type ResStringPool struct {
	Header  ResStringPoolHeader
	Strings []string
	Styles  []string
}

type ResXMLTreeNode struct {
	Header     ResChunkHeader
	LineNumber uint32
	Comment    ResStringPoolRef
}

type ResXMLTreeNamespaceExt struct {
	Prefix ResStringPoolRef
	Uri    ResStringPoolRef
}

type ResXMLTreeAttrExt struct {
	NS             ResStringPoolRef
	Name           ResStringPoolRef
	AttributeStart uint16
	AttributeSize  uint16
	AttributeCount uint16
	IdIndex        uint16
	ClassIndex     uint16
	StyleIndex     uint16
}

type ResXMLTreeAttribute struct {
	NS         ResStringPoolRef
	Name       ResStringPoolRef
	RawValue   ResStringPoolRef
	TypedValue ResValue
}

const (
	TYPE_NULL            = 0x00
	TYPE_REFERENCE       = 0x01
	TYPE_ATTRIBUTE       = 0x02
	TYPE_STRING          = 0x03
	TYPE_FLOAT           = 0x04
	TYPE_DIMENSION       = 0x05
	TYPE_FRACTION        = 0x06
	TYPE_FIRST_INT       = 0x10
	TYPE_INT_DEC         = 0x10
	TYPE_INT_HEX         = 0x11
	TYPE_INT_BOOLEAN     = 0x12
	TYPE_FIRST_COLOR_INT = 0x1c
	TYPE_INT_COLOR_ARGB8 = 0x1c
	TYPE_INT_COLOR_RGB8  = 0x1d
	TYPE_INT_COLOR_ARGB4 = 0x1e
	TYPE_INT_COLOR_RGB4  = 0x1f
	TYPE_LAST_COLOR_INT  = 0x1f
	TYPE_LAST_INT        = 0x1f
)

type ResValue struct {
	Size     uint16
	Res0     uint8
	DataType uint8
	Data     uint32
}

type ResXMLTreeEndElementExt struct {
	NS   ResStringPoolRef
	Name ResStringPoolRef
}

func NewFile(r io.ReaderAt) (*File, error) {
	f := new(File)
	fmt.Fprintf(&f.XMLBuffer, "<?xml version=\"1.0\" encoding=\"utf-8\"?>")
	sr := io.NewSectionReader(r, 0, 1<<63-1)

	header := new(ResChunkHeader)
	binary.Read(sr, binary.LittleEndian, header)
	offset := uint32(header.HeaderSize)

	for offset < header.Size {
		sr.Seek(int64(offset), os.SEEK_SET)
		chunkHeader := &ResChunkHeader{}
		binary.Read(sr, binary.LittleEndian, chunkHeader)

		var err error
		chunkReader := io.NewSectionReader(r, int64(offset), int64(chunkHeader.Size))
		switch chunkHeader.Type {
		case RES_STRING_POOL_TYPE:
			f.stringPool, err = ReadStringPool(chunkReader)
		case RES_XML_RESOURCE_MAP_TYPE:
			f.resourceMap, err = ReadResourceMap(chunkReader)
		case RES_XML_START_NAMESPACE_TYPE:
			err = f.ReadStartNamespace(chunkReader)
		case RES_XML_END_NAMESPACE_TYPE:
			err = f.ReadEndNamespace(chunkReader)
		case RES_XML_START_ELEMENT_TYPE:
			err = f.ReadStartElement(chunkReader)
		case RES_XML_END_ELEMENT_TYPE:
			err = f.ReadEndElement(chunkReader)
		}
		if err != nil {
			return nil, err
		}

		offset += chunkHeader.Size
	}
	return f, nil
}

func (f *File) GetString(ref ResStringPoolRef) string {
	if ref == NilResStringPoolRef {
		return ""
	}
	return f.stringPool.Strings[int(ref)]
}

func ReadStringPool(sr *io.SectionReader) (*ResStringPool, error) {
	sp := new(ResStringPool)
	binary.Read(sr, binary.LittleEndian, &sp.Header)

	stringStarts := make([]uint32, sp.Header.StringCount)
	binary.Read(sr, binary.LittleEndian, stringStarts)
	styleStarts := make([]uint32, sp.Header.StyleCount)
	binary.Read(sr, binary.LittleEndian, styleStarts)

	sp.Strings = make([]string, sp.Header.StringCount)
	for i, start := range stringStarts {
		var str string
		var err error
		if (sp.Header.Flags & UTF8_FLAG) == 0 {
			str, err = ReadUTF16(sr, int64(sp.Header.StringStart+start))
		} else {
			str, err = ReadUTF8(sr, int64(sp.Header.StringStart+start))
		}
		if err != nil {
			return nil, err
		}
		sp.Strings[i] = str
	}

	sp.Styles = make([]string, sp.Header.StyleCount)
	for i, start := range styleStarts {
		var str string
		var err error
		if (sp.Header.Flags & UTF8_FLAG) == 0 {
			str, err = ReadUTF16(sr, int64(sp.Header.StylesStart+start))
		} else {
			str, err = ReadUTF8(sr, int64(sp.Header.StylesStart+start))
		}
		if err != nil {
			return nil, err
		}
		sp.Styles[i] = str
	}

	return sp, nil
}

func ReadUTF16(sr *io.SectionReader, offset int64) (string, error) {
	// read lenth of string
	var size int
	var first, second uint16
	sr.Seek(offset, os.SEEK_SET)
	if err := binary.Read(sr, binary.LittleEndian, &first); err != nil {
		return "", err
	}
	if (first & 0x8000) != 0 {
		if err := binary.Read(sr, binary.LittleEndian, &second); err != nil {
			return "", err
		}
		size = (int(first&0x7FFF) << 16) + int(second)
	} else {
		size = int(first)
	}

	// read string value
	buf := make([]uint16, size)
	if err := binary.Read(sr, binary.LittleEndian, buf); err != nil {
		return "", err
	}
	return string(utf16.Decode(buf)), nil
}

func ReadUTF8(sr *io.SectionReader, offset int64) (string, error) {
	// read lenth of string
	var size int
	var first, second uint8
	sr.Seek(offset, os.SEEK_SET)
	if err := binary.Read(sr, binary.LittleEndian, &first); err != nil {
		return "", err
	}
	if (first & 0x80) != 0 {
		if err := binary.Read(sr, binary.LittleEndian, &second); err != nil {
			return "", err
		}
		size = (int(first&0x7F) << 8) + int(second)
	} else {
		size = int(first)
	}

	buf := make([]uint8, size)
	if err := binary.Read(sr, binary.LittleEndian, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}

func ReadResourceMap(sr *io.SectionReader) ([]uint32, error) {
	header := new(ResChunkHeader)
	binary.Read(sr, binary.LittleEndian, header)
	count := (header.Size - uint32(header.HeaderSize)) / 4
	resourceMap := make([]uint32, count)
	if err := binary.Read(sr, binary.LittleEndian, resourceMap); err != nil {
		return nil, err
	}
	return resourceMap, nil
}

func (f *File) ReadStartNamespace(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	binary.Read(sr, binary.LittleEndian, header)
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	namespace := new(ResXMLTreeNamespaceExt)
	binary.Read(sr, binary.LittleEndian, namespace)

	if f.notPrecessedNS == nil {
		f.notPrecessedNS = make(map[ResStringPoolRef]ResStringPoolRef)
	}
	f.notPrecessedNS[namespace.Uri] = namespace.Prefix

	if f.namespaces == nil {
		f.namespaces = make(map[ResStringPoolRef]ResStringPoolRef)
	}
	f.namespaces[namespace.Uri] = namespace.Prefix

	return nil
}

func (f *File) ReadEndNamespace(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	binary.Read(sr, binary.LittleEndian, header)
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	namespace := new(ResXMLTreeNamespaceExt)
	binary.Read(sr, binary.LittleEndian, namespace)
	delete(f.namespaces, namespace.Uri)
	return nil
}

func (f *File) ReadStartElement(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	binary.Read(sr, binary.LittleEndian, header)
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	ext := new(ResXMLTreeAttrExt)
	binary.Read(sr, binary.LittleEndian, ext)

	fmt.Fprintf(&f.XMLBuffer, "<%s", f.AddNamespace(ext.NS, ext.Name))

	// output XML namespaces
	if f.notPrecessedNS != nil {
		for uri, prefix := range f.notPrecessedNS {
			fmt.Fprintf(&f.XMLBuffer, " xmlns:%s=\"", f.GetString(prefix))
			xml.Escape(&f.XMLBuffer, []byte(f.GetString(uri)))
			fmt.Fprint(&f.XMLBuffer, "\"")
		}
		f.notPrecessedNS = nil
	}

	// process attributes
	offset := int64(ext.AttributeStart + header.Header.HeaderSize)
	for i := 0; i < int(ext.AttributeCount); i++ {
		sr.Seek(offset, os.SEEK_SET)
		attr := new(ResXMLTreeAttribute)
		binary.Read(sr, binary.LittleEndian, attr)

		var value string
		if attr.RawValue != NilResStringPoolRef {
			value = f.GetString(attr.RawValue)
		} else {
			data := attr.TypedValue.Data
			switch attr.TypedValue.DataType {
			case TYPE_NULL:
				value = ""
			case TYPE_REFERENCE:
				value = fmt.Sprintf("@0x%08X", data)
			case TYPE_INT_DEC:
				value = fmt.Sprintf("%d", data)
			case TYPE_INT_HEX:
				value = fmt.Sprintf("0x%08X", data)
			case TYPE_INT_BOOLEAN:
				if data != 0 {
					value = "true"
				} else {
					value = "false"
				}
			default:
				value = fmt.Sprintf("@0x%08X", data)
			}
		}

		fmt.Fprintf(&f.XMLBuffer, " %s=\"", f.AddNamespace(attr.NS, attr.Name))
		xml.Escape(&f.XMLBuffer, []byte(value))
		fmt.Fprint(&f.XMLBuffer, "\"")
		offset += int64(ext.AttributeSize)
	}
	fmt.Fprint(&f.XMLBuffer, ">")
	return nil
}

func (f *File) ReadEndElement(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	if err := binary.Read(sr, binary.LittleEndian, header); err != nil {
		return err
	}
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	ext := new(ResXMLTreeEndElementExt)
	if err := binary.Read(sr, binary.LittleEndian, ext); err != nil {
		return err
	}
	fmt.Fprintf(&f.XMLBuffer, "</%s>", f.AddNamespace(ext.NS, ext.Name))
	return nil
}

func (f *File) AddNamespace(ns, name ResStringPoolRef) string {
	if ns != NilResStringPoolRef {
		prefix := f.GetString(f.namespaces[ns])
		return fmt.Sprintf("%s:%s", prefix, f.GetString(name))
	} else {
		return f.GetString(name)
	}
}

func main() {
	f, _ := os.Open("AndroidManifest.xml")
	xml, _ := NewFile(f)
	fmt.Println(xml.XMLBuffer.String())
}
<commit_msg>RES_XML_TYPEも他のリソースと同様に扱う<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"unicode/utf16"
)

type File struct {
	stringPool     *ResStringPool
	resourceMap    []uint32
	notPrecessedNS map[ResStringPoolRef]ResStringPoolRef
	namespaces     map[ResStringPoolRef]ResStringPoolRef
	XMLBuffer      bytes.Buffer
}

const (
	RES_NULL_TYPE        = 0x0000
	RES_STRING_POOL_TYPE = 0x0001
	RES_TABLE_TYPE       = 0x0002
	RES_XML_TYPE         = 0x0003

	// Chunk types in RES_XML_TYPE
	RES_XML_FIRST_CHUNK_TYPE     = 0x0100
	RES_XML_START_NAMESPACE_TYPE = 0x0100
	RES_XML_END_NAMESPACE_TYPE   = 0x0101
	RES_XML_START_ELEMENT_TYPE   = 0x0102
	RES_XML_END_ELEMENT_TYPE     = 0x0103
	RES_XML_CDATA_TYPE           = 0x0104
	RES_XML_LAST_CHUNK_TYPE      = 0x017f

	// This contains a uint32_t array mapping strings in the string
	// pool back to resource identifiers.  It is optional.
	RES_XML_RESOURCE_MAP_TYPE = 0x0180

	// Chunk types in RES_TABLE_TYPE
	RES_TABLE_PACKAGE_TYPE   = 0x0200
	RES_TABLE_TYPE_TYPE      = 0x0201
	RES_TABLE_TYPE_SPEC_TYPE = 0x0202
)

type ResChunkHeader struct {
	Type       uint16
	HeaderSize uint16
	Size       uint32
}

const SORTED_FLAG = 1 << 0
const UTF8_FLAG = 1 << 8

type ResStringPoolRef uint32

const NilResStringPoolRef = ResStringPoolRef(0xFFFFFFFF)

type ResStringPoolHeader struct {
	Header      ResChunkHeader
	StringCount uint32
	StyleCount  uint32
	Flags       uint32
	StringStart uint32
	StylesStart uint32
}

type ResStringPool struct {
	Header  ResStringPoolHeader
	Strings []string
	Styles  []string
}

type ResXMLTreeNode struct {
	Header     ResChunkHeader
	LineNumber uint32
	Comment    ResStringPoolRef
}

type ResXMLTreeNamespaceExt struct {
	Prefix ResStringPoolRef
	Uri    ResStringPoolRef
}

type ResXMLTreeAttrExt struct {
	NS             ResStringPoolRef
	Name           ResStringPoolRef
	AttributeStart uint16
	AttributeSize  uint16
	AttributeCount uint16
	IdIndex        uint16
	ClassIndex     uint16
	StyleIndex     uint16
}

type ResXMLTreeAttribute struct {
	NS         ResStringPoolRef
	Name       ResStringPoolRef
	RawValue   ResStringPoolRef
	TypedValue ResValue
}

const (
	TYPE_NULL            = 0x00
	TYPE_REFERENCE       = 0x01
	TYPE_ATTRIBUTE       = 0x02
	TYPE_STRING          = 0x03
	TYPE_FLOAT           = 0x04
	TYPE_DIMENSION       = 0x05
	TYPE_FRACTION        = 0x06
	TYPE_FIRST_INT       = 0x10
	TYPE_INT_DEC         = 0x10
	TYPE_INT_HEX         = 0x11
	TYPE_INT_BOOLEAN     = 0x12
	TYPE_FIRST_COLOR_INT = 0x1c
	TYPE_INT_COLOR_ARGB8 = 0x1c
	TYPE_INT_COLOR_RGB8  = 0x1d
	TYPE_INT_COLOR_ARGB4 = 0x1e
	TYPE_INT_COLOR_RGB4  = 0x1f
	TYPE_LAST_COLOR_INT  = 0x1f
	TYPE_LAST_INT        = 0x1f
)

type ResValue struct {
	Size     uint16
	Res0     uint8
	DataType uint8
	Data     uint32
}

type ResXMLTreeEndElementExt struct {
	NS   ResStringPoolRef
	Name ResStringPoolRef
}

func NewFile(r io.ReaderAt) (*File, error) {
	f := new(File)
	f.readChunk(r, 0)
	return f, nil
}

func (f *File) readChunk(r io.ReaderAt, offset int64) (*ResChunkHeader, error) {
	sr := io.NewSectionReader(r, offset, 1<<63-1-offset)
	chunkHeader := &ResChunkHeader{}
	sr.Seek(0, os.SEEK_SET)
	if err := binary.Read(sr, binary.LittleEndian, chunkHeader); err != nil {
		return nil, err
	}

	var err error
	sr.Seek(0, os.SEEK_SET)
	switch chunkHeader.Type {
	case RES_XML_TYPE:
		err = f.readXML(sr)
	case RES_STRING_POOL_TYPE:
		f.stringPool, err = ReadStringPool(sr)
	case RES_XML_RESOURCE_MAP_TYPE:
		f.resourceMap, err = ReadResourceMap(sr)
	case RES_XML_START_NAMESPACE_TYPE:
		err = f.ReadStartNamespace(sr)
	case RES_XML_END_NAMESPACE_TYPE:
		err = f.ReadEndNamespace(sr)
	case RES_XML_START_ELEMENT_TYPE:
		err = f.ReadStartElement(sr)
	case RES_XML_END_ELEMENT_TYPE:
		err = f.ReadEndElement(sr)
	}
	if err != nil {
		return nil, err
	}

	return chunkHeader, nil
}

func (f *File) readXML(sr *io.SectionReader) error {
	fmt.Fprintf(&f.XMLBuffer, "<?xml version=\"1.0\" encoding=\"utf-8\"?>")

	header := new(ResChunkHeader)
	binary.Read(sr, binary.LittleEndian, header)
	offset := int64(header.HeaderSize)
	for offset < int64(header.Size) {
		chunkHeader, err := f.readChunk(sr, offset)
		if err != nil {
			return err
		}
		offset += int64(chunkHeader.Size)
	}
	return nil
}

func (f *File) GetString(ref ResStringPoolRef) string {
	if ref == NilResStringPoolRef {
		return ""
	}
	return f.stringPool.Strings[int(ref)]
}

func ReadStringPool(sr *io.SectionReader) (*ResStringPool, error) {
	sp := new(ResStringPool)
	binary.Read(sr, binary.LittleEndian, &sp.Header)

	stringStarts := make([]uint32, sp.Header.StringCount)
	binary.Read(sr, binary.LittleEndian, stringStarts)
	styleStarts := make([]uint32, sp.Header.StyleCount)
	binary.Read(sr, binary.LittleEndian, styleStarts)

	sp.Strings = make([]string, sp.Header.StringCount)
	for i, start := range stringStarts {
		var str string
		var err error
		if (sp.Header.Flags & UTF8_FLAG) == 0 {
			str, err = ReadUTF16(sr, int64(sp.Header.StringStart+start))
		} else {
			str, err = ReadUTF8(sr, int64(sp.Header.StringStart+start))
		}
		if err != nil {
			return nil, err
		}
		sp.Strings[i] = str
	}

	sp.Styles = make([]string, sp.Header.StyleCount)
	for i, start := range styleStarts {
		var str string
		var err error
		if (sp.Header.Flags & UTF8_FLAG) == 0 {
			str, err = ReadUTF16(sr, int64(sp.Header.StylesStart+start))
		} else {
			str, err = ReadUTF8(sr, int64(sp.Header.StylesStart+start))
		}
		if err != nil {
			return nil, err
		}
		sp.Styles[i] = str
	}

	return sp, nil
}

func ReadUTF16(sr *io.SectionReader, offset int64) (string, error) {
	// read lenth of string
	var size int
	var first, second uint16
	sr.Seek(offset, os.SEEK_SET)
	if err := binary.Read(sr, binary.LittleEndian, &first); err != nil {
		return "", err
	}
	if (first & 0x8000) != 0 {
		if err := binary.Read(sr, binary.LittleEndian, &second); err != nil {
			return "", err
		}
		size = (int(first&0x7FFF) << 16) + int(second)
	} else {
		size = int(first)
	}

	// read string value
	buf := make([]uint16, size)
	if err := binary.Read(sr, binary.LittleEndian, buf); err != nil {
		return "", err
	}
	return string(utf16.Decode(buf)), nil
}

func ReadUTF8(sr *io.SectionReader, offset int64) (string, error) {
	// read lenth of string
	var size int
	var first, second uint8
	sr.Seek(offset, os.SEEK_SET)
	if err := binary.Read(sr, binary.LittleEndian, &first); err != nil {
		return "", err
	}
	if (first & 0x80) != 0 {
		if err := binary.Read(sr, binary.LittleEndian, &second); err != nil {
			return "", err
		}
		size = (int(first&0x7F) << 8) + int(second)
	} else {
		size = int(first)
	}

	buf := make([]uint8, size)
	if err := binary.Read(sr, binary.LittleEndian, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}

func ReadResourceMap(sr *io.SectionReader) ([]uint32, error) {
	header := new(ResChunkHeader)
	binary.Read(sr, binary.LittleEndian, header)
	count := (header.Size - uint32(header.HeaderSize)) / 4
	resourceMap := make([]uint32, count)
	if err := binary.Read(sr, binary.LittleEndian, resourceMap); err != nil {
		return nil, err
	}
	return resourceMap, nil
}

func (f *File) ReadStartNamespace(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	binary.Read(sr, binary.LittleEndian, header)
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	namespace := new(ResXMLTreeNamespaceExt)
	binary.Read(sr, binary.LittleEndian, namespace)

	if f.notPrecessedNS == nil {
		f.notPrecessedNS = make(map[ResStringPoolRef]ResStringPoolRef)
	}
	f.notPrecessedNS[namespace.Uri] = namespace.Prefix

	if f.namespaces == nil {
		f.namespaces = make(map[ResStringPoolRef]ResStringPoolRef)
	}
	f.namespaces[namespace.Uri] = namespace.Prefix

	return nil
}

func (f *File) ReadEndNamespace(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	binary.Read(sr, binary.LittleEndian, header)
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	namespace := new(ResXMLTreeNamespaceExt)
	binary.Read(sr, binary.LittleEndian, namespace)
	delete(f.namespaces, namespace.Uri)
	return nil
}

func (f *File) ReadStartElement(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	binary.Read(sr, binary.LittleEndian, header)
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	ext := new(ResXMLTreeAttrExt)
	binary.Read(sr, binary.LittleEndian, ext)

	fmt.Fprintf(&f.XMLBuffer, "<%s", f.AddNamespace(ext.NS, ext.Name))

	// output XML namespaces
	if f.notPrecessedNS != nil {
		for uri, prefix := range f.notPrecessedNS {
			fmt.Fprintf(&f.XMLBuffer, " xmlns:%s=\"", f.GetString(prefix))
			xml.Escape(&f.XMLBuffer, []byte(f.GetString(uri)))
			fmt.Fprint(&f.XMLBuffer, "\"")
		}
		f.notPrecessedNS = nil
	}

	// process attributes
	offset := int64(ext.AttributeStart + header.Header.HeaderSize)
	for i := 0; i < int(ext.AttributeCount); i++ {
		sr.Seek(offset, os.SEEK_SET)
		attr := new(ResXMLTreeAttribute)
		binary.Read(sr, binary.LittleEndian, attr)

		var value string
		if attr.RawValue != NilResStringPoolRef {
			value = f.GetString(attr.RawValue)
		} else {
			data := attr.TypedValue.Data
			switch attr.TypedValue.DataType {
			case TYPE_NULL:
				value = ""
			case TYPE_REFERENCE:
				value = fmt.Sprintf("@0x%08X", data)
			case TYPE_INT_DEC:
				value = fmt.Sprintf("%d", data)
			case TYPE_INT_HEX:
				value = fmt.Sprintf("0x%08X", data)
			case TYPE_INT_BOOLEAN:
				if data != 0 {
					value = "true"
				} else {
					value = "false"
				}
			default:
				value = fmt.Sprintf("@0x%08X", data)
			}
		}

		fmt.Fprintf(&f.XMLBuffer, " %s=\"", f.AddNamespace(attr.NS, attr.Name))
		xml.Escape(&f.XMLBuffer, []byte(value))
		fmt.Fprint(&f.XMLBuffer, "\"")
		offset += int64(ext.AttributeSize)
	}
	fmt.Fprint(&f.XMLBuffer, ">")
	return nil
}

func (f *File) ReadEndElement(sr *io.SectionReader) error {
	header := new(ResXMLTreeNode)
	if err := binary.Read(sr, binary.LittleEndian, header); err != nil {
		return err
	}
	sr.Seek(int64(header.Header.HeaderSize), os.SEEK_SET)
	ext := new(ResXMLTreeEndElementExt)
	if err := binary.Read(sr, binary.LittleEndian, ext); err != nil {
		return err
	}
	fmt.Fprintf(&f.XMLBuffer, "</%s>", f.AddNamespace(ext.NS, ext.Name))
	return nil
}

func (f *File) AddNamespace(ns, name ResStringPoolRef) string {
	if ns != NilResStringPoolRef {
		prefix := f.GetString(f.namespaces[ns])
		return fmt.Sprintf("%s:%s", prefix, f.GetString(name))
	} else {
		return f.GetString(name)
	}
}

func main() {
	f, _ := os.Open("AndroidManifest.xml")
	xml, _ := NewFile(f)
	fmt.Println(xml.XMLBuffer.String())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"gopkg.in/docopt/docopt.go.v0"
	"gopkg.in/hashicorp/logutils.v0"
)

//go:generate go run misc/include_jsons.go

const (
	binName   = "testman"
	binTitle  = binName + "/" + binVersion
	envAPIKey = "COVEREDCI_API_KEY"
)

var (
	isDebug     bool
	apiRoot     string
	initURL     string
	nextURL     string
	docsURL     string
	clientUtils = &http.Client{}
)

func init() {
	log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.LUTC)

	isDebug = "0.0.0" == binVersion

	if isDebug {
		apiRoot = "http://localhost:1042/1"
		docsURL = "http://localhost:2042/1/blob"
	} else {
		apiRoot = "https://test.coveredci.com/1"
		docsURL = "https://lint.coveredci.com/1/blob"
	}
	initURL = apiRoot + "/init"
	nextURL = apiRoot + "/next"

	unstacheInit()

	loadSchemas()

	makePwdID()
}

func main() {
	os.Exit(actualMain())
}

func usage() (map[string]interface{}, error) {
	usage := `testman

Usage:
  testman [-vvv] test
  testman [-vvv] validate
  testman -h | --help
  testman -V | --version

Options:
  -v, -vv, -vvv  Verbosity level
  -h, --help     Show this screen
  -V, --version  Show version`

	return docopt.Parse(usage, nil, true, binTitle, true)
}

func actualMain() int {
	args, err := usage()
	if err != nil {
		log.Println("!args: ", err)
		return retryOrReport()
	}

	logFile := pwdID + ".log"
	logCatchall, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE, 0640)
	if err != nil {
		log.Println(err)
		return retryOrReport()
	}
	defer logCatchall.Close()
	logFiltered := &logutils.LevelFilter{
		Levels:   []logutils.LogLevel{"DBG", "NFO", "ERR", "NOP"},
		MinLevel: logLevel(args),
		Writer:   os.Stderr,
	}
	log.SetOutput(io.MultiWriter(logCatchall, logFiltered))
	log.Println("[ERR]", binTitle, logFile, args)

	if !isDebug {
		if code := isRunningLatest(); code != 0 {
			return code
		}
	}

	apiKey := getAPIKey()
	if args["validate"].(bool) {
		return doValidate(apiKey)
	}

	// args["test"].(bool) = true
	return doTest(apiKey)
}

func ensureDeleted(path string) {
	if err := os.Remove(path); err != nil && os.IsExist(err) {
		fmt.Println(err)
		log.Panic("[ERR] ", err)
	}
}

func getAPIKey() string {
	apiKey := os.Getenv(envAPIKey)
	if isDebug {
		apiKey = "42"
	}
	return apiKey
}

func logLevel(args map[string]interface{}) logutils.LogLevel {
	var lvl string
	switch args["-v"].(int) {
	case 1:
		lvl = "ERR"
	case 2:
		lvl = "NFO"
	case 3:
		lvl = "DBG"
	default:
		lvl = "NOP"
	}
	return logutils.LogLevel(lvl)
}

func isRunningLatest() int {
	latest, err := getLatestRelease()
	if err != nil {
		return retryOrReport()
	}

	ko, err := isOutOfDate(binVersion, latest)
	if err != nil {
		return retryOrReport()
	}
	if ko {
		err := fmt.Errorf("A newer version of %s is out: %s (you have %s)", binName, latest, binVersion)
		log.Println("[ERR]", err)
		fmt.Println(err)
		return 3
	}

	return 0
}

func doValidate(apiKey string) int {
	if yml, err := readYML(); err == nil {
		if _, err := validateDocs(apiKey, yml); err != nil {
			return 2
		}
		return 0
	}
	return retryOrReport()
}

func doTest(apiKey string) int {
	if _, err := os.Stat(shell()); os.IsNotExist(err) {
		log.Println(shell() + " is required")
		return 5
	}

	if apiKey == "" {
		log.Println("$" + envAPIKey + " is unset")
		return 4
	}

	envSerializedPath := pwdID + ".env"
	ensureDeleted(envSerializedPath)
	if err := snapEnv(envSerializedPath); err != nil {
		return retryOrReport()
	}
	defer ensureDeleted(envSerializedPath)

	cfg, cmd, err := initDialogue(apiKey)
	if err != nil {
		if _, ok := err.(*docsInvalidError); ok {
			return 2
		}
		return retryOrReport()
	}

	for {
		if cmd.Kind() == "done" {
			return testOutcome(cmd.(doneCmd))
		}

		if cmd, err = next(cfg, cmd); err != nil {
			return retryOrReport()
		}
	}
}

func retryOrReport() int {
	issues := "https://github.com/CoveredCI/testman/issues"
	email := "hi@coveredci.co"
	fmt.Println("Looks like something went wrong... Maybe try again?")
	fmt.Printf("\tYou may want to take a look at %s.log\n", pwdID)
	fmt.Printf("\tor come by %s\n", issues)
	fmt.Printf("\tor drop us a line at %s\n", email)
	fmt.Println("Thanks & sorry about this :)")
	return 1
}
<commit_msg>ports: tmp endpoints (#17)<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"gopkg.in/docopt/docopt.go.v0"
	"gopkg.in/hashicorp/logutils.v0"
)

//go:generate go run misc/include_jsons.go

const (
	binName   = "testman"
	binTitle  = binName + "/" + binVersion
	envAPIKey = "COVEREDCI_API_KEY"
)

var (
	isDebug     bool
	apiRoot     string
	initURL     string
	nextURL     string
	docsURL     string
	clientUtils = &http.Client{}
)

func init() {
	log.SetFlags(log.Lshortfile | log.Lmicroseconds | log.LUTC)

	isDebug = "0.0.0" == binVersion

	if isDebug {
		apiRoot = "http://localhost:1042/1"
		docsURL = "http://localhost:2042/1/blob"
	} else {
		apiRoot = "http://test.dev.coveredci.com:1042/1"      //FIXME
		docsURL = "http://lint.dev.coveredci.com:2042/1/blob" //FIXME
	}
	initURL = apiRoot + "/init"
	nextURL = apiRoot + "/next"

	unstacheInit()

	loadSchemas()

	makePwdID()
}

func main() {
	os.Exit(actualMain())
}

func usage() (map[string]interface{}, error) {
	usage := `testman

Usage:
  testman [-vvv] test
  testman [-vvv] validate
  testman -h | --help
  testman -V | --version

Options:
  -v, -vv, -vvv  Verbosity level
  -h, --help     Show this screen
  -V, --version  Show version`

	return docopt.Parse(usage, nil, true, binTitle, true)
}

func actualMain() int {
	args, err := usage()
	if err != nil {
		log.Println("!args: ", err)
		return retryOrReport()
	}

	logFile := pwdID + ".log"
	logCatchall, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE, 0640)
	if err != nil {
		log.Println(err)
		return retryOrReport()
	}
	defer logCatchall.Close()
	logFiltered := &logutils.LevelFilter{
		Levels:   []logutils.LogLevel{"DBG", "NFO", "ERR", "NOP"},
		MinLevel: logLevel(args),
		Writer:   os.Stderr,
	}
	log.SetOutput(io.MultiWriter(logCatchall, logFiltered))
	log.Println("[ERR]", binTitle, logFile, args)

	if !isDebug {
		if code := isRunningLatest(); code != 0 {
			return code
		}
	}

	apiKey := getAPIKey()
	if args["validate"].(bool) {
		return doValidate(apiKey)
	}

	// args["test"].(bool) = true
	return doTest(apiKey)
}

func ensureDeleted(path string) {
	if err := os.Remove(path); err != nil && os.IsExist(err) {
		fmt.Println(err)
		log.Panic("[ERR] ", err)
	}
}

func getAPIKey() string {
	apiKey := os.Getenv(envAPIKey)
	if isDebug {
		apiKey = "42"
	}
	return apiKey
}

func logLevel(args map[string]interface{}) logutils.LogLevel {
	var lvl string
	switch args["-v"].(int) {
	case 1:
		lvl = "ERR"
	case 2:
		lvl = "NFO"
	case 3:
		lvl = "DBG"
	default:
		lvl = "NOP"
	}
	return logutils.LogLevel(lvl)
}

func isRunningLatest() int {
	latest, err := getLatestRelease()
	if err != nil {
		return retryOrReport()
	}

	ko, err := isOutOfDate(binVersion, latest)
	if err != nil {
		return retryOrReport()
	}
	if ko {
		err := fmt.Errorf("A newer version of %s is out: %s (you have %s)", binName, latest, binVersion)
		log.Println("[ERR]", err)
		fmt.Println(err)
		return 3
	}

	return 0
}

func doValidate(apiKey string) int {
	if yml, err := readYML(); err == nil {
		if _, err := validateDocs(apiKey, yml); err != nil {
			return 2
		}
		return 0
	}
	return retryOrReport()
}

func doTest(apiKey string) int {
	if _, err := os.Stat(shell()); os.IsNotExist(err) {
		log.Println(shell() + " is required")
		return 5
	}

	if apiKey == "" {
		log.Println("$" + envAPIKey + " is unset")
		return 4
	}

	envSerializedPath := pwdID + ".env"
	ensureDeleted(envSerializedPath)
	if err := snapEnv(envSerializedPath); err != nil {
		return retryOrReport()
	}
	defer ensureDeleted(envSerializedPath)

	cfg, cmd, err := initDialogue(apiKey)
	if err != nil {
		if _, ok := err.(*docsInvalidError); ok {
			return 2
		}
		return retryOrReport()
	}

	for {
		if cmd.Kind() == "done" {
			return testOutcome(cmd.(doneCmd))
		}

		if cmd, err = next(cfg, cmd); err != nil {
			return retryOrReport()
		}
	}
}

func retryOrReport() int {
	issues := "https://github.com/CoveredCI/testman/issues"
	email := "hi@coveredci.co"
	fmt.Println("Looks like something went wrong... Maybe try again?")
	fmt.Printf("\tYou may want to take a look at %s.log\n", pwdID)
	fmt.Printf("\tor come by %s\n", issues)
	fmt.Printf("\tor drop us a line at %s\n", email)
	fmt.Println("Thanks & sorry about this :)")
	return 1
}
<|endoftext|>


<commit_before>// Count lines of input per second on stdin
package main

import (
	"bufio"
	"fmt"
	"os"
	"time"
	"flag"
)

func readLines(c chan int) {
	count := 0
	bio := bufio.NewReader(os.Stdin)
	for {
		_, more, err := bio.ReadLine()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if more {
			continue
		}
		count += 1
		c <- count
	}
}

func main() {
	var d time.Duration
	flag.DurationVar(&d, "i", time.Second, "Update interval")
	flag.Parse()
	line := 0
	count := 0
	c := make(chan int)
	tick := time.Tick(d)
	go readLines(c)

	for {
		select {
		// print counts
		case <-tick:
			fmt.Println(float64(line-count)/d.Seconds(), "/sec")
			count = line
		// update counts
		case line = <-c:
		}
	}
}
<commit_msg>Add flag to print timestamp with rate<commit_after>// Count lines of input per second on stdin
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"time"
)

func readLines(c chan int) {
	count := 0
	bio := bufio.NewReader(os.Stdin)
	for {
		_, more, err := bio.ReadLine()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if more {
			continue
		}
		count += 1
		c <- count
	}
}

func main() {
	var d time.Duration
	var t bool
	flag.DurationVar(&d, "i", time.Second, "Update interval")
	flag.BoolVar(&t, "t", false, "Include timestamp")
	flag.Parse()
	line := 0
	count := 0
	c := make(chan int)
	tick := time.Tick(d)
	go readLines(c)

	for {
		select {
		// print counts
		case <-tick:
			prnt := fmt.Sprintf("%v /sec", float64(line-count)/d.Seconds())
			if t {
				prnt = fmt.Sprintf("%s\t%s", prnt, time.Now().UTC().Format("Mon Jan 2 15:04:05 UTC 2006"))
			}
			fmt.Println(prnt)
			count = line
		// update counts
		case line = <-c:
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mrjones/oauth"
)

const (
	requestTokenURL   string = "https://api.twitter.com/oauth/request_token"
	authorizeTokenURL string = "https://api.twitter.com/oauth/authorize"
	accessTokenURL    string = "https://api.twitter.com/oauth/access_token"
	apiBase           string = "https://api.twitter.com/1.1/"
	apiGetFavorites   string = apiBase + "favorites/list.json?count=200"
	apiUnFavorite     string = apiBase + "favorites/destroy.json?id="
	apiRequestLimit   int    = 75
)

var (
	lastTweetID string
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	flag.StringVar(&lastTweetID, "tweetid", "", "Get media with an ID less than (that is, older than) or equal to the specified ID.")
	flag.StringVar(&lastTweetID, "i", "", "Get media with an ID less than (that is, older than) or equal to the specified ID.")
	flag.Parse()
}

func getHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}

		return home
	}
	return os.Getenv("HOME")
}

func getConfig() (string, map[string]string, error) {
	cfgFolder := os.Getenv("HOME")

	if cfgFolder == "" && runtime.GOOS == "windows" {
		cfgFolder = os.Getenv("APPDATA")

		if cfgFolder == "" {
			cfgFolder = filepath.Join(os.Getenv("USERPROFILE"), "Application Data", "twitter-favorite-pics")
		}

		cfgFolder = filepath.Join(cfgFolder, "twitter-favorite-pics")
	} else {
		cfgFolder = filepath.Join(cfgFolder, ".config", "twitter-favorite-pics")
	}

	if err := os.MkdirAll(cfgFolder, 0700); err != nil {
		return "", nil, err
	}

	cfgFile := filepath.Join(cfgFolder, "settings.json")

	cfg := map[string]string{}

	buf, err := ioutil.ReadFile(cfgFile)
	if err != nil && !os.IsNotExist(err) {
		return "", nil, err
	}

	if err != nil {
		var consumerKey, consumerSecret, dlPath, filterAccount, dlWithoutAsk, unFavAfterDL string
		fmt.Print("Enter consumer key: ")
		fmt.Scanln(&consumerKey)
		fmt.Print("Enter consumer secret: ")
		fmt.Scanln(&consumerSecret)
		fmt.Print("Enter download path: ")
		fmt.Scanln(&dlPath)
		fmt.Print("Enter twitter screen name that want to filter for download (separate by comma): ")
		fmt.Scanln(&filterAccount)
		fmt.Print("Continue download without asking? (y/N): ")
		fmt.Scanln(&dlWithoutAsk)
		dlWithoutAsk = strings.ToLower(dlWithoutAsk)
		fmt.Print("Un-favorite tweet after download? (y/N): ")
		fmt.Scanln(&unFavAfterDL)
		unFavAfterDL = strings.ToLower(unFavAfterDL)

		cfg["ConsumerKey"] = consumerKey
		cfg["ConsumerSecret"] = consumerSecret
		cfg["DownloadPath"] = dlPath
		cfg["FilterAccount"] = filterAccount

		if dlWithoutAsk == "y" || dlWithoutAsk == "yes" {
			cfg["DownloadWithoutAsking"] = "true"
		} else {
			cfg["DownloadWithoutAsking"] = "false"
		}

		if unFavAfterDL == "y" || unFavAfterDL == "yes" {
			cfg["UnFavAfterDownload"] = "true"
		} else {
			cfg["UnFavAfterDownload"] = "false"
		}
	} else {
		err = json.Unmarshal(buf, &cfg)
		if err != nil {
			return "", nil, fmt.Errorf("Could not unmarshal %v: %v", cfgFile, err)
		}
	}

	return cfgFile, cfg, nil
}

func writeConfig(cfg map[string]string, cfgFile string) error {
	buf, err := json.MarshalIndent(cfg, "", "  ")
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(cfgFile, buf, 0700)
	if err != nil {
		return err
	}

	return nil
}

func openBrowser(url string) error {
	browser := "xdg-open"
	args := []string{url}

	if runtime.GOOS == "windows" {
		browser = "rundll32.exe"
		args = []string{"url.dll,FileProtocolHandler", url}
	} else if runtime.GOOS == "darwin" {
		browser = "open"
	}

	browser, err := exec.LookPath(browser)
	if err != nil {
		return err
	}

	cmd := exec.Command(browser, args...)
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	if err != nil {
		return err
	}

	return nil
}

func getAuthorizeToken(c *oauth.Consumer, cfg map[string]string) (*oauth.AccessToken, error) {
	accessToken, foundToken := cfg["AccessToken"]
	accessSecret, foundSecret := cfg["AccessSecret"]

	var authorizeToken *oauth.AccessToken

	if foundToken && foundSecret {
		authorizeToken = &oauth.AccessToken{Token: accessToken, Secret: accessSecret}
	} else {
		reqToken, url, err := c.GetRequestTokenAndUrl("")
		if err != nil {
			return nil, err
		}

		fmt.Println("(1) Go to: " + url)
		fmt.Println("(2) Grant access, you should get back a verification code.")
		fmt.Print("(3) Enter that verification code here: ")

		err = openBrowser(url)
		if err != nil {
			return nil, err
		}

		verificationCode := ""
		fmt.Scanln(&verificationCode)

		authorizeToken, err = c.AuthorizeToken(reqToken, verificationCode)
		if err != nil {
			return nil, err
		}
	}

	return authorizeToken, nil
}

func unFavoriteTweet(wg *sync.WaitGroup, client *http.Client, tweetID string) {
	defer wg.Done()

	url := apiUnFavorite + tweetID

	reqBody := map[string]string{"id": tweetID}
	jsonBody, _ := json.Marshal(reqBody)
	resp, err := client.Post(url, "application/json", bytes.NewBuffer(jsonBody))
	if err != nil {
		log.Println(err)
	}

	if resp.StatusCode != http.StatusOK {
		log.Printf("%v: %v", url, resp.Status)
	}
}

func downloadWorker(wg *sync.WaitGroup, url string, dlPath string, fileName string) {
	defer wg.Done()

	resp, err := http.Get(url)
	if err != nil {
		log.Println(err)
	}

	defer resp.Body.Close()

	if err := os.MkdirAll(dlPath, 0755); err != nil {
		log.Println(err)
	}

	f, err := os.Create(filepath.Join(dlPath, fileName))
	if err != nil {
		log.Println(err)
	}

	_, err = io.Copy(f, resp.Body)
	if err != nil {
		log.Println(err)
	}

	f.Close()
}

func downloadMedia(client *http.Client, url string, dlPath string, filterAccount []string, unFav bool) (string, error) {
	resp, err := client.Get(url)
	if err != nil {
		return "", err
	}

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf(resp.Status)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var fav FavoriteList

	if err := json.Unmarshal([]byte(body), &fav); err != nil {
		return "", err
	}

	var wg sync.WaitGroup

	for _, v := range fav {
		i := sort.SearchStrings(filterAccount, v.User.ScreenName)

		if (i < len(filterAccount) && filterAccount[i] == v.User.ScreenName) || len(filterAccount) <= 0 {
			for _, val := range v.Entities.Media {
				largeMediaURL := val.MediaURL + ":large"
				sl := strings.Split(val.MediaURL, "/")
				fileName := sl[len(sl)-1]

				wg.Add(1)
				go downloadWorker(&wg, largeMediaURL, filepath.Join(dlPath, v.User.ScreenName), fileName)
			}

			if unFav && len(v.Entities.Media) > 0 {
				wg.Add(1)
				go unFavoriteTweet(&wg, client, v.IDStr)
			}
		}

		lastTweetID = v.IDStr
		fmt.Printf("Get from: %v\n", lastTweetID)
	}

	wg.Wait()

	return lastTweetID, nil
}

func main() {
	if lastTweetID != "" {
		fmt.Printf("Specify tweet ID: %v\n", lastTweetID)
	}

	cfgFile, cfg, err := getConfig()
	if err != nil {
		log.Fatalln("Get configuration file failed: ", err)
	}

	var filterAccount []string
	if len(cfg["FilterAccount"]) > 0 {
		fmt.Printf("Filter account: %v\n", cfg["FilterAccount"])

		filterAccount = strings.Split(cfg["FilterAccount"], ",")
		sort.Strings(filterAccount)
	}

	var dlWithoutAsk bool
	if strings.ToLower(cfg["DownloadWithoutAsking"]) == "true" {
		dlWithoutAsk = true
	} else {
		dlWithoutAsk = false
	}

	var unFav bool
	if strings.ToLower(cfg["UnFavAfterDownload"]) == "true" {
		unFav = true
	} else {
		unFav = false
	}

	c := oauth.NewConsumer(
		cfg["ConsumerKey"],
		cfg["ConsumerSecret"],
		oauth.ServiceProvider{
			RequestTokenUrl:   requestTokenURL,
			AuthorizeTokenUrl: authorizeTokenURL,
			AccessTokenUrl:    accessTokenURL,
		},
	)

	authorizeToken, err := getAuthorizeToken(c, cfg)
	if err != nil {
		log.Fatalln(err)
	}

	cfg["AccessToken"] = authorizeToken.Token
	cfg["AccessSecret"] = authorizeToken.Secret
	err = writeConfig(cfg, cfgFile)
	if err != nil {
		log.Fatalln(err)
	}

	client, err := c.MakeHttpClient(authorizeToken)
	if err != nil {
		log.Fatalln(err)
	}

	dlPath, foundDLPath := cfg["DownloadPath"]
	if !foundDLPath {
		currentPath, err := filepath.Abs(filepath.Dir(os.Args[0]))
		if err != nil {
			log.Fatalln(err)
		}

		dlPath = filepath.Join(currentPath, "downloads")
	}
	if dlPath[:2] == "~/" {
		dlPath = filepath.Join(getHomeDir(), dlPath[2:])
	}
	dlPath = filepath.Join(dlPath, "twitter-favorite-pics")

	continueDL := "y"
	reqCounter := 0
	for continueDL == "y" || dlWithoutAsk {
		if reqCounter >= apiRequestLimit {
			fmt.Printf("API request quota exceeded, wait for 16 minute to unlock...")
			time.Sleep(16 * time.Minute)
			reqCounter = 0
		}

		url := apiGetFavorites
		if lastTweetID != "" {
			url = apiGetFavorites + "&max_id=" + lastTweetID
		}

		lastTweetID, err = downloadMedia(client, url, dlPath, filterAccount, unFav)
		if err != nil {
			log.Fatalln(err)
			break
		}

		if !dlWithoutAsk {
			fmt.Print("Type 'y' to continue: ")
			fmt.Scanln(&continueDL)
		}

		reqCounter++
	}

	fmt.Printf("All media is stored in: %v\n", dlPath)
}
<commit_msg>fix: process from specified tweet<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mrjones/oauth"
)

const (
	requestTokenURL   string = "https://api.twitter.com/oauth/request_token"
	authorizeTokenURL string = "https://api.twitter.com/oauth/authorize"
	accessTokenURL    string = "https://api.twitter.com/oauth/access_token"
	apiBase           string = "https://api.twitter.com/1.1/"
	apiGetFavorites   string = apiBase + "favorites/list.json?count=200"
	apiUnFavorite     string = apiBase + "favorites/destroy.json?id="
	apiRequestLimit   int    = 75
)

var (
	tweetID string
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	flag.StringVar(&tweetID, "tweetid", "", "Get media with an ID less than (that is, older than) or equal to the specified ID.")
	flag.StringVar(&tweetID, "i", "", "Get media with an ID less than (that is, older than) or equal to the specified ID.")
	flag.Parse()
}

func getHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}

		return home
	}
	return os.Getenv("HOME")
}

func getConfig() (string, map[string]string, error) {
	cfgFolder := os.Getenv("HOME")

	if cfgFolder == "" && runtime.GOOS == "windows" {
		cfgFolder = os.Getenv("APPDATA")

		if cfgFolder == "" {
			cfgFolder = filepath.Join(os.Getenv("USERPROFILE"), "Application Data", "twitter-favorite-pics")
		}

		cfgFolder = filepath.Join(cfgFolder, "twitter-favorite-pics")
	} else {
		cfgFolder = filepath.Join(cfgFolder, ".config", "twitter-favorite-pics")
	}

	if err := os.MkdirAll(cfgFolder, 0700); err != nil {
		return "", nil, err
	}

	cfgFile := filepath.Join(cfgFolder, "settings.json")

	cfg := map[string]string{}

	buf, err := ioutil.ReadFile(cfgFile)
	if err != nil && !os.IsNotExist(err) {
		return "", nil, err
	}

	if err != nil {
		var consumerKey, consumerSecret, dlPath, filterAccount, dlWithoutAsk, unFavAfterDL string
		fmt.Print("Enter consumer key: ")
		fmt.Scanln(&consumerKey)
		fmt.Print("Enter consumer secret: ")
		fmt.Scanln(&consumerSecret)
		fmt.Print("Enter download path: ")
		fmt.Scanln(&dlPath)
		fmt.Print("Enter twitter screen name that want to filter for download (separate by comma): ")
		fmt.Scanln(&filterAccount)
		fmt.Print("Continue download without asking? (y/N): ")
		fmt.Scanln(&dlWithoutAsk)
		dlWithoutAsk = strings.ToLower(dlWithoutAsk)
		fmt.Print("Un-favorite tweet after download? (y/N): ")
		fmt.Scanln(&unFavAfterDL)
		unFavAfterDL = strings.ToLower(unFavAfterDL)

		cfg["ConsumerKey"] = consumerKey
		cfg["ConsumerSecret"] = consumerSecret
		cfg["DownloadPath"] = dlPath
		cfg["FilterAccount"] = filterAccount

		if dlWithoutAsk == "y" || dlWithoutAsk == "yes" {
			cfg["DownloadWithoutAsking"] = "true"
		} else {
			cfg["DownloadWithoutAsking"] = "false"
		}

		if unFavAfterDL == "y" || unFavAfterDL == "yes" {
			cfg["UnFavAfterDownload"] = "true"
		} else {
			cfg["UnFavAfterDownload"] = "false"
		}
	} else {
		err = json.Unmarshal(buf, &cfg)
		if err != nil {
			return "", nil, fmt.Errorf("Could not unmarshal %v: %v", cfgFile, err)
		}
	}

	return cfgFile, cfg, nil
}

func writeConfig(cfg map[string]string, cfgFile string) error {
	buf, err := json.MarshalIndent(cfg, "", "  ")
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(cfgFile, buf, 0700)
	if err != nil {
		return err
	}

	return nil
}

func openBrowser(url string) error {
	browser := "xdg-open"
	args := []string{url}

	if runtime.GOOS == "windows" {
		browser = "rundll32.exe"
		args = []string{"url.dll,FileProtocolHandler", url}
	} else if runtime.GOOS == "darwin" {
		browser = "open"
	}

	browser, err := exec.LookPath(browser)
	if err != nil {
		return err
	}

	cmd := exec.Command(browser, args...)
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	if err != nil {
		return err
	}

	return nil
}

func getAuthorizeToken(c *oauth.Consumer, cfg map[string]string) (*oauth.AccessToken, error) {
	accessToken, foundToken := cfg["AccessToken"]
	accessSecret, foundSecret := cfg["AccessSecret"]

	var authorizeToken *oauth.AccessToken

	if foundToken && foundSecret {
		authorizeToken = &oauth.AccessToken{Token: accessToken, Secret: accessSecret}
	} else {
		reqToken, url, err := c.GetRequestTokenAndUrl("")
		if err != nil {
			return nil, err
		}

		fmt.Println("(1) Go to: " + url)
		fmt.Println("(2) Grant access, you should get back a verification code.")
		fmt.Print("(3) Enter that verification code here: ")

		err = openBrowser(url)
		if err != nil {
			return nil, err
		}

		verificationCode := ""
		fmt.Scanln(&verificationCode)

		authorizeToken, err = c.AuthorizeToken(reqToken, verificationCode)
		if err != nil {
			return nil, err
		}
	}

	return authorizeToken, nil
}

func unFavoriteTweet(wg *sync.WaitGroup, client *http.Client, entityID string) {
	defer wg.Done()

	url := apiUnFavorite + entityID

	reqBody := map[string]string{"id": entityID}
	jsonBody, _ := json.Marshal(reqBody)
	resp, err := client.Post(url, "application/json", bytes.NewBuffer(jsonBody))
	if err != nil {
		log.Println(err)
	}

	if resp.StatusCode != http.StatusOK {
		log.Printf("%v: %v\n", url, resp.Status)
	}
}

func downloadWorker(wg *sync.WaitGroup, url string, dlPath string, fileName string) {
	defer wg.Done()

	log.Printf("Get %v\n", fileName)

	resp, err := http.Get(url)
	if err != nil {
		log.Println(err)
	}

	defer resp.Body.Close()

	if err := os.MkdirAll(dlPath, 0755); err != nil {
		log.Println(err)
	}

	f, err := os.Create(filepath.Join(dlPath, fileName))
	if err != nil {
		log.Println(err)
	}

	_, err = io.Copy(f, resp.Body)
	if err != nil {
		log.Println(err)
	}

	f.Close()
}

func downloadMedia(client *http.Client, url string, dlPath string, filterAccount []string, unFav bool) (string, error) {
	resp, err := client.Get(url)
	if err != nil {
		return "", err
	}

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf(resp.Status)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var fav FavoriteList

	if err := json.Unmarshal([]byte(body), &fav); err != nil {
		return "", err
	}

	var lastTweetID string
	var wg sync.WaitGroup

	for _, v := range fav {
		i := sort.SearchStrings(filterAccount, v.User.ScreenName)

		if (i < len(filterAccount) && filterAccount[i] == v.User.ScreenName) || len(filterAccount) <= 0 {
			for _, val := range v.Entities.Media {
				largeMediaURL := val.MediaURL + ":large"
				sl := strings.Split(val.MediaURL, "/")
				fileName := sl[len(sl)-1]

				wg.Add(1)
				go downloadWorker(&wg, largeMediaURL, filepath.Join(dlPath, v.User.ScreenName), fileName)
			}

			if unFav && len(v.Entities.Media) > 0 {
				wg.Add(1)
				go unFavoriteTweet(&wg, client, v.IDStr)
			}
		}

		lastTweetID = v.IDStr
		log.Printf("Tweet ID: %v\n", lastTweetID)
	}

	wg.Wait()

	return lastTweetID, nil
}

func main() {
	cfgFile, cfg, err := getConfig()
	if err != nil {
		log.Fatalln("Get configuration file failed: ", err)
	}

	var filterAccount []string
	if len(cfg["FilterAccount"]) > 0 {
		fmt.Printf("Filter account: %v\n", cfg["FilterAccount"])

		filterAccount = strings.Split(cfg["FilterAccount"], ",")
		sort.Strings(filterAccount)
	}

	var dlWithoutAsk bool
	if strings.ToLower(cfg["DownloadWithoutAsking"]) == "true" {
		dlWithoutAsk = true
	} else {
		dlWithoutAsk = false
	}

	var unFav bool
	if strings.ToLower(cfg["UnFavAfterDownload"]) == "true" {
		unFav = true
	} else {
		unFav = false
	}

	c := oauth.NewConsumer(
		cfg["ConsumerKey"],
		cfg["ConsumerSecret"],
		oauth.ServiceProvider{
			RequestTokenUrl:   requestTokenURL,
			AuthorizeTokenUrl: authorizeTokenURL,
			AccessTokenUrl:    accessTokenURL,
		},
	)

	authorizeToken, err := getAuthorizeToken(c, cfg)
	if err != nil {
		log.Fatalln(err)
	}

	cfg["AccessToken"] = authorizeToken.Token
	cfg["AccessSecret"] = authorizeToken.Secret
	err = writeConfig(cfg, cfgFile)
	if err != nil {
		log.Fatalln(err)
	}

	client, err := c.MakeHttpClient(authorizeToken)
	if err != nil {
		log.Fatalln(err)
	}

	dlPath, foundDLPath := cfg["DownloadPath"]
	if !foundDLPath {
		currentPath, err := filepath.Abs(filepath.Dir(os.Args[0]))
		if err != nil {
			log.Fatalln(err)
		}

		dlPath = filepath.Join(currentPath, "downloads")
	}
	if dlPath[:2] == "~/" {
		dlPath = filepath.Join(getHomeDir(), dlPath[2:])
	}
	dlPath = filepath.Join(dlPath, "twitter-favorite-pics")

	var lastTweetID string
	if tweetID != "" {
		lastTweetID = tweetID
		fmt.Printf("Specify tweet ID: %v\n", tweetID)

	}

	continueDL := "y"
	reqCounter := 0
	for continueDL == "y" || dlWithoutAsk {
		if reqCounter >= apiRequestLimit {
			log.Printf("Last tweet ID: %v\n", lastTweetID)
			log.Println("API request quota exceeded, wait for 16 minute to unlock...")
			time.Sleep(16 * time.Minute)
			reqCounter = 0
		}

		url := apiGetFavorites
		if lastTweetID != "" {
			url = apiGetFavorites + "&max_id=" + lastTweetID
		}

		lastTweetID, err = downloadMedia(client, url, dlPath, filterAccount, unFav)
		if err != nil {
			log.Fatalln(err)
			break
		}

		if !dlWithoutAsk {
			fmt.Print("Type 'y' to continue: ")
			fmt.Scanln(&continueDL)
		}

		reqCounter++
	}

	log.Printf("Last tweet ID: %v\n", lastTweetID)
	log.Printf("All media is stored in: %v\n", dlPath)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

const aURL = "http://artii.herokuapp.com"

func main() {
	args := os.Args[1:]

	if len(args) == 0 {
		fmt.Printf("Usage:\n")
		return
	}

	switch args[0] {
	case "fonts":
		fmt.Printf("%v", fontList())
		return
	}

	fmt.Println(draw(args))
}

func fontList() string {
	url := aURL + "/fonts_list"

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	s := string(body) + "\n"
	return s
}

func draw(s []string) string {
	f := strings.Split(s[0], " ")
	js := strings.Join(f, "+")
	url := fmt.Sprintf("%s/make?text=%s", aURL, js)

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	a := string(body)
	return a
}
<commit_msg>split string before passing to draw function<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

const aURL = "http://artii.herokuapp.com"

func main() {
	args := os.Args[1:]

	if len(args) == 0 {
		fmt.Printf("Usage:\n")
		return
	}

	// first argument should be a string
	// so split on spaces to separate words
	s := strings.Split(args[0], " ")

	// if only one word and it is fonts
	// print the available fonts
	if len(s) == 1 && s[0] == "fonts" {
		fmt.Printf("%v", fontList())
		return
	}

	// pass in slice of words to draw function
	fmt.Println(draw(s))
}

func fontList() string {
	url := aURL + "/fonts_list"

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	s := string(body) + "\n"
	return s
}

func draw(s []string) string {
	js := strings.Join(s, "+")
	url := fmt.Sprintf("%s/make?text=%s", aURL, js)

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	a := string(body)
	return a
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/acmacalister/skittles"
	"io"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
)

var addr = flag.Int("addr", 6379, "http service address")

// Mutex Locked map/dictionary
type store struct {
	dict map[string]string
	lock *sync.RWMutex
}

// Our server's "client" for processing redis commands.
type client struct {
	conn   *net.Conn
	reader *bufio.Reader
	store  *store
}

func main() {
	fmt.Println(skittles.Cyan("Redis Server started..."))
	ln, err := net.Listen("tcp", fmt.Sprintf(":%d", *addr))
	if err != nil {
		log.Fatal(skittles.BoldRed(err))
	}

	s := store{dict: make(map[string]string), lock: &sync.RWMutex{}}

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println(skittles.BoldRed(err))
			continue
		}
		c := client{conn: &conn, store: &s}
		go c.process()
	}
}

func (store *store) Get(key string) string {
	store.lock.RLock()
	defer store.lock.RUnlock()

	return store.dict[key]
}

func (store *store) Set(key string, val string) {
	store.lock.Lock()
	defer store.lock.Unlock()

	store.dict[key] = val
}

func (c *client) process() {
	defer client.conn.Close()
	client.reader = bufio.NewReader(client.conn)
}
<commit_msg>add store item<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/acmacalister/skittles"
	"io"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
)

var addr = flag.Int("addr", 6379, "http service address")

const (
	CommandKey = iota
	CommandString
	CommandHash
	CommandList
	CommandSet
	CommandSortedSet
	CommandHyperLogLog
	CommandPubSub
	CommandTransaction
	CommandConnection
	CommandServer
)

// Struct for what type we are storing in our store.
type storeItem struct {
	val         string
	commandType int
}

// Mutex Locked map/dictionary
type store struct {
	dict map[string]storeItem
	lock *sync.RWMutex
}

// Our server's "client" for processing redis commands.
type client struct {
	conn   *net.Conn
	reader *bufio.Reader
	store  *store
}

func main() {
	fmt.Println(skittles.Cyan("Redis Server started..."))
	ln, err := net.Listen("tcp", fmt.Sprintf(":%d", *addr))
	if err != nil {
		log.Fatal(skittles.BoldRed(err))
	}

	s := store{dict: make(map[string]string), lock: &sync.RWMutex{}}

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println(skittles.BoldRed(err))
			continue
		}
		c := client{conn: &conn, store: &s}
		go c.process()
	}
}

func (store *store) Get(key string) storeItem {
	store.lock.RLock()
	defer store.lock.RUnlock()

	return store.dict[key]
}

func (store *store) Set(key string, val string) {
	store.lock.Lock()
	defer store.lock.Unlock()

	item := storeItem{val: val, commandType: CommandString}
	store.dict[key] = val
}

func (c *client) process() {
	defer client.conn.Close()
	client.reader = bufio.NewReader(client.conn)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/slyrz/newscat/html"
	"github.com/slyrz/newscat/model"
	"os"
)

func printChunks(chunks []*html.Chunk) {
	var last *html.Chunk = nil
	for _, chunk := range chunks {
		delim, pre, pos := "", "", ""
		// If the last chunk and the current chunk are part of the same HTML block,
		// we seperate them by a space character. If they are in different blocks,
		// we use two newline characters.
		switch {
		case last == nil:
			delim = ""
		case last.Block != chunk.Block:
			delim = "\n\n"
		case last.Block == chunk.Block:
			delim = " "
		}
		// Use bold font for headings, emphasized and bold text.
		switch chunk.Base.Data {
		case "h1", "h2", "h3", "h4", "h5", "h6", "em", "strong", "b":
			pre, pos = "\x1b[39;1m", "\x1b[0m"
		default:
			pre, pos = "", ""
		}
		fmt.Printf("%s%s%s%s", delim, pre, chunk.Text, pos)
		last = chunk
	}
	fmt.Println()
}

func main() {
	ext := model.NewExtractor()
	for _, arg := range os.Args[1:] {
		file, err := os.Open(arg)
		if err != nil {
			panic(err)
		}
		defer file.Close()

		doc, err := html.NewDocument(file)
		if err != nil {
			panic(err)
		}
		printChunks(ext.Extract(doc))
	}
}
<commit_msg>support reading input from URLs and stdin<commit_after>package main

import (
	"fmt"
	"github.com/slyrz/newscat/html"
	"github.com/slyrz/newscat/model"
	"io"
	"net/http"
	"os"
	"strings"
)

func printChunks(chunks []*html.Chunk) {
	var last *html.Chunk = nil
	for _, chunk := range chunks {
		delim, pre, pos := "", "", ""
		// If the last chunk and the current chunk are part of the same HTML block,
		// we seperate them by a space character. If they are in different blocks,
		// we use two newline characters.
		switch {
		case last == nil:
			delim = ""
		case last.Block != chunk.Block:
			delim = "\n\n"
		case last.Block == chunk.Block:
			delim = " "
		}
		// Use bold font for headings, emphasized and bold text.
		switch chunk.Base.Data {
		case "h1", "h2", "h3", "h4", "h5", "h6", "em", "strong", "b":
			pre, pos = "\x1b[39;1m", "\x1b[0m"
		default:
			pre, pos = "", ""
		}
		fmt.Printf("%s%s%s%s", delim, pre, chunk.Text, pos)
		last = chunk
	}
	fmt.Println()
}

func main() {
	inputData := make(chan io.Reader, 4)
	// Open all input (either paths / URLs passed as command line arguments or
	// read from stdin) and write it to inputData channel.
	go func() {
		if args := os.Args[1:]; len(args) > 0 {
			for _, arg := range args {
				if strings.HasPrefix(arg, "http://") {
					if resp, err := http.Get(arg); err == nil {
						inputData <- resp.Body
					}
				} else {
					if file, err := os.Open(arg); err == nil {
						inputData <- file
					}
				}
			}
		} else {
			inputData <- os.Stdin
		}
		close(inputData)
	}()

	ext := model.NewExtractor()
	// Read input from inputData channel and perform content extraction.
	for data := range inputData {
		// TODO: Warn if parsing Document failed.
		if doc, err := html.NewDocument(data); err == nil {
			// TODO: Print warning if no chunks were extracted.
			if chunks := ext.Extract(doc); len(chunks) > 0 {
				printChunks(chunks)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"runtime/pprof"
)

var (
	cpuprofile = flag.String("cpuprofile", "", "If not empty, collects CPU profile samples and writes the profile to the given file before the program exits")
	memprofile = flag.String("memprofile", "", "If not empty, writes memory heap allocations to the given file before the program exits")
)

var torrent string

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 0 {
		log.Println("Don't want arguments")
		usage()
	}

	if *cpuprofile != "" {
		cpuf, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(cpuf)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		defer func(file string) {
			memf, err := os.Create(file)
			if err != nil {
				log.Fatal(err)
			}
			pprof.WriteHeapProfile(memf)
		}(*memprofile)
	}

	// Bitshare dir
	u, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't watch dir: ", err)
	}
	pathArgs := []string{u.HomeDir, ".local", "share", "bitshare"}
	bitshareDir := filepath.Join(pathArgs...)

	// Watcher
	if fileDir == "." {
		fmt.Println("fileDir option is missing")
		flag.Usage()
	}
	_, err = os.Stat(fileDir)
	if err != nil {
		fmt.Printf("%s is an invalid dir: %s\n", fileDir, err)
		os.Exit(1)
	}
	watcher := NewWatcher(bitshareDir, filepath.Clean(fileDir))

	log.Println("Starting.")

	torrentSessions := make(map[string]*TorrentSession)

	// torrentDir and torrentFile
	user, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't get current user: ", err)
	}
	torrentDir := filepath.Join(user.HomeDir, ".local", "share", "bitshare")
	if _, err := os.Stat(torrentDir); os.IsNotExist(err) {
		err := os.MkdirAll(torrentDir, os.ModeDir|0755)
		if err != nil {
			log.Fatal("Couldn't make home dir:", err)
		}
	}
	//torrentFile := filepath.Join(torrentDir, "current.torrent")

	// External listener
	conChan, listenPort, err := listenForPeerConnections()
	if err != nil {
		log.Fatal("Couldn't listen for peers connection: ", err)
	}

	// quitChan
	quitChan := listenSigInt()

	// DB for synchronisation
	couchdb := NewDB()

	// LPD
	lpd := &Announcer{}
	if *useLPD {
		lpd = startLPD(torrentSessions, listenPort)
	}

mainLoop:
	for {
		select {
		case <-quitChan:
			for _, ts := range torrentSessions {
				err := ts.Quit()
				if err != nil {
					log.Println("Failed: ", err)
				} else {
					log.Println("Done")
				}
			}
			break mainLoop
		case c := <-conChan:
			log.Printf("New bt connection for ih %x", c.infohash)
			if ts, ok := torrentSessions[c.infohash]; ok {
				ts.AcceptNewPeer(c)
			}
		case announce := <-lpd.announces:
			hexhash, err := hex.DecodeString(announce.infohash)
			if err != nil {
				log.Println("Err with hex-decoding:", err)
				break
			}
			if ts, ok := torrentSessions[string(hexhash)]; ok {
				log.Printf("Received LPD announce for ih %s", announce.infohash)
				ts.hintNewPeer(announce.peer)
			}
		case magnet := <-couchdb.newTorrent:
			log.Println("new torrent:", magnet)

			ts, err := startSession(magnet, torrentSessions, listenPort)
			if err != nil {
				log.Fatal("Couldn't start new session: ", err)
			}

			go func() {
				for meta := range ts.NewTorrent {
					meta.saveToDisk(bitshareDir)

					currentFile := filepath.Join(bitshareDir, "current")
					ihhex := fmt.Sprintf("%x", meta.InfoHash)
					ioutil.WriteFile(currentFile, []byte(ihhex), 0600)
				}
			}()

			torrentSessions[ts.m.InfoHash] = ts
		case ih := <-watcher.PingNewTorrent:
			couchdb.PushNewTorrent(ih)

			torrentFile := filepath.Join(bitshareDir, fmt.Sprintf("%x", ih))
			startSession(torrentFile, torrentSessions, listenPort)
		}
	}

}

func startSession(torrent string, torrentSessions map[string]*TorrentSession, listenPort int) (ts *TorrentSession, err error) {

	meta, err := NewMetaInfo(torrent)
	if err != nil {
		return
	}
	if ts, ok := torrentSessions[meta.InfoHash]; ok {
		return ts, nil
	}

	for idx, ts := range torrentSessions {
		err := ts.Quit()
		if err != nil {
			log.Println("Failed: ", err)
		}

		delete(torrentSessions, idx)
	}

	ts, err = NewTorrentSession(torrent, listenPort)
	if err != nil {
		log.Println("Could not create torrent session.", err)
		return
	}
	log.Printf("Starting torrent session for %x", ts.m.InfoHash)
	go ts.DoTorrent()

	torrentSessions[ts.m.InfoHash] = ts

	return
}

func usage() {
	log.Printf("usage: Taipei-Torrent [options] (torrent-file | torrent-url)")

	flag.PrintDefaults()
	os.Exit(2)
}

func listenSigInt() chan os.Signal {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	return c
}

func startLPD(torrentSessions map[string]*TorrentSession, listenPort int) (lpd *Announcer) {
	lpd, err := NewAnnouncer(listenPort)
	if err != nil {
		log.Println("Couldn't listen for Local Peer Discoveries: ", err)
		return
	} else {
		for _, ts := range torrentSessions {
			lpd.Announce(ts.m.InfoHash)
		}
	}
	return
}
<commit_msg>Announce new torrents in LPD<commit_after>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"runtime/pprof"
)

var (
	cpuprofile = flag.String("cpuprofile", "", "If not empty, collects CPU profile samples and writes the profile to the given file before the program exits")
	memprofile = flag.String("memprofile", "", "If not empty, writes memory heap allocations to the given file before the program exits")
)

var torrent string

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 0 {
		log.Println("Don't want arguments")
		usage()
	}

	if *cpuprofile != "" {
		cpuf, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(cpuf)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		defer func(file string) {
			memf, err := os.Create(file)
			if err != nil {
				log.Fatal(err)
			}
			pprof.WriteHeapProfile(memf)
		}(*memprofile)
	}

	// Bitshare dir
	u, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't watch dir: ", err)
	}
	pathArgs := []string{u.HomeDir, ".local", "share", "bitshare"}
	bitshareDir := filepath.Join(pathArgs...)

	// Watcher
	if fileDir == "." {
		fmt.Println("fileDir option is missing")
		flag.Usage()
	}
	_, err = os.Stat(fileDir)
	if err != nil {
		fmt.Printf("%s is an invalid dir: %s\n", fileDir, err)
		os.Exit(1)
	}
	watcher := NewWatcher(bitshareDir, filepath.Clean(fileDir))

	log.Println("Starting.")

	torrentSessions := make(map[string]*TorrentSession)

	// torrentDir and torrentFile
	user, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't get current user: ", err)
	}
	torrentDir := filepath.Join(user.HomeDir, ".local", "share", "bitshare")
	if _, err := os.Stat(torrentDir); os.IsNotExist(err) {
		err := os.MkdirAll(torrentDir, os.ModeDir|0755)
		if err != nil {
			log.Fatal("Couldn't make home dir:", err)
		}
	}
	//torrentFile := filepath.Join(torrentDir, "current.torrent")

	// External listener
	conChan, listenPort, err := listenForPeerConnections()
	if err != nil {
		log.Fatal("Couldn't listen for peers connection: ", err)
	}

	// quitChan
	quitChan := listenSigInt()

	// DB for synchronisation
	couchdb := NewDB()

	// LPD
	lpd := &Announcer{}
	if *useLPD {
		lpd = startLPD(torrentSessions, listenPort)
	}

mainLoop:
	for {
		select {
		case <-quitChan:
			for _, ts := range torrentSessions {
				err := ts.Quit()
				if err != nil {
					log.Println("Failed: ", err)
				} else {
					log.Println("Done")
				}
			}
			break mainLoop
		case c := <-conChan:
			log.Printf("New bt connection for ih %x", c.infohash)
			if ts, ok := torrentSessions[c.infohash]; ok {
				ts.AcceptNewPeer(c)
			}
		case announce := <-lpd.announces:
			hexhash, err := hex.DecodeString(announce.infohash)
			if err != nil {
				log.Println("Err with hex-decoding:", err)
				break
			}
			if ts, ok := torrentSessions[string(hexhash)]; ok {
				log.Printf("Received LPD announce for ih %s", announce.infohash)
				ts.hintNewPeer(announce.peer)
			}
		case magnet := <-couchdb.newTorrent:
			log.Println("new torrent:", magnet)

			ts, err := startSession(magnet, torrentSessions, listenPort, lpd)
			if err != nil {
				log.Fatal("Couldn't start new session: ", err)
			}

			go func() {
				for meta := range ts.NewTorrent {
					meta.saveToDisk(bitshareDir)

					currentFile := filepath.Join(bitshareDir, "current")
					ihhex := fmt.Sprintf("%x", meta.InfoHash)
					ioutil.WriteFile(currentFile, []byte(ihhex), 0600)
				}
			}()

			torrentSessions[ts.m.InfoHash] = ts
		case ih := <-watcher.PingNewTorrent:
			couchdb.PushNewTorrent(ih)

			torrentFile := filepath.Join(bitshareDir, fmt.Sprintf("%x", ih))
			startSession(torrentFile, torrentSessions, listenPort, lpd)
		}
	}

}

func startSession(torrent string, torrentSessions map[string]*TorrentSession, listenPort int, lpd *Announcer) (ts *TorrentSession, err error) {

	meta, err := NewMetaInfo(torrent)
	if err != nil {
		return
	}
	if ts, ok := torrentSessions[meta.InfoHash]; ok {
		return ts, nil
	}

	for idx, ts := range torrentSessions {
		err := ts.Quit()
		if err != nil {
			log.Println("Failed: ", err)
		}

		delete(torrentSessions, idx)
	}

	ts, err = NewTorrentSession(torrent, listenPort)
	if err != nil {
		log.Println("Could not create torrent session.", err)
		return
	}
	log.Printf("Starting torrent session for %x", ts.m.InfoHash)
	go ts.DoTorrent()

	torrentSessions[ts.m.InfoHash] = ts

	lpd.Announce(ts.m.InfoHash)
	return
}

func usage() {
	log.Printf("usage: Taipei-Torrent [options] (torrent-file | torrent-url)")

	flag.PrintDefaults()
	os.Exit(2)
}

func listenSigInt() chan os.Signal {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	return c
}

func startLPD(torrentSessions map[string]*TorrentSession, listenPort int) (lpd *Announcer) {
	lpd, err := NewAnnouncer(listenPort)
	if err != nil {
		log.Println("Couldn't listen for Local Peer Discoveries: ", err)
		return
	} else {
		for _, ts := range torrentSessions {
			lpd.Announce(ts.m.InfoHash)
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"regexp"

	"github.com/PuerkitoBio/goquery"
)

var teaSites = []string{
	"https://verdanttea.com/",
}

var teaTypes = []string{
	`(?i)\btea\b`,
	`(?i)\boolong\b`,
	`(?i)\bblack\b`,
	`(?i)\bwhite\b`,
	`(?i)\bgrean\b`,
	`(?i)\bherbal\b`,
	`(?i)\byellow\b`,
	`(?i)\bfermented\b`,
}

func Match(pattern string, in string) bool {
	matches := regexp.MustCompile(pattern).MatchString(in)
	return matches
}

var tags = []string{
	//	"span",
	//	"div",
	//	"p",
	//	"text",
	//	"li",
	//	"button",
	"a",
	//	"select",
	//	"option",
	//	"h1",
	//	"h2",
	//	"h3",
	//	"h4",
	//	"article",
}

type Tea struct {
	name string
	link string
	data string
}

type MaybeTea struct {
	Tea
}

func (t *MaybeTea) ConfirmTeaType() bool {
	doc := t.GetDocument()
	allText := doc.Find("body").Text()

	return MatchString(t.name, allText)
}

func (t *MaybeTea) GetDocument() *goquery.Document {
	doc, err := goquery.NewDocument(t.link)
	if err != nil {
		log.Fatal(err)
	}

	return doc
}

func MatchString(toFind string, in string) bool {
	matched, err := regexp.MatchString(toFind, in)
	if err != nil {
		log.Fatal(err)
	}

	return matched
}

/*
	The crawler must be able to keep track of information about the user flow
	spawn multiple crawlers?
	how to find matching data between pages?
	if a link's labeled as a certain thing which is then found in a header on the page it leads to, that's a tea
	but it's not always that easy, sometimes the descriptor is not a link
	but the link is only near the descriptor
	how to determine that a link and a descriptor go together?
	either descriptor is child to the link or the link and the descriptor have a common parent
	the href and the descriptor will overlap in some way
	confirm that tea type is real by getting next page in flow
	confirm that it's in the page's data
*/
type Crawler struct {
	links       []string
	seen        map[string]bool
	possibleTea []*MaybeTea
	foundTea    []*Tea
	data        []string
}

func (t *Crawler) GetNextLink() string {
	totalLinks := len(t.links)

	if totalLinks > 0 {
		next := t.links[0:1][0]

		if totalLinks > 1 {
			t.links = t.links[1:]
		} else {
			t.links = make([]string, 0)
		}

		if !t.seen[next] {
			return next
		}
		return t.GetNextLink()
	}
	return ""
}

func (t *Crawler) ScrapeSites() *Crawler {
	nextLink := t.GetNextLink()
	fmt.Println("nextLink", nextLink)

	if nextLink != "" {
		doc, err := goquery.NewDocument(nextLink)
		if err != nil {
			fmt.Println("There was an error while getting the document for this link: %s", nextLink)
			return t.ScrapeSites()
		}

		t.ScrapePage(doc)
		return t.ScrapeSites()
	} else {
		fmt.Println("done")
		return t
	}
}

func (t *Crawler) ScrapePage(doc *goquery.Document) *Crawler {
	for _, teaType := range teaTypes {
		for _, tag := range tags {
			found := doc.Find(tag).FilterFunction(func(i int, node *goquery.Selection) bool {
				text := node.Text()
				return Match(teaType, text)
			})

			found.Each(func(i int, s *goquery.Selection) {
				if tag == "a" {
					href, exists := s.Attr("href")
					if exists {
						t.links = append(t.links, href)
					}
				}

				text := s.Text()
				t.data = append(t.data, text)
			})

		}
	}

	return t

}

/*
	visit site
	look for teaTypes
	if hyperlink, crawl if not seen
	if not hyperlink, save for language processing
*/

func ScrapeSite() {
	tg := Crawler{
		teaSites,
		make(map[string]bool),
		make([]*MaybeTea, 0),
		make([]*Tea, 0),
		make([]string, 0),
	}

	tg.ScrapeSites()
}

func main() {
	ScrapeSite()
}
<commit_msg>Crawler looks only for hardcoded types of tea<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"

	"github.com/PuerkitoBio/goquery"
)

var teaSites = []string{
	"https://verdanttea.com/",
}

var teaTypes = []string{
	`(?i)\bpu'er\b`,
	`(?i)\bpuer\b`,
	`(?i)\bpu 'er\b`,
	`(?i)\bpu er\b`,
	`(?i)\boolong\b`,
	`(?i)\bblack\b`,
	`(?i)\bwhite\b`,
	`(?i)\bgrean\b`,
	`(?i)\bherbal\b`,
	`(?i)\byellow\b`,
	`(?i)\bfermented\b`,
}

func Match(pattern string, in string) bool {
	matches := regexp.MustCompile(pattern).MatchString(in)
	return matches
}

var tags = []string{
	//	"span",
	//	"div",
	//	"p",
	//	"text",
	//	"li",
	//	"button",
	"a",
	//	"select",
	//	"option",
	//	"h1",
	//	"h2",
	//	"h3",
	//	"h4",
	//	"article",
}

type Tea struct {
	name string
	link string
	data string
}

type MaybeTea struct {
	Tea
}

func (t *MaybeTea) ConfirmTeaType() bool {
	doc := t.GetDocument()
	allText := doc.Find("body").Text()

	return MatchString(t.name, allText)
}

func (t *MaybeTea) GetDocument() *goquery.Document {
	doc, err := goquery.NewDocument(t.link)
	if err != nil {
		log.Fatal(err)
	}

	return doc
}

func MatchString(toFind string, in string) bool {
	matched, err := regexp.MatchString(toFind, in)
	if err != nil {
		log.Fatal(err)
	}

	return matched
}

/*
	The crawler must be able to keep track of information about the user flow
	spawn multiple crawlers?
	how to find matching data between pages?
	if a link's labeled as a certain thing which is then found in a header on the page it leads to, that's a tea
	but it's not always that easy, sometimes the descriptor is not a link
	but the link is only near the descriptor
	how to determine that a link and a descriptor go together?
	either descriptor is child to the link or the link and the descriptor have a common parent
	the href and the descriptor will overlap in some way
	confirm that tea type is real by getting next page in flow
	confirm that it's in the page's data
*/
type Crawler struct {
	links       []string
	seen        map[string]bool
	possibleTea []*MaybeTea
	foundTea    []*Tea
	data        []string
}

func (t *Crawler) GetNextLink() string {
	totalLinks := len(t.links)

	if totalLinks > 0 {
		next := t.links[0:1][0]

		if totalLinks > 1 {
			t.links = t.links[1:]
		} else {
			t.links = make([]string, 0)
		}

		if !t.seen[next] {
			t.seen[next] = true
			return next
		}
		return t.GetNextLink()
	}
	return ""
}

func (t *Crawler) ScrapeSites() *Crawler {
	nextLink := t.GetNextLink()
	fmt.Println("nextLink", nextLink)

	if nextLink != "" {
		doc, err := goquery.NewDocument(nextLink)
		if err != nil {
			fmt.Println("There was an error while getting the document for this link: %s", nextLink)
			return t.ScrapeSites()
		}

		t.ScrapePage(doc)
		return t.ScrapeSites()
	} else {
		fmt.Println("done")
		return t
	}
}

func (t *Crawler) ScrapePage(doc *goquery.Document) *Crawler {
	for _, teaType := range teaTypes {
		for _, tag := range tags {
			found := doc.Find(tag).FilterFunction(func(i int, node *goquery.Selection) bool {
				text := node.Text()
				return Match(teaType, text)
			})

			found.Each(func(i int, s *goquery.Selection) {
				if tag == "a" {
					href, exists := s.Attr("href")
					if exists {
						t.links = append(t.links, href)
					}
				}

				text := s.Text()
				t.data = append(t.data, text)
			})

		}
	}

	return t

}

/*
	visit site
	look for teaTypes
	if hyperlink, crawl if not seen
	if not hyperlink, save for language processing
*/

func ScrapeSite() {
	tg := Crawler{
		teaSites,
		make(map[string]bool),
		make([]*MaybeTea, 0),
		make([]*Tea, 0),
		make([]string, 0),
	}

	tg.ScrapeSites()
}

func main() {
	ScrapeSite()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	email        = flag.String("auth.email", "", "authorization email")
	key          = flag.String("auth.key", "", "authorization key")
	baseURL      = flag.String("url", "https://api.cloudflare.com/client/v4/logs", "URL for CloudFlare logs API")
	start        = flag.Int64("start", -1, "the unix epoch timestamp to start downloading from")
	maxPast      = flag.Duration("max", 72*time.Hour, "the maximum time in the past the start can be")
	end          = flag.Int64("end", time.Now().Unix(), "the unix epoch timestamp to end downloading at")
	fileInterval = flag.Duration("interval", 1*time.Minute, "the time interval to save files in")
	dir          = flag.String("dir", os.TempDir(), "directory to download logs in")
)

const (
	fileTimeLayout = "logs-2006_01_02-15_04_05.log.gz"
	checkpointFile = "checkpoint"
)

func main() {
	validateFlags()
	log.Printf("Downloading to %s", *dir)
	downloadLogs()
}

// downloadLogs connects to log downloading service and saves
// logs to local disk.
func downloadLogs() {
	var (
		// Global interval to download.
		startT = time.Unix(*start, 0).UTC()
		endT   = time.Unix(*end, 0).UTC()

		// Interval to download for current file. Rounds down to the closest minute
		// to ensure that you do not get files that cross intervals
		s = startT.Truncate(time.Minute)
		e time.Time
	)
	for {
		if !s.Before(endT) {
			return
		}
		e = s.Add(*fileInterval)
		if e.After(endT) {
			e = endT
		}
		saveLogs(s, e)
		// saves the checkpoint file after the file is successfully downloaded
		saveCheckpoint(e)
		s = s.Add(*fileInterval)
	}
}

// saveLogs downloads logs for the period [s, e) and saves
// saves them to a file.
func saveLogs(s, e time.Time) {
	log.Printf("Downloading logs from %v to %v", s, e)
	u := fmt.Sprintf("%s?start=%d&end=%d", *baseURL, s.Unix(), e.Unix())
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Fatalf("Failed to create request: %v", err)
	}

	req.Header.Add("X-Auth-Email", *email)
	req.Header.Add("X-Auth-Key", *key)
	req.Header.Add("Accept-Encoding", "gzip")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalf("Failed to make request: %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		log.Fatalf("Received non-2xx status code: %d", resp.StatusCode)
	}

	fname := filepath.Join(*dir, s.Format(fileTimeLayout))
	f, err := os.Create(fname)
	if err != nil {
		log.Fatalf("Failed to create log file (%s): %v", fname, err)
	}
	defer f.Close()

	io.Copy(f, resp.Body)
}

// saveCheckpoint saves the last downloaded state in a file
// to resume download from.
func saveCheckpoint(t time.Time) {
	fp := filepath.Join(*dir, checkpointFile)
	os.Remove(fp)
	f, err := os.Create(fp)
	if err != nil {
		log.Fatalf("Failed to create checkpoint file (%s): %v", fp, err)
	}
	defer f.Close()
	if _, err := f.WriteString(strconv.FormatInt(t.Unix(), 10)); err != nil {
		log.Fatalf("Failed to write to checkpoint file (%s): %v", fp, err)
	}
}

// validateFlags parses flags, initializes appropriate ones
// from checkpointFile, and performs some sanity checks.
func validateFlags() {
	flag.Parse()
	if len(*email) == 0 {
		log.Fatal("No auth.email provided")
	}
	if len(*key) == 0 {
		log.Fatal("No auth.key provided")
	}
	if len(*baseURL) == 0 {
		log.Fatal("No url provided")
	}
	if *start < 0 {
		fp := filepath.Join(*dir, checkpointFile)
		b, err := ioutil.ReadFile(fp)
		if err != nil {
			log.Fatalf("Failed to read checkpoint file (%s): %v", fp, err)
		}
		s, err := strconv.ParseInt(string(b), 10, 0)
		if err != nil || s < 0 {
			log.Fatalf("Corrupt checkpoint file (%s)", fp)
		}
		*start = s
	}
	if time.Since(time.Unix(*start, 0)) > *maxPast {
		log.Fatalf("Start is more than %v old", maxPast)
	}
	if *end < 0 {
		log.Fatalf("The provided end (%d) is < 0", *end)
	}
	if !(*end > *start) {
		log.Fatalf("The provided end (%d) is not after start (%d)", *end, *start)
	}
	if _, err := os.Stat(*dir); os.IsNotExist(err) {
		log.Fatalf("The provided dir (%s) does not exist", *dir)
	}
}
<commit_msg>added boundary for fileInterval and updated baseURL<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	email    = flag.String("auth.email", "", "authorization email")
	key      = flag.String("auth.key", "", "authorization key")
	baseURL  = flag.String("url", "", "URL for CloudFlare logs API - https://api.cloudflare.com/client/v4/zones/<zone tag>/logs/requests")
	start    = flag.Int64("start", -1, "the unix epoch timestamp to start downloading from")
	maxPast  = flag.Duration("max", 72*time.Hour, "the maximum time in the past the start can be")
	end      = flag.Int64("end", time.Now().Unix(), "the unix epoch timestamp to end downloading at")
	interval = flag.Duration("interval", 1*time.Minute, "the time interval to save files in")
	dir      = flag.String("dir", os.TempDir(), "directory to download logs in")
)

const (
	fileTimeLayout = "logs-2006_01_02-15_04_05.log.gz"
	checkpointFile = "checkpoint"
)

func main() {
	validateFlags()
	log.Printf("Downloading to %s", *dir)
	downloadLogs()
}

// downloadLogs connects to log downloading service and saves
// logs to local disk.
func downloadLogs() {
	var (
		// Global interval to download.
		startT = time.Unix(*start, 0).UTC()
		endT   = time.Unix(*end, 0).UTC()

		// Interval to download for current file. Rounds down to the closest minute
		// to ensure that you do not get files that cross intervals
		s = startT.Truncate(time.Minute)
		e time.Time
	)
	for {
		if !s.Before(endT) {
			return
		}
		e = s.Add(*interval)
		if e.After(endT) {
			e = endT
		}
		saveLogs(s, e)
		// saves the checkpoint file after the file is successfully downloaded
		saveCheckpoint(e)
		s = s.Add(*interval)
	}
}

// saveLogs downloads logs for the period [s, e) and saves
// saves them to a file.
func saveLogs(s, e time.Time) {
	log.Printf("Downloading logs from %v to %v", s, e)
	u := fmt.Sprintf("%s?start=%d&end=%d", *baseURL, s.Unix(), e.Unix())
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Fatalf("Failed to create request: %v", err)
	}

	req.Header.Add("X-Auth-Email", *email)
	req.Header.Add("X-Auth-Key", *key)
	req.Header.Add("Accept-Encoding", "gzip")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalf("Failed to make request: %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		log.Fatalf("Received non-2xx status code: %d", resp.StatusCode)
	}

	fname := filepath.Join(*dir, s.Format(fileTimeLayout))
	f, err := os.Create(fname)
	if err != nil {
		log.Fatalf("Failed to create log file (%s): %v", fname, err)
	}
	defer f.Close()

	io.Copy(f, resp.Body)
}

// saveCheckpoint saves the last downloaded state in a file
// to resume download from.
func saveCheckpoint(t time.Time) {
	fp := filepath.Join(*dir, checkpointFile)
	os.Remove(fp)
	f, err := os.Create(fp)
	if err != nil {
		log.Fatalf("Failed to create checkpoint file (%s): %v", fp, err)
	}
	defer f.Close()
	if _, err := f.WriteString(strconv.FormatInt(t.Unix(), 10)); err != nil {
		log.Fatalf("Failed to write to checkpoint file (%s): %v", fp, err)
	}
}

// validateFlags parses flags, initializes appropriate ones
// from checkpointFile, and performs some sanity checks.
func validateFlags() {
	flag.Parse()
	if len(*email) == 0 {
		log.Fatal("No auth.email provided")
	}
	if len(*key) == 0 {
		log.Fatal("No auth.key provided")
	}
	if len(*baseURL) == 0 {
		log.Fatal("No url provided")
	}
	if *start < 0 {
		fp := filepath.Join(*dir, checkpointFile)
		b, err := ioutil.ReadFile(fp)
		if err != nil {
			log.Fatalf("Failed to read checkpoint file (%s): %v", fp, err)
		}
		s, err := strconv.ParseInt(string(b), 10, 0)
		if err != nil || s < 0 {
			log.Fatalf("Corrupt checkpoint file (%s)", fp)
		}
		*start = s
	}
	if time.Since(time.Unix(*start, 0)) > *maxPast {
		log.Fatalf("Start is more than %v old", maxPast)
	}
	if *end < 0 {
		log.Fatalf("The provided end (%d) is < 0", *end)
	}
	if !(*end > *start) {
		log.Fatalf("The provided end (%d) is not after start (%d)", *end, *start)
	}
	if _, err := os.Stat(*dir); os.IsNotExist(err) {
		log.Fatalf("The provided dir (%s) does not exist", *dir)
	}
	if *interval < time.Duration(1*time.Second) {
		log.Fatalf("The interval of time is less than one second")
	}
	if *interval > time.Duration(24*time.Hour) {
		log.Fatalf("The interval of time is greater than twenty-four hours")
	}
}
<|endoftext|>