<commit_before>// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm)
// go-aah/essentials source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package ess

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Excludes is handly filepath match manipulation
type Excludes []string

// Validate helps to evalute the pattern are valid
// `Match` method is from error and focus on match
func (e *Excludes) Validate() error {
	for _, pattern := range *e {
		if _, err := filepath.Match(pattern, "abc/b/c"); err != nil {
			return fmt.Errorf("unable to evalute pattern: %v", pattern)
		}
	}
	return nil
}

// Match evalutes given file with available patterns returns true if matches
// otherwise false. `Match` internally uses the `filepath.Match`
//
// Note: `Match` ignore pattern errors, use `Validate` method to ensure
// you have correct exclude patterns
func (e *Excludes) Match(file string) bool {
	for _, pattern := range *e {
		if match, _ := filepath.Match(pattern, file); match {
			return match
		}
	}
	return false
}

// IsFileExists return true is file or directory is exists, otherwise returns
// false. It also take cares of symlink path as well
func IsFileExists(filename string) bool {
	_, err := os.Lstat(filename)
	return err == nil
}

// IsDirEmpty returns true if the given directory is empty also returns true if
// directory not exists. Otherwise returns false
func IsDirEmpty(path string) bool {
	if !IsFileExists(path) {
		// directory not exists
		return true
	}
	dir, _ := os.Open(path)
	defer CloseQuietly(dir)
	results, _ := dir.Readdir(1)
	return len(results) == 0
}

// IsDir returns true if the given `path` is directory otherwise returns false.
// Also returns false if path is not exists
func IsDir(path string) bool {
	info, err := os.Lstat(path)
	return err == nil && info.IsDir()
}

// ApplyFileMode applies the given file mode to the target{file|directory}
func ApplyFileMode(target string, mode os.FileMode) error {
	err := os.Chmod(target, mode)
	if err != nil {
		return fmt.Errorf("unable to apply mode: %v, to given file or directory: %v", mode, target)
	}
	return nil
}

// LineCnt counts no. of lines on file
func LineCnt(fileName string) int {
	f, err := os.Open(fileName)
	if err != nil {
		return 0
	}
	defer CloseQuietly(f)

	return LineCntr(f)
}

// LineCntr counts no. of lines for given reader
func LineCntr(r io.Reader) int {
	buf := make([]byte, 8196)
	count := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return count
		}

		count += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return count
}

// Walk method extends filepath.Walk to also follows symlinks.
// Always returns the path of the file or directory also path
// is inline to name of symlink
func Walk(srcDir string, walkFn filepath.WalkFunc) error {
	return doWalk(srcDir, srcDir, walkFn)
}

func doWalk(fname string, linkName string, walkFn filepath.WalkFunc) error {
	fsWalkFn := func(path string, info os.FileInfo, err error) error {
		var name string
		name, err = filepath.Rel(fname, path)
		if err != nil {
			return err
		}

		path = filepath.Join(linkName, name)

		if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink {
			var symlinkPath string
			symlinkPath, err = filepath.EvalSymlinks(path)
			if err != nil {
				return err
			}

			// https://github.com/golang/go/blob/master/src/path/filepath/path.go#L392
			info, err = os.Lstat(symlinkPath)
			if err != nil {
				return walkFn(path, info, err)
			}

			if info.IsDir() {
				return doWalk(symlinkPath, path, walkFn)
			}
		}

		return walkFn(path, info, err)
	}

	return filepath.Walk(fname, fsWalkFn)
}

// CopyFile copies the given source file into destination
func CopyFile(dest, src string) (int64, error) {
	if !IsFileExists(src) {
		return 0, fmt.Errorf("source file does not exists: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return 0, fmt.Errorf("destination file already exists: %v", dest)
	}

	destFile, err := os.Create(dest)
	if err != nil {
		return 0, fmt.Errorf("unable to create dest file: %v", dest)
	}
	defer CloseQuietly(destFile)

	srcFile, err := os.Open(src)
	if err != nil {
		return 0, fmt.Errorf("unable to open source file: %v", src)
	}
	defer CloseQuietly(srcFile)

	copiedBytes, err := io.Copy(destFile, srcFile)
	if err != nil {
		return 0, fmt.Errorf("unable to copy file from %v to %v (%v)",
			src, dest, err)
	}

	return copiedBytes, nil
}

// CopyDir copies entire directory, sub directories and files into destination
// and it excludes give file matches
func CopyDir(dest, src string, excludes Excludes) error {
	if !IsFileExists(src) {
		return fmt.Errorf("source dir does not exists: %v", src)
	}

	src = filepath.Clean(src)
	srcInfo, _ := os.Lstat(src)
	if !srcInfo.IsDir() {
		return fmt.Errorf("source is not directory: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return fmt.Errorf("destination dir already exists: %v", dest)
	}

	if err := excludes.Validate(); err != nil {
		return err
	}

	return Walk(src, func(srcPath string, info os.FileInfo, err error) error {
		if excludes.Match(filepath.Base(srcPath)) {
			if info.IsDir() {
				// excluding directory
				return filepath.SkipDir
			}
			// excluding file
			return nil
		}

		relativeSrcPath := strings.TrimLeft(srcPath[len(src):], string(filepath.Separator))
		destPath := filepath.Join(dest, relativeSrcPath)

		if info.IsDir() {
			// directory permissions is not preserved from source
			return MkDirAll(destPath, 0755)
		}

		// copy source into destination
		if _, err = CopyFile(destPath, srcPath); err != nil {
			return err
		}

		// Apply source permision into target as well
		// so file permissions are preserved
		return ApplyFileMode(destPath, info.Mode())
	})
}

// DeleteFiles method deletes give files or directories. ensure your supplying
// appropriate paths.
func DeleteFiles(files ...string) (errs []error) {
	for _, f := range files {
		if !IsFileExists(f) {
			errs = append(errs, fmt.Errorf("path does not exists: %s", f))
			continue
		}

		var err error
		if IsDir(f) {
			err = os.RemoveAll(f)
		} else {
			err = os.Remove(f)
		}

		if err != nil {
			errs = append(errs, err)
		}
	}

	return
}

// DirsPath method returns all directories absolute path from given base path recursively.
func DirsPath(basePath string, recursive bool) (pdirs []string, err error) {
	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if info.IsDir() {
				pdirs = append(pdirs, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if v.IsDir() {
			pdirs = append(pdirs, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// FilesPath method returns all files absolute path from given base path recursively.
func FilesPath(basePath string, recursive bool) (files []string, err error) {
	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				files = append(files, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if !v.IsDir() {
			files = append(files, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// StripExt method returns name of the file without extension.
//    E.g.: index.html => index
func StripExt(name string) string {
	if IsStrEmpty(name) {
		return name
	}

	idx := strings.IndexByte(name, '.')
	if idx > 0 {
		return name[:idx]
	}

	return name
}
<commit_msg>use the last index instead of first<commit_after>// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm)
// go-aah/essentials source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package ess

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Excludes is handly filepath match manipulation
type Excludes []string

// Validate helps to evalute the pattern are valid
// `Match` method is from error and focus on match
func (e *Excludes) Validate() error {
	for _, pattern := range *e {
		if _, err := filepath.Match(pattern, "abc/b/c"); err != nil {
			return fmt.Errorf("unable to evalute pattern: %v", pattern)
		}
	}
	return nil
}

// Match evalutes given file with available patterns returns true if matches
// otherwise false. `Match` internally uses the `filepath.Match`
//
// Note: `Match` ignore pattern errors, use `Validate` method to ensure
// you have correct exclude patterns
func (e *Excludes) Match(file string) bool {
	for _, pattern := range *e {
		if match, _ := filepath.Match(pattern, file); match {
			return match
		}
	}
	return false
}

// IsFileExists return true is file or directory is exists, otherwise returns
// false. It also take cares of symlink path as well
func IsFileExists(filename string) bool {
	_, err := os.Lstat(filename)
	return err == nil
}

// IsDirEmpty returns true if the given directory is empty also returns true if
// directory not exists. Otherwise returns false
func IsDirEmpty(path string) bool {
	if !IsFileExists(path) {
		// directory not exists
		return true
	}
	dir, _ := os.Open(path)
	defer CloseQuietly(dir)
	results, _ := dir.Readdir(1)
	return len(results) == 0
}

// IsDir returns true if the given `path` is directory otherwise returns false.
// Also returns false if path is not exists
func IsDir(path string) bool {
	info, err := os.Lstat(path)
	return err == nil && info.IsDir()
}

// ApplyFileMode applies the given file mode to the target{file|directory}
func ApplyFileMode(target string, mode os.FileMode) error {
	err := os.Chmod(target, mode)
	if err != nil {
		return fmt.Errorf("unable to apply mode: %v, to given file or directory: %v", mode, target)
	}
	return nil
}

// LineCnt counts no. of lines on file
func LineCnt(fileName string) int {
	f, err := os.Open(fileName)
	if err != nil {
		return 0
	}
	defer CloseQuietly(f)

	return LineCntr(f)
}

// LineCntr counts no. of lines for given reader
func LineCntr(r io.Reader) int {
	buf := make([]byte, 8196)
	count := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return count
		}

		count += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return count
}

// Walk method extends filepath.Walk to also follows symlinks.
// Always returns the path of the file or directory also path
// is inline to name of symlink
func Walk(srcDir string, walkFn filepath.WalkFunc) error {
	return doWalk(srcDir, srcDir, walkFn)
}

func doWalk(fname string, linkName string, walkFn filepath.WalkFunc) error {
	fsWalkFn := func(path string, info os.FileInfo, err error) error {
		var name string
		name, err = filepath.Rel(fname, path)
		if err != nil {
			return err
		}

		path = filepath.Join(linkName, name)

		if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink {
			var symlinkPath string
			symlinkPath, err = filepath.EvalSymlinks(path)
			if err != nil {
				return err
			}

			// https://github.com/golang/go/blob/master/src/path/filepath/path.go#L392
			info, err = os.Lstat(symlinkPath)
			if err != nil {
				return walkFn(path, info, err)
			}

			if info.IsDir() {
				return doWalk(symlinkPath, path, walkFn)
			}
		}

		return walkFn(path, info, err)
	}

	return filepath.Walk(fname, fsWalkFn)
}

// CopyFile copies the given source file into destination
func CopyFile(dest, src string) (int64, error) {
	if !IsFileExists(src) {
		return 0, fmt.Errorf("source file does not exists: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return 0, fmt.Errorf("destination file already exists: %v", dest)
	}

	destFile, err := os.Create(dest)
	if err != nil {
		return 0, fmt.Errorf("unable to create dest file: %v", dest)
	}
	defer CloseQuietly(destFile)

	srcFile, err := os.Open(src)
	if err != nil {
		return 0, fmt.Errorf("unable to open source file: %v", src)
	}
	defer CloseQuietly(srcFile)

	copiedBytes, err := io.Copy(destFile, srcFile)
	if err != nil {
		return 0, fmt.Errorf("unable to copy file from %v to %v (%v)",
			src, dest, err)
	}

	return copiedBytes, nil
}

// CopyDir copies entire directory, sub directories and files into destination
// and it excludes give file matches
func CopyDir(dest, src string, excludes Excludes) error {
	if !IsFileExists(src) {
		return fmt.Errorf("source dir does not exists: %v", src)
	}

	src = filepath.Clean(src)
	srcInfo, _ := os.Lstat(src)
	if !srcInfo.IsDir() {
		return fmt.Errorf("source is not directory: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return fmt.Errorf("destination dir already exists: %v", dest)
	}

	if err := excludes.Validate(); err != nil {
		return err
	}

	return Walk(src, func(srcPath string, info os.FileInfo, err error) error {
		if excludes.Match(filepath.Base(srcPath)) {
			if info.IsDir() {
				// excluding directory
				return filepath.SkipDir
			}
			// excluding file
			return nil
		}

		relativeSrcPath := strings.TrimLeft(srcPath[len(src):], string(filepath.Separator))
		destPath := filepath.Join(dest, relativeSrcPath)

		if info.IsDir() {
			// directory permissions is not preserved from source
			return MkDirAll(destPath, 0755)
		}

		// copy source into destination
		if _, err = CopyFile(destPath, srcPath); err != nil {
			return err
		}

		// Apply source permision into target as well
		// so file permissions are preserved
		return ApplyFileMode(destPath, info.Mode())
	})
}

// DeleteFiles method deletes give files or directories. ensure your supplying
// appropriate paths.
func DeleteFiles(files ...string) (errs []error) {
	for _, f := range files {
		if !IsFileExists(f) {
			errs = append(errs, fmt.Errorf("path does not exists: %s", f))
			continue
		}

		var err error
		if IsDir(f) {
			err = os.RemoveAll(f)
		} else {
			err = os.Remove(f)
		}

		if err != nil {
			errs = append(errs, err)
		}
	}

	return
}

// DirsPath method returns all directories absolute path from given base path recursively.
func DirsPath(basePath string, recursive bool) (pdirs []string, err error) {
	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if info.IsDir() {
				pdirs = append(pdirs, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if v.IsDir() {
			pdirs = append(pdirs, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// FilesPath method returns all files absolute path from given base path recursively.
func FilesPath(basePath string, recursive bool) (files []string, err error) {
	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				files = append(files, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if !v.IsDir() {
			files = append(files, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// StripExt method returns name of the file without extension.
//    E.g.: index.html => index
func StripExt(name string) string {
	if IsStrEmpty(name) {
		return name
	}

	idx := strings.LastIndexByte(name, '.')
	if idx > 0 {
		return name[:idx]
	}

	return name
}
<|endoftext|>


<commit_before>// Copyright © 2017 The vt-go authors. All Rights Reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vt

import (
	"bytes"
	"encoding/json"
	"io"
	"mime/multipart"
	"net/url"
	"os"
)

type progressReader struct {
	reader     io.Reader
	total      int64
	read       int64
	progressCh chan<- float32
}

func (pr *progressReader) Read(p []byte) (int, error) {
	n, err := pr.reader.Read(p)
	pr.read += int64(n)
	if pr.progressCh != nil {
		pr.progressCh <- float32(pr.read) / float32(pr.total) * 100
	}
	return n, err
}

// FileScanner represents a file scanner.
type FileScanner struct {
	cli *Client
}

// Scan sends a file to VirusTotal for scanning. The file content is read from
// the r io.Reader and sent to VirusTotal with the provided file name which can
// be left blank. The function also sends a float32 through the progress channel
// indicating the percentage of the file that has been already uploaded. The
// progress channel can be nil if the caller is not interested in receiving
// upload progress updates. An analysis object is returned as soon as the file
// is uploaded.
func (s *FileScanner) Scan(r io.Reader, filename string, progress chan<- float32) (*Object, error) {

	var uploadURL *url.URL
	var payloadSize int64

	b := bytes.Buffer{}

	// Create multipart writer for the file
	w := multipart.NewWriter(&b)
	f, err := w.CreateFormFile("file", filename)
	if err != nil {
		return nil, err
	}

	// Copy data from input stream to the multiparted file
	if payloadSize, err = io.Copy(f, r); err != nil {
		return nil, err
	}

	w.Close()

	if payloadSize > maxFileSize {
		return nil, fmt.Errorf("file size can't be larger than %d bytes", maxFileSize)
	} else if payloadSize > maxPayloadSize {
		// Payload is bigger than supported by AppEngine in a POST request,
		// let's ask for an upload URL.
		var u string
		if _, err := s.cli.GetData(URL("files/upload_url"), &u); err != nil {
			return nil, err
		}
		if uploadURL, err = url.Parse(u); err != nil {
			return nil, err
		}
	} else {
		uploadURL = URL("files")
	}

	pr := &progressReader{
		reader:     &b,
		total:      int64(b.Len()),
		progressCh: progress}

	headers := map[string]string{"Content-Type": w.FormDataContentType()}

	httpResp, err := s.cli.sendRequest("POST", uploadURL, pr, headers)
	if err != nil {
		return nil, err
	}
	defer httpResp.Body.Close()

	apiResp, err := s.cli.parseResponse(httpResp)
	if err != nil {
		return nil, err
	}

	analysis := &Object{}
	if err := json.Unmarshal(apiResp.Data, analysis); err != nil {
		return nil, err
	}

	return analysis, nil
}

// ScanFile sends a file to VirusTotal for scanning. This function is similar to
// Scan but it receive an *os.File instead of a io.Reader and a file name.
func (s *FileScanner) ScanFile(f *os.File, progress chan<- float32) (*Object, error) {
	return s.Scan(f, f.Name(), progress)
}
<commit_msg>Add missing include.<commit_after>// Copyright © 2017 The vt-go authors. All Rights Reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vt

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/url"
	"os"
)

type progressReader struct {
	reader     io.Reader
	total      int64
	read       int64
	progressCh chan<- float32
}

func (pr *progressReader) Read(p []byte) (int, error) {
	n, err := pr.reader.Read(p)
	pr.read += int64(n)
	if pr.progressCh != nil {
		pr.progressCh <- float32(pr.read) / float32(pr.total) * 100
	}
	return n, err
}

// FileScanner represents a file scanner.
type FileScanner struct {
	cli *Client
}

// Scan sends a file to VirusTotal for scanning. The file content is read from
// the r io.Reader and sent to VirusTotal with the provided file name which can
// be left blank. The function also sends a float32 through the progress channel
// indicating the percentage of the file that has been already uploaded. The
// progress channel can be nil if the caller is not interested in receiving
// upload progress updates. An analysis object is returned as soon as the file
// is uploaded.
func (s *FileScanner) Scan(r io.Reader, filename string, progress chan<- float32) (*Object, error) {

	var uploadURL *url.URL
	var payloadSize int64

	b := bytes.Buffer{}

	// Create multipart writer for the file
	w := multipart.NewWriter(&b)
	f, err := w.CreateFormFile("file", filename)
	if err != nil {
		return nil, err
	}

	// Copy data from input stream to the multiparted file
	if payloadSize, err = io.Copy(f, r); err != nil {
		return nil, err
	}

	w.Close()

	if payloadSize > maxFileSize {
		return nil, fmt.Errorf("file size can't be larger than %d bytes", maxFileSize)
	} else if payloadSize > maxPayloadSize {
		// Payload is bigger than supported by AppEngine in a POST request,
		// let's ask for an upload URL.
		var u string
		if _, err := s.cli.GetData(URL("files/upload_url"), &u); err != nil {
			return nil, err
		}
		if uploadURL, err = url.Parse(u); err != nil {
			return nil, err
		}
	} else {
		uploadURL = URL("files")
	}

	pr := &progressReader{
		reader:     &b,
		total:      int64(b.Len()),
		progressCh: progress}

	headers := map[string]string{"Content-Type": w.FormDataContentType()}

	httpResp, err := s.cli.sendRequest("POST", uploadURL, pr, headers)
	if err != nil {
		return nil, err
	}
	defer httpResp.Body.Close()

	apiResp, err := s.cli.parseResponse(httpResp)
	if err != nil {
		return nil, err
	}

	analysis := &Object{}
	if err := json.Unmarshal(apiResp.Data, analysis); err != nil {
		return nil, err
	}

	return analysis, nil
}

// ScanFile sends a file to VirusTotal for scanning. This function is similar to
// Scan but it receive an *os.File instead of a io.Reader and a file name.
func (s *FileScanner) ScanFile(f *os.File, progress chan<- float32) (*Object, error) {
	return s.Scan(f, f.Name(), progress)
}
<|endoftext|>


<commit_before>package renter

import (
	"sync"
	"testing"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter"
	"github.com/NebulousLabs/Sia/node"
	"github.com/NebulousLabs/Sia/siatest"
	"github.com/NebulousLabs/fastrand"
)

// TestRenter executes a number of subtests using the same TestGroup to
// save time on initialization
func TestRenter(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	// Create a group for the subtests
	groupParams := siatest.GroupParams{
		Hosts:   5,
		Renters: 1,
		Miners:  1,
	}
	tg, err := siatest.NewGroupFromTemplate(groupParams)
	if err != nil {
		t.Fatal("Failed to create group: ", err)
	}
	defer func() {
		if err := tg.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	// Specify subtests to run
	subTests := []struct {
		name string
		test func(*testing.T, *siatest.TestGroup)
	}{
		{"UploadDownload", testUploadDownload},
		{"DownloadMultipleLargeSectors", testDownloadMultipleLargeSectors},
		{"TestRenterLocalRepair", testRenterLocalRepair},
		{"TestRenterRemoteRepair", testRenterRemoteRepair},
	}
	// Run subtests
	for _, subtest := range subTests {
		t.Run(subtest.name, func(t *testing.T) {
			subtest.test(t, tg)
		})
	}
}

// testUploadDownload is a subtest that uses an existing TestGroup to test if
// uploading and downloading a file works
func testUploadDownload(t *testing.T, tg *siatest.TestGroup) {
	// Grab the first of the group's renters
	renter := tg.Renters()[0]
	// Upload file, creating a piece for each host in the group
	dataPieces := uint64(1)
	parityPieces := uint64(len(tg.Hosts())) - dataPieces
	fileSize := 100 + siatest.Fuzz()
	localFile, remoteFile, err := renter.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
	if err != nil {
		t.Fatal("Failed to upload a file for testing: ", err)
	}
	// Download the file synchronously directly into memory
	_, err = renter.DownloadByStream(remoteFile)
	if err != nil {
		t.Fatal(err)
	}
	// Download the file synchronously to a file on disk
	_, err = renter.DownloadToDisk(remoteFile, false)
	if err != nil {
		t.Fatal(err)
	}
	// Download the file asynchronously and wait for the download to finish.
	localFile, err = renter.DownloadToDisk(remoteFile, true)
	if err != nil {
		t.Error(err)
	}
	if err := renter.WaitForDownload(localFile, remoteFile); err != nil {
		t.Error(err)
	}
	// Stream the file.
	_, err = renter.Stream(remoteFile)
	if err != nil {
		t.Fatal(err)
	}
	// Stream the file partially a few times. At least 1 byte is streamed.
	for i := 0; i < 5; i++ {
		from := fastrand.Intn(fileSize - 1)             // [0..fileSize-2]
		to := from + 1 + fastrand.Intn(fileSize-from-1) // [from+1..fileSize-1]
		_, err = renter.StreamPartial(remoteFile, localFile, uint64(from), uint64(to))
		if err != nil {
			t.Fatal(err)
		}
	}
}

// testDownloadMultipleLargeSectors downloads multiple large files (>5 Sectors)
// in parallel and makes sure that the downloads are blocking each other.
func testDownloadMultipleLargeSectors(t *testing.T, tg *siatest.TestGroup) {
	// parallelDownloads is the number of downloads that are run in parallel.
	parallelDownloads := 10
	// fileSize is the size of the downloaded file.
	fileSize := int(10*modules.SectorSize) + siatest.Fuzz()
	// set download limits and reset them after test.
	// uniqueRemoteFiles is the number of files that will be uploaded to the
	// network. Downloads will choose the remote file to download randomly.
	uniqueRemoteFiles := 5
	// Grab the first of the group's renters
	renter := tg.Renters()[0]
	// set download limits and reset them after test.
	if err := renter.RenterPostRateLimit(int64(fileSize)*2, 0); err != nil {
		t.Fatal("failed to set renter bandwidth limit", err)
	}
	defer func() {
		if err := renter.RenterPostRateLimit(0, 0); err != nil {
			t.Error("failed to reset renter bandwidth limit", err)
		}
	}()

	// Upload files
	dataPieces := uint64(len(tg.Hosts())) - 1
	parityPieces := uint64(1)
	remoteFiles := make([]*siatest.RemoteFile, 0, uniqueRemoteFiles)
	for i := 0; i < uniqueRemoteFiles; i++ {
		_, remoteFile, err := renter.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
		if err != nil {
			t.Fatal("Failed to upload a file for testing: ", err)
		}
		remoteFiles = append(remoteFiles, remoteFile)
	}

	// Randomly download using download to file and download to stream methods.
	wg := new(sync.WaitGroup)
	for i := 0; i < parallelDownloads; i++ {
		wg.Add(1)
		go func() {
			var err error
			var rf = remoteFiles[fastrand.Intn(len(remoteFiles))]
			if fastrand.Intn(2) == 0 {
				_, err = renter.DownloadByStream(rf)
			} else {
				_, err = renter.DownloadToDisk(rf, false)
			}
			if err != nil {
				t.Error("Download failed:", err)
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

// testRenterLocalRepair tests if a renter correctly repairs a file from disk
// after a host goes offline.
func testRenterLocalRepair(t *testing.T, tg *siatest.TestGroup) {
	// Grab the first of the group's renters
	renter := tg.Renters()[0]

	// Check that we have enough hosts for this test.
	if len(tg.Hosts()) < 2 {
		t.Fatal("This test requires at least 2 hosts")
	}

	// Set fileSize and redundancy for upload
	fileSize := int(modules.SectorSize)
	dataPieces := uint64(1)
	parityPieces := uint64(len(tg.Hosts())) - dataPieces

	// Upload file
	_, remoteFile, err := renter.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
	if err != nil {
		t.Fatal(err)
	}
	// Get the file info of the fully uploaded file. Tha way we can compare the
	// redundancies later.
	fi, err := renter.FileInfo(remoteFile)
	if err != nil {
		t.Fatal("failed to get file info", err)
	}

	// Take down one of the hosts and check if redundancy decreases.
	if err := tg.RemoveNode(tg.Hosts()[0]); err != nil {
		t.Fatal("Failed to shutdown host", err)
	}
	expectedRedundancy := float64(dataPieces+parityPieces-1) / float64(dataPieces)
	if err := renter.WaitForDecreasingRedundancy(remoteFile, expectedRedundancy); err != nil {
		t.Fatal("Redundancy isn't decreasing", err)
	}
	// We should still be able to download
	if _, err := renter.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
	// Bring up a new host and check if redundancy increments again.
	if err := tg.AddNodes(node.HostTemplate); err != nil {
		t.Fatal("Failed to create a new host", err)
	}
	if err := renter.WaitForUploadRedundancy(remoteFile, fi.Redundancy); err != nil {
		t.Fatal("File wasn't repaired", err)
	}
	// We should be able to download
	if _, err := renter.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
}

// testRenterRemoteRepair tests if a renter correctly repairs a file by
// downloading it after a host goes offline.
func testRenterRemoteRepair(t *testing.T, tg *siatest.TestGroup) {
	// Grab the first of the group's renters
	r := tg.Renters()[0]

	// Check that we have enough hosts for this test.
	if len(tg.Hosts()) < 2 {
		t.Fatal("This test requires at least 2 hosts")
	}

	// Set fileSize and redundancy for upload
	fileSize := int(modules.SectorSize)
	dataPieces := uint64(1)
	parityPieces := uint64(len(tg.Hosts())) - dataPieces

	// Upload file
	localFile, remoteFile, err := r.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
	if err != nil {
		t.Fatal(err)
	}
	// Get the file info of the fully uploaded file. Tha way we can compare the
	// redundancieslater.
	fi, err := r.FileInfo(remoteFile)
	if err != nil {
		t.Fatal("failed to get file info", err)
	}

	// Delete the file locally.
	if err := localFile.Delete(); err != nil {
		t.Fatal("failed to delete local file", err)
	}

	// Take down all of the parity hosts and check if redundancy decreases.
	for i := uint64(0); i < parityPieces; i++ {
		if err := tg.RemoveNode(tg.Hosts()[0]); err != nil {
			t.Fatal("Failed to shutdown host", err)
		}
	}
	expectedRedundancy := float64(dataPieces+parityPieces-1) / float64(dataPieces)
	if err := r.WaitForDecreasingRedundancy(remoteFile, expectedRedundancy); err != nil {
		t.Fatal("Redundancy isn't decreasing", err)
	}
	// We should still be able to download
	if _, err := r.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
	// Bring up new parity hosts and check if redundancy increments again.
	if err := tg.AddNodeN(node.HostTemplate, int(parityPieces)); err != nil {
		t.Fatal("Failed to create a new host", err)
	}
	// When doing remote repair the redundancy might not reach 100%.
	expectedRedundancy = (1.0 - renter.RemoteRepairDownloadThreshold) * fi.Redundancy
	if err := r.WaitForUploadRedundancy(remoteFile, expectedRedundancy); err != nil {
		t.Fatal("File wasn't repaired", err)
	}
	// We should be able to download
	if _, err := r.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
}
<commit_msg>add test for stream caching<commit_after>package renter

import (
	"sync"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter"
	"github.com/NebulousLabs/Sia/node"
	"github.com/NebulousLabs/Sia/siatest"
	"github.com/NebulousLabs/fastrand"
)

// TestRenter executes a number of subtests using the same TestGroup to
// save time on initialization
func TestRenter(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	// Create a group for the subtests
	groupParams := siatest.GroupParams{
		Hosts:   5,
		Renters: 1,
		Miners:  1,
	}
	tg, err := siatest.NewGroupFromTemplate(groupParams)
	if err != nil {
		t.Fatal("Failed to create group: ", err)
	}
	defer func() {
		if err := tg.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	// Specify subtests to run
	subTests := []struct {
		name string
		test func(*testing.T, *siatest.TestGroup)
	}{
		{"TestRenterStreamingCache", testRenterStreamingCache},
		{"TestUploadDownload", testUploadDownload},
		{"TestDownloadMultipleLargeSectors", testDownloadMultipleLargeSectors},
		{"TestRenterLocalRepair", testRenterLocalRepair},
		{"TestRenterRemoteRepair", testRenterRemoteRepair},
	}
	// Run subtests
	for _, subtest := range subTests {
		t.Run(subtest.name, func(t *testing.T) {
			subtest.test(t, tg)
		})
	}
}

// testUploadDownload is a subtest that uses an existing TestGroup to test if
// uploading and downloading a file works
func testUploadDownload(t *testing.T, tg *siatest.TestGroup) {
	// Grab the first of the group's renters
	renter := tg.Renters()[0]
	// Upload file, creating a piece for each host in the group
	dataPieces := uint64(1)
	parityPieces := uint64(len(tg.Hosts())) - dataPieces
	fileSize := 100 + siatest.Fuzz()
	localFile, remoteFile, err := renter.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
	if err != nil {
		t.Fatal("Failed to upload a file for testing: ", err)
	}
	// Download the file synchronously directly into memory
	_, err = renter.DownloadByStream(remoteFile)
	if err != nil {
		t.Fatal(err)
	}
	// Download the file synchronously to a file on disk
	_, err = renter.DownloadToDisk(remoteFile, false)
	if err != nil {
		t.Fatal(err)
	}
	// Download the file asynchronously and wait for the download to finish.
	localFile, err = renter.DownloadToDisk(remoteFile, true)
	if err != nil {
		t.Error(err)
	}
	if err := renter.WaitForDownload(localFile, remoteFile); err != nil {
		t.Error(err)
	}
	// Stream the file.
	_, err = renter.Stream(remoteFile)
	if err != nil {
		t.Fatal(err)
	}
	// Stream the file partially a few times. At least 1 byte is streamed.
	for i := 0; i < 5; i++ {
		from := fastrand.Intn(fileSize - 1)             // [0..fileSize-2]
		to := from + 1 + fastrand.Intn(fileSize-from-1) // [from+1..fileSize-1]
		_, err = renter.StreamPartial(remoteFile, localFile, uint64(from), uint64(to))
		if err != nil {
			t.Fatal(err)
		}
	}
}

// testDownloadMultipleLargeSectors downloads multiple large files (>5 Sectors)
// in parallel and makes sure that the downloads are blocking each other.
func testDownloadMultipleLargeSectors(t *testing.T, tg *siatest.TestGroup) {
	// parallelDownloads is the number of downloads that are run in parallel.
	parallelDownloads := 10
	// fileSize is the size of the downloaded file.
	fileSize := int(10*modules.SectorSize) + siatest.Fuzz()
	// set download limits and reset them after test.
	// uniqueRemoteFiles is the number of files that will be uploaded to the
	// network. Downloads will choose the remote file to download randomly.
	uniqueRemoteFiles := 5
	// Grab the first of the group's renters
	renter := tg.Renters()[0]
	// set download limits and reset them after test.
	if err := renter.RenterPostRateLimit(int64(fileSize)*2, 0); err != nil {
		t.Fatal("failed to set renter bandwidth limit", err)
	}
	defer func() {
		if err := renter.RenterPostRateLimit(0, 0); err != nil {
			t.Error("failed to reset renter bandwidth limit", err)
		}
	}()

	// Upload files
	dataPieces := uint64(len(tg.Hosts())) - 1
	parityPieces := uint64(1)
	remoteFiles := make([]*siatest.RemoteFile, 0, uniqueRemoteFiles)
	for i := 0; i < uniqueRemoteFiles; i++ {
		_, remoteFile, err := renter.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
		if err != nil {
			t.Fatal("Failed to upload a file for testing: ", err)
		}
		remoteFiles = append(remoteFiles, remoteFile)
	}

	// Randomly download using download to file and download to stream methods.
	wg := new(sync.WaitGroup)
	for i := 0; i < parallelDownloads; i++ {
		wg.Add(1)
		go func() {
			var err error
			var rf = remoteFiles[fastrand.Intn(len(remoteFiles))]
			if fastrand.Intn(2) == 0 {
				_, err = renter.DownloadByStream(rf)
			} else {
				_, err = renter.DownloadToDisk(rf, false)
			}
			if err != nil {
				t.Error("Download failed:", err)
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

// testRenterLocalRepair tests if a renter correctly repairs a file from disk
// after a host goes offline.
func testRenterLocalRepair(t *testing.T, tg *siatest.TestGroup) {
	// Grab the first of the group's renters
	renter := tg.Renters()[0]

	// Check that we have enough hosts for this test.
	if len(tg.Hosts()) < 2 {
		t.Fatal("This test requires at least 2 hosts")
	}

	// Set fileSize and redundancy for upload
	fileSize := int(modules.SectorSize)
	dataPieces := uint64(1)
	parityPieces := uint64(len(tg.Hosts())) - dataPieces

	// Upload file
	_, remoteFile, err := renter.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
	if err != nil {
		t.Fatal(err)
	}
	// Get the file info of the fully uploaded file. Tha way we can compare the
	// redundancies later.
	fi, err := renter.FileInfo(remoteFile)
	if err != nil {
		t.Fatal("failed to get file info", err)
	}

	// Take down one of the hosts and check if redundancy decreases.
	if err := tg.RemoveNode(tg.Hosts()[0]); err != nil {
		t.Fatal("Failed to shutdown host", err)
	}
	expectedRedundancy := float64(dataPieces+parityPieces-1) / float64(dataPieces)
	if err := renter.WaitForDecreasingRedundancy(remoteFile, expectedRedundancy); err != nil {
		t.Fatal("Redundancy isn't decreasing", err)
	}
	// We should still be able to download
	if _, err := renter.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
	// Bring up a new host and check if redundancy increments again.
	if err := tg.AddNodes(node.HostTemplate); err != nil {
		t.Fatal("Failed to create a new host", err)
	}
	if err := renter.WaitForUploadRedundancy(remoteFile, fi.Redundancy); err != nil {
		t.Fatal("File wasn't repaired", err)
	}
	// We should be able to download
	if _, err := renter.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
}

// testRenterRemoteRepair tests if a renter correctly repairs a file by
// downloading it after a host goes offline.
func testRenterRemoteRepair(t *testing.T, tg *siatest.TestGroup) {
	// Grab the first of the group's renters
	r := tg.Renters()[0]

	// Check that we have enough hosts for this test.
	if len(tg.Hosts()) < 2 {
		t.Fatal("This test requires at least 2 hosts")
	}

	// Set fileSize and redundancy for upload
	fileSize := int(modules.SectorSize)
	dataPieces := uint64(1)
	parityPieces := uint64(len(tg.Hosts())) - dataPieces

	// Upload file
	localFile, remoteFile, err := r.UploadNewFileBlocking(fileSize, dataPieces, parityPieces)
	if err != nil {
		t.Fatal(err)
	}
	// Get the file info of the fully uploaded file. Tha way we can compare the
	// redundancieslater.
	fi, err := r.FileInfo(remoteFile)
	if err != nil {
		t.Fatal("failed to get file info", err)
	}

	// Delete the file locally.
	if err := localFile.Delete(); err != nil {
		t.Fatal("failed to delete local file", err)
	}

	// Take down all of the parity hosts and check if redundancy decreases.
	for i := uint64(0); i < parityPieces; i++ {
		if err := tg.RemoveNode(tg.Hosts()[0]); err != nil {
			t.Fatal("Failed to shutdown host", err)
		}
	}
	expectedRedundancy := float64(dataPieces+parityPieces-1) / float64(dataPieces)
	if err := r.WaitForDecreasingRedundancy(remoteFile, expectedRedundancy); err != nil {
		t.Fatal("Redundancy isn't decreasing", err)
	}
	// We should still be able to download
	if _, err := r.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
	// Bring up new parity hosts and check if redundancy increments again.
	if err := tg.AddNodeN(node.HostTemplate, int(parityPieces)); err != nil {
		t.Fatal("Failed to create a new host", err)
	}
	// When doing remote repair the redundancy might not reach 100%.
	expectedRedundancy = (1.0 - renter.RemoteRepairDownloadThreshold) * fi.Redundancy
	if err := r.WaitForUploadRedundancy(remoteFile, expectedRedundancy); err != nil {
		t.Fatal("File wasn't repaired", err)
	}
	// We should be able to download
	if _, err := r.DownloadByStream(remoteFile); err != nil {
		t.Fatal("Failed to download file", err)
	}
}

// testRenterStreamingCache checks if the chunk cache works correctly.
func testRenterStreamingCache(t *testing.T, tg *siatest.TestGroup) {
	// Grab the first of the group's renters
	r := tg.Renters()[0]

	// Set fileSize and redundancy for upload
	dataPieces := uint64(1)
	parityPieces := uint64(len(tg.Hosts())) - dataPieces

	// Set the bandwidth limit to 1 chunk per second.
	pieceSize := modules.SectorSize - crypto.TwofishOverhead
	chunkSize := int64(pieceSize * dataPieces)
	if err := r.RenterPostRateLimit(chunkSize, chunkSize); err != nil {
		t.Fatal(err)
	}

	// Upload a file that is a single chunk big.
	_, remoteFile, err := r.UploadNewFileBlocking(int(chunkSize), dataPieces, parityPieces)
	if err != nil {
		t.Fatal(err)
	}

	// Download the same chunk 250 times. This should take at least 250 seconds
	// without caching but not more than 30 with caching.
	start := time.Now()
	for i := 0; i < 250; i++ {
		if _, err := r.Stream(remoteFile); err != nil {
			t.Fatal(err)
		}
		if time.Since(start) > time.Second*30 {
			t.Fatal("download took longer than 30 seconds")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"flag"
	"github.com/syndtr/goleveldb/leveldb"
)

var (
	// go的正则在匹配ReadAll出来的数据时有点奇怪，必须用[\w\W]{0,}?来代替\n

	// 最大同时开启线程数
	maxThread int
	// 用于获取版块名称
	getForumName = regexp.MustCompile(`<h1 class="xs2">[\w\W]{0,}?<a[^>]*>([^<]*)</a>`)
	// 用于获取版块帖子分页数量
	getForumPageNumber = regexp.MustCompile(`<a href="[^"]+" class="last">\.\.\. ([0-9]+)</a>`)
	// 用于获取版块介绍
	getForumIntroduction = regexp.MustCompile(`<div id="forum_rules_[0-9]*"[^>]*>([\w\W]{0,}?)(?:</div>[\w\W]{0,}?){3}<div class="drag">`)
	// 用于获取帖子信息
	getPostInfo = regexp.MustCompile(`id="normalthread_[0-9]+"[\w\W]{0,}?class="icn"[\w\W]{0,}?href="([^"]*)"[\w\W]{0,}?\n[\w\W]{0,}?>([^<]*)[\w\W]{0,}?class="xst" >([^<]*)[\w\W]{0,}?[\w\W]{0,}?>([^<]*)[\w\W]{0,}?(?:)?([^<]*)`)
	// 用于获取帖子内容
	getPostBody = regexp.MustCompile(`<div class="pcb">([\w\W]{0,}?)<div id="comment_[0-9]*" class="cm">`)
)

func main() {
	flag.IntVar(&maxThread, "m", 10, "最大线程数量")
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Println("请在命令后添加论坛版块FID参数")
		fmt.Println("例如：")
		fmt.Println(os.Args[0] + " 45 -m=10")
		fmt.Println("其中m为可选参数，用于设置最大线程数量")
		fmt.Println("FID可从版块URL中寻找")
		fmt.Println("数据会保存在db/FID路径中")
		return
	}

	printInfo("最大线程数量", maxThread)

	// 论坛版块fid
	fid := flag.Arg(0)
	printInfo("版块FID", fid)

	resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=1")
	if err != nil {
		printError("http.Get", err)
		os.Exit(1)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		printError("ReadAll", err)
		os.Exit(1)
	}

	if bytes.Contains(body, []byte(`<p>抱歉，指定的版块不存在</p>`)) {
		printError("GetForum", "版块不存在，请检查PID是否正确")
		os.Exit(1)
	}

	// 获取版块名称
	n := getForumName.FindSubmatch(body)
	if len(n) == 0 {
		printError("GetForumName", "获取版块名称出错")
		os.Exit(1)
	}
	forumName := string(n[1])
	printInfo("版块名称", forumName)

	// 获取版块帖子分页数量
	n = getForumPageNumber.FindSubmatch(body)
	if len(n) == 0 {
		printError("GetForumPageNumber", "获取版块分页数量出错")
		os.Exit(1)
	}
	maxPagesNum, err := strconv.Atoi(string(n[1]))
	if err != nil {
		printError("Atoi", err)
		os.Exit(1)
	}
	printInfo("本版块全部分页数量", maxPagesNum)

	// 获取版块介绍
	var forumIntroduction string
	n = getForumIntroduction.FindSubmatch(body)
	if len(n) == 0 {
		printError("GetForumIntroduction", "获取版块介绍出错，版块介绍为空")
	} else {
		forumIntroduction = string(n[1])
	}

	// 创建数据库
	db, err := leveldb.OpenFile("db/"+fid, nil)
	if err != nil {
		printError("OpenDB", err)
		os.Exit(1)
	}
	defer db.Close()

	// 保存版块信息到数据库
	buf, err := encode(&forumInfo{
		Name:         forumName,
		PageNumber:   maxPagesNum,
		Introduction: forumIntroduction,
	})
	if err != nil {
		printError("EncodeForumInfo", err)
	}
	err = db.Put([]byte("info"), buf, nil)
	if err != nil {
		printError("PutForumInfo", err)
		os.Exit(1)
	}

	var done = make(chan int)
	// 用于统计还有多少页未完成
	var pageAmount = maxPagesNum

	getPage := func(page int) {
		postList, err := getPagesList(fid, page)
		if err != nil {
			printError("getPagesList"+string(page), err)
		} else {
			// 获取每个帖子的内容
			for i, v := range postList {
				// for用于重试
				for {
					resp, err := http.Get("http://www.mcbbs.net/" + v.Url)
					if err != nil {
						printError("GetPost", err)
					} else {
						defer resp.Body.Close()

						// 检查服务器是否返回成功
						if resp.StatusCode != 200 {
							// 服务器错误
							printError("GetPost.ServerError", "服务器错误，错误码：", resp.StatusCode)
							if resp.StatusCode == 404 {
								// 如果为404，则没有重试的必要，跳出重试
								printError("GetPost.Retry", "帖子《"+v.Title+"》不存在")
								break
							}
						} else {
							body, err := ioutil.ReadAll(resp.Body)
							if err != nil {
								printError("GetPost.ReadAll", err)
							} else {
								n := getPostBody.FindSubmatch(body)
								// 检查是否获取body成功
								if len(n) == 0 {
									printError("GetPost.FindSubmatch", "未找到页面内文章部分")
									// 获取失败，一般是帖子被屏蔽或者论坛数据库错误
									break
								} else {
									postBody := string(n[1])
									// 存入Body
									postList[i].Body = postBody
									printInfo("GetPost", v.Title, "[OK]")
									// 跳出循环重试
									break
								}
							}
						}
					}
					// 获取失败，重试
					printError("GetPost.Retry", "获取帖子《"+v.Title+"》失败，正在重试")
				}
			}
		}
		// 编码
		byt, err := encode(postList)
		if err != nil {
			printError("Encode", err)
			return
		}
		// 存入数据库
		err = db.Put([]byte("page_"+strconv.Itoa(page)), byt, nil)
		if err != nil {
			printError("db.Put", err)
			return
		}

		pageAmount--

		printInfo("OK", "线程", page, "执行完毕")
		printInfo("OK", "版块分页", page, "中的所有帖子已经储存到本地，还有", pageAmount, "页正在下载中")
		done <- page
	}

	// 如果页面数量小于线程数量
	// 直接开启maxPagesNum个线程
	if maxPagesNum < maxThread {
		for i := 0; i < maxPagesNum; i++ {
			go getPage(i + 1)
		}
	} else {
		// 开启设置数量的线程
		for i := 0; i < maxThread; i++ {
			go getPage(i + 1)
		}

		// 用于统计已开启线程数
		var i = maxThread

		// 单个线程执行完毕，补充线程
		// 使线程数量始终保持设置的最大数量
		for {
			// 等待线程完成
			<-done
			i++
			// 因为前面已经开启了maxThread个线程
			// 但是没有返回maxThread个done
			// 所以i最终会等于  maxThread + maxPagesNum
			// 所以需要检查已完成页数是否超出总页数
			if i <= maxPagesNum {
				// 开启新线程
				go getPage(i)
			} else {
				// 所有页面已下载完毕
				break
			}
		}
	}

	printInfo("OK", "FID为", fid, "的版块中的所有帖子已储存到本地")
}

// 获取单页面的帖子列表
func getPagesList(fid string, pageNum int) (pageList []postInfo, err error) {
	for {
		resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=" + strconv.Itoa(pageNum))
		if err != nil {
			printError("GetPageList", err)
		} else {
			defer resp.Body.Close()

			// 检查服务器是否返回成功
			if resp.StatusCode != 200 {
				// 服务器错误
				printError("GetPost.ServerError", "服务器错误，错误码：", resp.StatusCode)
			} else {
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					printError("GetPageList.ReadAll", err)
				} else {
					/*
						// getPage所匹配的信息
						// 获取到的参数已用“m[x]说明”标出

						[m[2]分类] m[3]标题

						 ...234
						New

						m[4]作者
						m[6]发帖时间2/span>

						111189

						谢普
						5 天前

					*/
					m := getPostInfo.FindAllStringSubmatch(string(body), -1)

					// 检查是否获取成功
					if len(m) == 0 {
						printError("GetPageList.FindAllStringSubmatch", "未找到分页内帖子")
						// 失败，跳出
					} else {
						// 处理单个帖子信息
						for _, v := range m {
							date := v[5]
							// 如果发帖时间1为空，那么使用发帖时间2
							if date == "" {
								date = v[6]
							}
							// 储存帖子信息
							postInf := postInfo{
								Url:      v[1],
								Category: v[2],
								Title:    v[3],
								Author:   v[4],
								Date:     date,
							}
							pageList = append(pageList, postInf)
						}
					}
					// 跳出重试
					break
				}
			}
		}
		// 获取失败，重试
		printError("getPageList", "获取版块分页", pageNum, "失败，正在重试")
	}

	return pageList, nil
}

func printInfo(s string, v ...interface{}) {
	log.Println(append([]interface{}{"[INFO]", s + ":"}, v...)...)
}

func printError(s string, v ...interface{}) {
	log.Println(append([]interface{}{"[ERROR]", s + ":"}, v...)...)
}

func encode(data interface{}) ([]byte, error) {
	buf := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(data)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func decode(data []byte, to interface{}) error {
	buf := bytes.NewBuffer(data)
	dec := gob.NewDecoder(buf)
	return dec.Decode(to)
}

type forumInfo struct {
	Name         string
	PageNumber   int
	Introduction string
}

type postInfo struct {
	Category string
	Url      string
	Title    string
	Author   string
	Date     string
	Body     string
}
<commit_msg>modify: 优化结构<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"flag"
	"github.com/syndtr/goleveldb/leveldb"
)

var (
	// go的正则在匹配ReadAll出来的数据时有点奇怪，必须用[\w\W]{0,}?来代替\n

	// 最大同时开启线程数
	maxThread int
	// 用于获取版块名称
	getForumName = regexp.MustCompile(`<h1 class="xs2">[\w\W]{0,}?<a[^>]*>([^<]*)</a>`)
	// 用于获取版块帖子分页数量
	getForumPageNumber = regexp.MustCompile(`<a href="[^"]+" class="last">\.\.\. ([0-9]+)</a>`)
	// 用于获取版块介绍
	getForumIntroduction = regexp.MustCompile(`<div id="forum_rules_[0-9]*"[^>]*>([\w\W]{0,}?)(?:</div>[\w\W]{0,}?){3}<div class="drag">`)
	// 用于获取帖子信息
	getPostInfo = regexp.MustCompile(`id="normalthread_[0-9]+"[\w\W]{0,}?class="icn"[\w\W]{0,}?href="([^"]*)"[\w\W]{0,}?\n[\w\W]{0,}?>([^<]*)[\w\W]{0,}?class="xst" >([^<]*)[\w\W]{0,}?[\w\W]{0,}?>([^<]*)[\w\W]{0,}?(?:)?([^<]*)`)
	// 用于获取帖子内容
	getPostBody = regexp.MustCompile(`<div class="pcb">([\w\W]{0,}?)<div id="comment_[0-9]*" class="cm">`)
)

func main() {
	flag.IntVar(&maxThread, "m", 10, "最大线程数量")
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Println("请在命令后添加论坛版块FID参数")
		fmt.Println("例如：")
		fmt.Println(os.Args[0] + " 45 -m=10")
		fmt.Println("其中m为可选参数，用于设置最大线程数量")
		fmt.Println("FID可从版块URL中寻找")
		fmt.Println("数据会保存在db/FID路径中")
		return
	}

	printInfo("最大线程数量", maxThread)

	// 论坛版块fid
	fid := flag.Arg(0)
	printInfo("版块FID", fid)

	resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=1")
	if err != nil {
		printError("http.Get", err)
		os.Exit(1)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		printError("ReadAll", err)
		os.Exit(1)
	}

	if bytes.Contains(body, []byte(`<p>抱歉，指定的版块不存在</p>`)) {
		printError("GetForum", "版块不存在，请检查PID是否正确")
		os.Exit(1)
	}

	// 获取版块名称
	n := getForumName.FindSubmatch(body)
	if len(n) == 0 {
		printError("GetForumName", "获取版块名称出错")
		os.Exit(1)
	}
	forumName := string(n[1])
	printInfo("版块名称", forumName)

	// 获取版块帖子分页数量
	n = getForumPageNumber.FindSubmatch(body)
	if len(n) == 0 {
		printError("GetForumPageNumber", "获取版块分页数量出错")
		os.Exit(1)
	}
	maxPagesNum, err := strconv.Atoi(string(n[1]))
	if err != nil {
		printError("Atoi", err)
		os.Exit(1)
	}
	printInfo("本版块全部分页数量", maxPagesNum)

	// 获取版块介绍
	var forumIntroduction string
	n = getForumIntroduction.FindSubmatch(body)
	if len(n) == 0 {
		printError("GetForumIntroduction", "获取版块介绍出错，版块介绍为空")
	} else {
		forumIntroduction = string(n[1])
	}

	// 创建数据库
	db, err := leveldb.OpenFile("db/"+fid, nil)
	if err != nil {
		printError("OpenDB", err)
		os.Exit(1)
	}
	defer db.Close()

	// 保存版块信息到数据库
	buf, err := encode(&forumInfo{
		Name:         forumName,
		PageNumber:   maxPagesNum,
		Introduction: forumIntroduction,
	})
	if err != nil {
		printError("EncodeForumInfo", err)
	}
	err = db.Put([]byte("info"), buf, nil)
	if err != nil {
		printError("PutForumInfo", err)
		os.Exit(1)
	}

	var done = make(chan int)
	// 用于统计还有多少页未完成
	var pageAmount = maxPagesNum

	getPage := func(page int) {
		postList, err := getPagesList(fid, page)
		if err != nil {
			printError("getPagesList"+string(page), err)
		} else {
			// 获取每个帖子的内容
			for i, v := range postList {
				// for用于重试
				for {
					resp, err := http.Get("http://www.mcbbs.net/" + v.Url)
					if err != nil {
						printError("GetPost", err)
					} else {
						defer resp.Body.Close()

						// 检查服务器是否返回成功
						if resp.StatusCode != 200 {
							// 服务器错误
							printError("GetPost.ServerError", "服务器错误，错误码：", resp.StatusCode)
							if resp.StatusCode == 404 {
								// 如果为404，则没有重试的必要，跳出重试
								printError("GetPost.Retry", "帖子《"+v.Title+"》不存在")
								break
							}
						} else {
							body, err := ioutil.ReadAll(resp.Body)
							if err != nil {
								printError("GetPost.ReadAll", err)
							} else {
								n := getPostBody.FindSubmatch(body)
								// 检查是否获取body成功
								if len(n) == 0 {
									printError("GetPost.FindSubmatch", "未找到页面内文章部分")
									// 获取失败，一般是帖子被屏蔽或者论坛数据库错误
									break
								} else {
									postBody := string(n[1])
									// 存入Body
									postList[i].Body = postBody
									printInfo("GetPost", v.Title, "[OK]")
									// 跳出循环重试
									break
								}
							}
						}
					}
					// 获取失败，重试
					printError("GetPost.Retry", "获取帖子《"+v.Title+"》失败，正在重试")
				}
			}
		}
		// 编码
		byt, err := encode(postList)
		if err != nil {
			printError("Encode", err)
			return
		}
		// 存入数据库
		err = db.Put([]byte("page_"+strconv.Itoa(page)), byt, nil)
		if err != nil {
			printError("db.Put", err)
			return
		}

		pageAmount--

		printInfo("OK", "线程", page, "执行完毕")
		printInfo("OK", "版块分页", page, "中的所有帖子已经储存到本地，还有", pageAmount, "页正在下载中")
		done <- page
	}

	// 如果页面数量小于线程数量
	// 直接开启maxPagesNum个线程
	if maxPagesNum < maxThread {
		maxThread = maxPagesNum
	}

	// 开启设置数量的线程
	for i := 0; i < maxThread; i++ {
		go getPage(i + 1)
	}

	// 用于统计已开启线程数
	var i = maxThread

	// 单个线程执行完毕，补充线程
	// 使线程数量始终保持设置的最大数量
	for {
		// 等待线程完成
		<-done
		i++
		// 因为前面已经开启了maxThread个线程
		// 但是没有返回maxThread个done
		// 所以i最终会等于  maxThread + maxPagesNum
		// 所以需要检查已完成页数是否超出总页数
		if i <= maxPagesNum {
			// 开启新线程
			go getPage(i)
		} else {
			// 所有页面已下载完毕
			break
		}
	}

	printInfo("OK", "FID为", fid, "的版块中的所有帖子已储存到本地")
}

// 获取单页面的帖子列表
func getPagesList(fid string, pageNum int) (pageList []postInfo, err error) {
	for {
		resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=" + strconv.Itoa(pageNum))
		if err != nil {
			printError("GetPageList", err)
		} else {
			defer resp.Body.Close()

			// 检查服务器是否返回成功
			if resp.StatusCode != 200 {
				// 服务器错误
				printError("GetPost.ServerError", "服务器错误，错误码：", resp.StatusCode)
			} else {
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					printError("GetPageList.ReadAll", err)
				} else {
					/*
						// getPage所匹配的信息
						// 获取到的参数已用“m[x]说明”标出

						[m[2]分类] m[3]标题

						 ...234
						New

						m[4]作者
						m[6]发帖时间2/span>

						111189

						谢普
						5 天前

					*/
					m := getPostInfo.FindAllStringSubmatch(string(body), -1)

					// 检查是否获取成功
					if len(m) == 0 {
						printError("GetPageList.FindAllStringSubmatch", "未找到分页内帖子")
						// 失败，跳出
					} else {
						// 处理单个帖子信息
						for _, v := range m {
							date := v[5]
							// 如果发帖时间1为空，那么使用发帖时间2
							if date == "" {
								date = v[6]
							}
							// 储存帖子信息
							postInf := postInfo{
								Url:      v[1],
								Category: v[2],
								Title:    v[3],
								Author:   v[4],
								Date:     date,
							}
							pageList = append(pageList, postInf)
						}
					}
					// 跳出重试
					break
				}
			}
		}
		// 获取失败，重试
		printError("getPageList", "获取版块分页", pageNum, "失败，正在重试")
	}

	return pageList, nil
}

func printInfo(s string, v ...interface{}) {
	log.Println(append([]interface{}{"[INFO]", s + ":"}, v...)...)
}

func printError(s string, v ...interface{}) {
	log.Println(append([]interface{}{"[ERROR]", s + ":"}, v...)...)
}

func encode(data interface{}) ([]byte, error) {
	buf := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(data)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func decode(data []byte, to interface{}) error {
	buf := bytes.NewBuffer(data)
	dec := gob.NewDecoder(buf)
	return dec.Decode(to)
}

type forumInfo struct {
	Name         string
	PageNumber   int
	Introduction string
}

type postInfo struct {
	Category string
	Url      string
	Title    string
	Author   string
	Date     string
	Body     string
}
<|endoftext|>


<commit_before>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package scraper // import "miniflux.app/reader/scraper"

// List of predefined scraper rules (alphabetically sorted)
// domain => CSS selectors
var predefinedRules = map[string]string{
	"bbc.co.uk":           "div.vxp-column--single, div.story-body__inner, ul.gallery-images__list",
	"cbc.ca":              ".story-content",
	"darkreading.com":     "#article-main:not(header)",
	"developpez.com":      "div[itemprop=articleBody]",
	"francetvinfo.fr":     ".text",
	"github.com":          "article.entry-content",
	"heise.de":            "header .article-content__lead, header .article-image, div.article-layout__content.article-content",
	"igen.fr":             "section.corps",
	"ing.dk":              "section.body",
	"lapresse.ca":         ".amorce, .entry",
	"lemonde.fr":          "article",
	"lepoint.fr":          ".art-text",
	"lesjoiesducode.fr":   ".blog-post-content img",
	"lesnumeriques.com":   ".text",
	"linux.com":           "div.content, div[property]",
	"medium.com":          ".section-content",
	"mac4ever.com":        "div[itemprop=articleBody]",
	"monwindows.com":      ".blog-post-body",
	"npr.org":             "#storytext",
	"oneindia.com":        ".io-article-body",
	"opensource.com":      "div[property]",
	"osnews.com":          "div.newscontent1",
	"phoronix.com":        "div.content",
	"pseudo-sciences.org": "#art_main",
	"raywenderlich.com":   "article",
	"slate.fr":            ".field-items",
	"techcrunch.com":      "div.article-entry",
	"theoatmeal.com":      "div#comic",
	"theregister.co.uk":   "#body",
	"universfreebox.com":  "#corps_corps",
	"version2.dk":         "section.body",
	"wdwnt.com":           "div.entry-content",
	"wired.com":           "main figure, article",
	"zeit.de":             ".summary, .article-body",
	"zdnet.com":           "div.storyBody",
	"openingsource.org":   "article.suxing-popup-gallery",
}
<commit_msg>Added scraper rule for financialsamurai.com<commit_after>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package scraper // import "miniflux.app/reader/scraper"

// List of predefined scraper rules (alphabetically sorted)
// domain => CSS selectors
var predefinedRules = map[string]string{
	"bbc.co.uk":           "div.vxp-column--single, div.story-body__inner, ul.gallery-images__list",
	"cbc.ca":              ".story-content",
	"darkreading.com":     "#article-main:not(header)",
	"developpez.com":      "div[itemprop=articleBody]",
	"financialsamurai.com": "article",
	"francetvinfo.fr":     ".text",
	"github.com":          "article.entry-content",
	"heise.de":            "header .article-content__lead, header .article-image, div.article-layout__content.article-content",
	"igen.fr":             "section.corps",
	"ing.dk":              "section.body",
	"lapresse.ca":         ".amorce, .entry",
	"lemonde.fr":          "article",
	"lepoint.fr":          ".art-text",
	"lesjoiesducode.fr":   ".blog-post-content img",
	"lesnumeriques.com":   ".text",
	"linux.com":           "div.content, div[property]",
	"medium.com":          ".section-content",
	"mac4ever.com":        "div[itemprop=articleBody]",
	"monwindows.com":      ".blog-post-body",
	"npr.org":             "#storytext",
	"oneindia.com":        ".io-article-body",
	"opensource.com":      "div[property]",
	"osnews.com":          "div.newscontent1",
	"phoronix.com":        "div.content",
	"pseudo-sciences.org": "#art_main",
	"raywenderlich.com":   "article",
	"slate.fr":            ".field-items",
	"techcrunch.com":      "div.article-entry",
	"theoatmeal.com":      "div#comic",
	"theregister.co.uk":   "#body",
	"universfreebox.com":  "#corps_corps",
	"version2.dk":         "section.body",
	"wdwnt.com":           "div.entry-content",
	"wired.com":           "main figure, article",
	"zeit.de":             ".summary, .article-body",
	"zdnet.com":           "div.storyBody",
	"openingsource.org":   "article.suxing-popup-gallery",
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"database/sql"
	"errors"
	sqlite3 "github.com/mattn/go-sqlite3"
	mcq "github.com/mediachain/concat/mc/query"
	pb "github.com/mediachain/concat/proto"
	multihash "github.com/multiformats/go-multihash"
)

var (
	NodeMustBeOffline = errors.New("Node must be offline")
)

func (node *Node) doGC(ctx context.Context) (int, error) {
	if node.status != StatusOffline {
		return 0, NodeMustBeOffline
	}

	gc := &GCDB{}
	err := gc.Open(node.home)
	if err != nil {
		return 0, err
	}
	defer gc.Close()

	err = gc.Merge(ctx, node.db)
	if err != nil {
		return 0, err
	}

	return gc.GC(ctx, node.ds)
}

func (node *Node) doCompact() error {
	if node.status != StatusOffline {
		return NodeMustBeOffline
	}

	node.ds.Compact()
	return nil
}

type GCDB struct {
	db        *sql.DB
	insertKey *sql.Stmt
	countKeys *sql.Stmt
}

func (gc *GCDB) Open(home string) error {
	// TODO on-disk index for large deletions
	const dbpath = ":memory:"
	db, err := sql.Open("sqlite3", dbpath)
	if err != nil {
		return err
	}
	gc.db = db

	_, err = db.Exec("CREATE TABLE Refs (key VARCHAR(64) PRIMARY KEY)")
	if err != nil {
		return err
	}

	insertKey, err := db.Prepare("INSERT INTO Refs VALUES (?)")
	if err != nil {
		return err
	}
	gc.insertKey = insertKey

	countKeys, err := db.Prepare("SELECT COUNT(1) FROM Refs WHERE key = ?")
	if err != nil {
		return err
	}
	gc.countKeys = countKeys

	return nil
}

func (gc *GCDB) Close() error {
	// TODO clear on-disk index
	return gc.db.Close()
}

func (gc *GCDB) Merge(ctx context.Context, db StatementDB) error {
	q, err := mcq.ParseQuery("SELECT * FROM *")
	if err != nil {
		return err
	}

	ch, err := db.QueryStream(ctx, q)
	if err != nil {
		return err
	}

	const batch = 1024
	keys := make(map[string]bool)

	for val := range ch {
		switch val := val.(type) {
		case *pb.Statement:
			gc.addKeys(val, keys)
			if len(keys) >= batch {
				err := gc.mergeKeys(keys)
				if err != nil {
					return err
				}
				keys = make(map[string]bool)
			}

		case StreamError:
			return val

		default:
			return BadResult
		}
	}

	if len(keys) > 0 {
		return gc.mergeKeys(keys)
	}

	return nil
}

func (gc *GCDB) addKeys(stmt *pb.Statement, keys map[string]bool) error {
	switch body := stmt.Body.Body.(type) {
	case *pb.StatementBody_Simple:
		gc.addSimpleKeys(body.Simple, keys)
		return nil

	case *pb.StatementBody_Compound:
		ss := body.Compound.Body
		for _, s := range ss {
			gc.addSimpleKeys(s, keys)
		}
		return nil

	case *pb.StatementBody_Envelope:
		stmts := body.Envelope.Body
		for _, stmt := range stmts {
			err := gc.addKeys(stmt, keys)
			if err != nil {
				return err
			}
		}
		return nil

	default:
		return BadStatementBody
	}
}

func (gc *GCDB) addSimpleKeys(s *pb.SimpleStatement, keys map[string]bool) {
	keys[s.Object] = true
	for _, dep := range s.Deps {
		keys[dep] = true
	}
}

func (gc *GCDB) mergeKeys(keys map[string]bool) error {
	tx, err := gc.db.Begin()
	if err != nil {
		return err
	}

	insertKey := tx.Stmt(gc.insertKey)

	for key, _ := range keys {
		_, err := insertKey.Exec(key)
		if err != nil {
			xerr, ok := err.(sqlite3.Error)
			if ok && xerr.ExtendedCode == sqlite3.ErrConstraintPrimaryKey {
				continue
			}
			tx.Rollback()
			return err
		}
	}

	return tx.Commit()
}

func (gc *GCDB) GC(ctx context.Context, ds Datastore) (count int, err error) {
	keys, err := ds.IterKeys(ctx)
	if err != nil {
		return
	}

	for key := range keys {
		var valid bool
		valid, err = gc.validKey(key)
		if err != nil {
			return
		}

		if valid {
			continue
		}

		err = ds.Delete(key)
		if err != nil {
			return
		}
		count += 1
	}

	if count > 0 {
		ds.Compact()
	}

	return
}

func (gc *GCDB) validKey(key Key) (bool, error) {
	key58 := multihash.Multihash(key).B58String()
	row := gc.countKeys.QueryRow(key58)

	var count int
	err := row.Scan(&count)
	if err != nil {
		return false, err
	}

	return count > 0, nil
}
<commit_msg>mcnode/gc: more accurate todo comment<commit_after>package main

import (
	"context"
	"database/sql"
	"errors"
	sqlite3 "github.com/mattn/go-sqlite3"
	mcq "github.com/mediachain/concat/mc/query"
	pb "github.com/mediachain/concat/proto"
	multihash "github.com/multiformats/go-multihash"
)

var (
	NodeMustBeOffline = errors.New("Node must be offline")
)

func (node *Node) doGC(ctx context.Context) (int, error) {
	if node.status != StatusOffline {
		return 0, NodeMustBeOffline
	}

	gc := &GCDB{}
	err := gc.Open(node.home)
	if err != nil {
		return 0, err
	}
	defer gc.Close()

	err = gc.Merge(ctx, node.db)
	if err != nil {
		return 0, err
	}

	return gc.GC(ctx, node.ds)
}

func (node *Node) doCompact() error {
	if node.status != StatusOffline {
		return NodeMustBeOffline
	}

	node.ds.Compact()
	return nil
}

type GCDB struct {
	db        *sql.DB
	insertKey *sql.Stmt
	countKeys *sql.Stmt
}

func (gc *GCDB) Open(home string) error {
	// TODO on-disk index for large datasets
	const dbpath = ":memory:"
	db, err := sql.Open("sqlite3", dbpath)
	if err != nil {
		return err
	}
	gc.db = db

	_, err = db.Exec("CREATE TABLE Refs (key VARCHAR(64) PRIMARY KEY)")
	if err != nil {
		return err
	}

	insertKey, err := db.Prepare("INSERT INTO Refs VALUES (?)")
	if err != nil {
		return err
	}
	gc.insertKey = insertKey

	countKeys, err := db.Prepare("SELECT COUNT(1) FROM Refs WHERE key = ?")
	if err != nil {
		return err
	}
	gc.countKeys = countKeys

	return nil
}

func (gc *GCDB) Close() error {
	// TODO clear on-disk index
	return gc.db.Close()
}

func (gc *GCDB) Merge(ctx context.Context, db StatementDB) error {
	q, err := mcq.ParseQuery("SELECT * FROM *")
	if err != nil {
		return err
	}

	ch, err := db.QueryStream(ctx, q)
	if err != nil {
		return err
	}

	const batch = 1024
	keys := make(map[string]bool)

	for val := range ch {
		switch val := val.(type) {
		case *pb.Statement:
			gc.addKeys(val, keys)
			if len(keys) >= batch {
				err := gc.mergeKeys(keys)
				if err != nil {
					return err
				}
				keys = make(map[string]bool)
			}

		case StreamError:
			return val

		default:
			return BadResult
		}
	}

	if len(keys) > 0 {
		return gc.mergeKeys(keys)
	}

	return nil
}

func (gc *GCDB) addKeys(stmt *pb.Statement, keys map[string]bool) error {
	switch body := stmt.Body.Body.(type) {
	case *pb.StatementBody_Simple:
		gc.addSimpleKeys(body.Simple, keys)
		return nil

	case *pb.StatementBody_Compound:
		ss := body.Compound.Body
		for _, s := range ss {
			gc.addSimpleKeys(s, keys)
		}
		return nil

	case *pb.StatementBody_Envelope:
		stmts := body.Envelope.Body
		for _, stmt := range stmts {
			err := gc.addKeys(stmt, keys)
			if err != nil {
				return err
			}
		}
		return nil

	default:
		return BadStatementBody
	}
}

func (gc *GCDB) addSimpleKeys(s *pb.SimpleStatement, keys map[string]bool) {
	keys[s.Object] = true
	for _, dep := range s.Deps {
		keys[dep] = true
	}
}

func (gc *GCDB) mergeKeys(keys map[string]bool) error {
	tx, err := gc.db.Begin()
	if err != nil {
		return err
	}

	insertKey := tx.Stmt(gc.insertKey)

	for key, _ := range keys {
		_, err := insertKey.Exec(key)
		if err != nil {
			xerr, ok := err.(sqlite3.Error)
			if ok && xerr.ExtendedCode == sqlite3.ErrConstraintPrimaryKey {
				continue
			}
			tx.Rollback()
			return err
		}
	}

	return tx.Commit()
}

func (gc *GCDB) GC(ctx context.Context, ds Datastore) (count int, err error) {
	keys, err := ds.IterKeys(ctx)
	if err != nil {
		return
	}

	for key := range keys {
		var valid bool
		valid, err = gc.validKey(key)
		if err != nil {
			return
		}

		if valid {
			continue
		}

		err = ds.Delete(key)
		if err != nil {
			return
		}
		count += 1
	}

	if count > 0 {
		ds.Compact()
	}

	return
}

func (gc *GCDB) validKey(key Key) (bool, error) {
	key58 := multihash.Multihash(key).B58String()
	row := gc.countKeys.QueryRow(key58)

	var count int
	err := row.Scan(&count)
	if err != nil {
		return false, err
	}

	return count > 0, nil
}
<|endoftext|>


<commit_before>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "0.9.10"
<commit_msg>Tag release v0.9.11<commit_after>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "0.9.11"
<|endoftext|>


<commit_before>// Copyright (c) 2010, Andrei Vieru. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"compress/lzma"
	"flag"
	"fmt"
	"log"
	"io"
	"os"
	"path"
	"runtime"
	"strings"
)

var (
	stdout     = flag.Bool("c", false, "write on standard output, keep original files unchanged")
	decompress = flag.Bool("d", false, "decompress")
	force      = flag.Bool("f", false, "force overwrite of output file")
	help       = flag.Bool("h", false, "print this help message")
	keep       = flag.Bool("k", false, "keep original files unchaned")
	suffix     = flag.String("s", "lzma", "use provided suffix on compressed files")
	level      = flag.Int("l", 5, "compression level [1 ... 9]")
	cores      = flag.Int("cores", 1, "number of cores to use for parallelization")

	stdin bool
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... [FILE]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "Compress or uncompress FILE (by default, compress FILE in-place).\n\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\nWith no FILE, or when FILE is -, read standard input.\n")
}

func exit(msg string) {
	usage()
	fmt.Fprintln(os.Stderr)
	log.Exitf("%s: check args: %s\n\n", os.Args[0], msg)
}

func setByUser(name string) (isSet bool) {
	flag.Visit(func(f *flag.Flag) {
		if f.Name == name {
			isSet = true
		}
	})
	return
}

func main() {
	flag.Parse()
	if *help == true {
		usage()
		os.Exit(0)
	}
	//if *stdout == true && *suffix != "lzma" {
	if *stdout == true && setByUser("s") == true {
		exit("stdout set, suffix not used")
	}
	if *stdout == true && *force == true {
		exit("stdout set, force not used")
	}
	if *stdout == true && *keep == true {
		exit("stdout set, keep is redundant")
	}
	if flag.NArg() > 1 {
		exit("too many file, provide at most one file at a time or check order of flags")
	}
	//if *decompress == true && *level != 5 {
	if *decompress == true && setByUser("l") == true {
		exit("compression level is used for compression only")
	}
	if *level < 1 || *level > 9 {
		exit("compression level out of range")
	}
	if *cores < 1 || *cores > 32 {
		exit("invalid number of cores")
	}

	runtime.GOMAXPROCS(*cores)

	var inFilePath string
	var outFilePath string
	if flag.NArg() == 0 || flag.NArg() == 1 && flag.Args()[0] == "-" { // parse args: read from stdin
		if *stdout != true {
			exit("reading from stdin, can write only to stdout")
		}
		//if *suffix != "lzma" {
		if setByUser("s") == true {
			exit("reading from stdin, suffix not needed")
		}
		stdin = true

	} else if flag.NArg() == 1 { // parse args: read from file
		inFilePath = flag.Args()[0]
		f, err := os.Lstat(inFilePath)
		if err != nil {
			log.Exit(err.String())
		}
		if f == nil {
			exit(fmt.Sprintf("file %s not found", inFilePath))
		}
		if !f.IsRegular() {
			exit(fmt.Sprintf("%s is not a regular file", inFilePath))
		}

		if *stdout == false { // parse args: write to file
			if *suffix == "" {
				exit("suffix can't be an empty string")
			}

			if *decompress == true {
				outFileDir, outFileName := path.Split(inFilePath)
				if strings.HasSuffix(outFileName, "."+*suffix) {
					if len(outFileName) > len("."+*suffix) {
						nstr := strings.Split(outFileName, ".", len(outFileName))
						estr := strings.Join(nstr[0:len(nstr)-1], ".")
						outFilePath = outFileDir + estr
					} else {
						log.Exitf("error: can't strip suffix .%s from file %s", *suffix, inFilePath)
					}
				} else {
					exit(fmt.Sprintf("file %s doesn't have suffix .%s", inFilePath, *suffix))
				}

			} else {
				outFilePath = inFilePath + "." + *suffix
			}

			f, err = os.Lstat(outFilePath)
			if err != nil && f != nil { // should be: ||| if err != nil && err != "file not found" ||| but i can't find the error's id
				log.Exit(err.String())
			}
			if f != nil && f.IsRegular() {
				if *force == true {
					err = os.Remove(outFilePath)
					if err != nil {
						log.Exit(err.String())
					}
				} else {
					exit(fmt.Sprintf("outFile %s exists. use force to overwrite", outFilePath))
				}
			} else if f != nil {
				exit(fmt.Sprintf("outFile %s exists and is not a regular file", outFilePath))
			}
		}
	}

	pr, pw := io.Pipe()
	//defer pr.Close()
	//defer pw.Close()

	if *decompress {
		// read from inFile into pw
		go func() {
			defer pw.Close()
			var inFile *os.File
			var err os.Error
			if stdin == true {
				inFile = os.Stdin
			} else {
				inFile, err = os.Open(inFilePath, os.O_RDONLY, 0400)
			}
			defer inFile.Close()
			if err != nil {
				log.Exit(err.String())
			}

			_, err = io.Copy(pw, inFile)
			if err != nil {
				log.Exit(err.String())
			}

		}()

		// write into outFile from z
		defer pr.Close()
		z := lzma.NewDecoder(pr)
		defer z.Close()
		var outFile *os.File
		var err os.Error
		if *stdout == true {
			outFile = os.Stdout
		} else {
			outFile, err = os.Open(outFilePath, os.O_CREAT|os.O_RDWR, 0644)
		}
		defer outFile.Close()
		if err != nil {
			log.Exit(err.String())
		}

		_, err = io.Copy(outFile, z)
		if err != nil {
			log.Exit(err.String())
		}

	} else {
		// read from inFile into z
		go func() {
			defer pw.Close()
			var z io.WriteCloser
			var inFile *os.File
			var err os.Error
			if stdin == true {
				inFile = os.Stdin
				defer inFile.Close()
				z = lzma.NewEncoderLevel(pw, *level)
				defer z.Close()
			} else {
				inFile, err = os.Open(inFilePath, os.O_RDONLY, 0400)
				defer inFile.Close()
				if err != nil {
					log.Exit(err.String())
				}
				f, err := os.Lstat(inFilePath)
				if err != nil {
					log.Exit(err.String())
				}
				z = lzma.NewEncoderSizeLevel(pw, int64(f.Size), *level)
				defer z.Close()
			}

			_, err = io.Copy(z, inFile)
			if err != nil {
				log.Exit(err.String())
			}
		}()

		// write into outFile from pr
		defer pr.Close()
		var outFile *os.File
		var err os.Error
		if *stdout == true {
			outFile = os.Stdout
		} else {
			outFile, err = os.Open(outFilePath, os.O_CREAT|os.O_RDWR, 0644)
		}
		defer outFile.Close()
		if err != nil {
			log.Exit(err.String())
		}

		_, err = io.Copy(outFile, pr)
		if err != nil {
			log.Exit(err.String())
		}
	}

	if *stdout == false && *keep == false {
		err := os.Remove(inFilePath)
		if err != nil {
			log.Exit(err.String())
		}
	}

}
<commit_msg>googlecode import added<commit_after>// Copyright (c) 2010, Andrei Vieru. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"compress/lzma"
	//"lzma.googlecode.com/hg/compress/lzma"
	"flag"
	"fmt"
	"log"
	"io"
	"os"
	"path"
	"runtime"
	"strings"
)

var (
	stdout     = flag.Bool("c", false, "write on standard output, keep original files unchanged")
	decompress = flag.Bool("d", false, "decompress")
	force      = flag.Bool("f", false, "force overwrite of output file")
	help       = flag.Bool("h", false, "print this help message")
	keep       = flag.Bool("k", false, "keep original files unchaned")
	suffix     = flag.String("s", "lzma", "use provided suffix on compressed files")
	level      = flag.Int("l", 5, "compression level [1 ... 9]")
	cores      = flag.Int("cores", 1, "number of cores to use for parallelization")

	stdin bool
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... [FILE]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "Compress or uncompress FILE (by default, compress FILE in-place).\n\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\nWith no FILE, or when FILE is -, read standard input.\n")
}

func exit(msg string) {
	usage()
	fmt.Fprintln(os.Stderr)
	log.Exitf("%s: check args: %s\n\n", os.Args[0], msg)
}

func setByUser(name string) (isSet bool) {
	flag.Visit(func(f *flag.Flag) {
		if f.Name == name {
			isSet = true
		}
	})
	return
}

func main() {
	flag.Parse()
	if *help == true {
		usage()
		os.Exit(0)
	}
	//if *stdout == true && *suffix != "lzma" {
	if *stdout == true && setByUser("s") == true {
		exit("stdout set, suffix not used")
	}
	if *stdout == true && *force == true {
		exit("stdout set, force not used")
	}
	if *stdout == true && *keep == true {
		exit("stdout set, keep is redundant")
	}
	if flag.NArg() > 1 {
		exit("too many file, provide at most one file at a time or check order of flags")
	}
	//if *decompress == true && *level != 5 {
	if *decompress == true && setByUser("l") == true {
		exit("compression level is used for compression only")
	}
	if *level < 1 || *level > 9 {
		exit("compression level out of range")
	}
	if *cores < 1 || *cores > 32 {
		exit("invalid number of cores")
	}

	runtime.GOMAXPROCS(*cores)

	var inFilePath string
	var outFilePath string
	if flag.NArg() == 0 || flag.NArg() == 1 && flag.Args()[0] == "-" { // parse args: read from stdin
		if *stdout != true {
			exit("reading from stdin, can write only to stdout")
		}
		//if *suffix != "lzma" {
		if setByUser("s") == true {
			exit("reading from stdin, suffix not needed")
		}
		stdin = true

	} else if flag.NArg() == 1 { // parse args: read from file
		inFilePath = flag.Args()[0]
		f, err := os.Lstat(inFilePath)
		if err != nil {
			log.Exit(err.String())
		}
		if f == nil {
			exit(fmt.Sprintf("file %s not found", inFilePath))
		}
		if !f.IsRegular() {
			exit(fmt.Sprintf("%s is not a regular file", inFilePath))
		}

		if *stdout == false { // parse args: write to file
			if *suffix == "" {
				exit("suffix can't be an empty string")
			}

			if *decompress == true {
				outFileDir, outFileName := path.Split(inFilePath)
				if strings.HasSuffix(outFileName, "."+*suffix) {
					if len(outFileName) > len("."+*suffix) {
						nstr := strings.Split(outFileName, ".", len(outFileName))
						estr := strings.Join(nstr[0:len(nstr)-1], ".")
						outFilePath = outFileDir + estr
					} else {
						log.Exitf("error: can't strip suffix .%s from file %s", *suffix, inFilePath)
					}
				} else {
					exit(fmt.Sprintf("file %s doesn't have suffix .%s", inFilePath, *suffix))
				}

			} else {
				outFilePath = inFilePath + "." + *suffix
			}

			f, err = os.Lstat(outFilePath)
			if err != nil && f != nil { // should be: ||| if err != nil && err != "file not found" ||| but i can't find the error's id
				log.Exit(err.String())
			}
			if f != nil && f.IsRegular() {
				if *force == true {
					err = os.Remove(outFilePath)
					if err != nil {
						log.Exit(err.String())
					}
				} else {
					exit(fmt.Sprintf("outFile %s exists. use force to overwrite", outFilePath))
				}
			} else if f != nil {
				exit(fmt.Sprintf("outFile %s exists and is not a regular file", outFilePath))
			}
		}
	}

	pr, pw := io.Pipe()
	//defer pr.Close()
	//defer pw.Close()

	if *decompress {
		// read from inFile into pw
		go func() {
			defer pw.Close()
			var inFile *os.File
			var err os.Error
			if stdin == true {
				inFile = os.Stdin
			} else {
				inFile, err = os.Open(inFilePath, os.O_RDONLY, 0400)
			}
			defer inFile.Close()
			if err != nil {
				log.Exit(err.String())
			}

			_, err = io.Copy(pw, inFile)
			if err != nil {
				log.Exit(err.String())
			}

		}()

		// write into outFile from z
		defer pr.Close()
		z := lzma.NewDecoder(pr)
		defer z.Close()
		var outFile *os.File
		var err os.Error
		if *stdout == true {
			outFile = os.Stdout
		} else {
			outFile, err = os.Open(outFilePath, os.O_CREAT|os.O_RDWR, 0644)
		}
		defer outFile.Close()
		if err != nil {
			log.Exit(err.String())
		}

		_, err = io.Copy(outFile, z)
		if err != nil {
			log.Exit(err.String())
		}

	} else {
		// read from inFile into z
		go func() {
			defer pw.Close()
			var z io.WriteCloser
			var inFile *os.File
			var err os.Error
			if stdin == true {
				inFile = os.Stdin
				defer inFile.Close()
				z = lzma.NewEncoderLevel(pw, *level)
				defer z.Close()
			} else {
				inFile, err = os.Open(inFilePath, os.O_RDONLY, 0400)
				defer inFile.Close()
				if err != nil {
					log.Exit(err.String())
				}
				f, err := os.Lstat(inFilePath)
				if err != nil {
					log.Exit(err.String())
				}
				z = lzma.NewEncoderSizeLevel(pw, int64(f.Size), *level)
				defer z.Close()
			}

			_, err = io.Copy(z, inFile)
			if err != nil {
				log.Exit(err.String())
			}
		}()

		// write into outFile from pr
		defer pr.Close()
		var outFile *os.File
		var err os.Error
		if *stdout == true {
			outFile = os.Stdout
		} else {
			outFile, err = os.Open(outFilePath, os.O_CREAT|os.O_RDWR, 0644)
		}
		defer outFile.Close()
		if err != nil {
			log.Exit(err.String())
		}

		_, err = io.Copy(outFile, pr)
		if err != nil {
			log.Exit(err.String())
		}
	}

	if *stdout == false && *keep == false {
		err := os.Remove(inFilePath)
		if err != nil {
			log.Exit(err.String())
		}
	}

}
<|endoftext|>


<commit_before>package connection

import (
	"encoding/hex"
	"log"
	"net"

	"github.com/piot/hasty-protocol/channel"
	"github.com/piot/hasty-protocol/chunk"
	"github.com/piot/hasty-protocol/commands"
	"github.com/piot/hasty-protocol/packet"
	"github.com/piot/hasty-protocol/packetserializers"
	"github.com/piot/hasty-protocol/serializer"
	"github.com/piot/hasty-protocol/timestamp"
	"github.com/piot/hasty-server/authorization"
	"github.com/piot/hasty-server/master"
	"github.com/piot/hasty-server/storage"
	"github.com/piot/hasty-server/subscribers"
)

type StreamInfo struct {
	lastOffsetSent uint64
}

type ConnectionHandler struct {
	conn          *net.Conn
	storage       *filestorage.StreamStorage
	subscribers   *subscribers.Subscribers
	streamInfos   map[uint32]*StreamInfo
	connectionID  packet.ConnectionID
	masterHandler *master.MasterCommandHandler
	chunkStreams  map[uint32]*chunk.Stream
}

// NewConnectionHandler : todo
func NewConnectionHandler(connection *net.Conn, masterHandler *master.MasterCommandHandler, storage *filestorage.StreamStorage, subs *subscribers.Subscribers, connectionID packet.ConnectionID) *ConnectionHandler {
	return &ConnectionHandler{connectionID: connectionID, masterHandler: masterHandler, conn: connection, storage: storage, subscribers: subs, streamInfos: map[uint32]*StreamInfo{}, chunkStreams: map[uint32]*chunk.Stream{}}
}

// HandleConnect : todo
func (in *ConnectionHandler) HandleConnect(cmd commands.Connect) error {
	log.Printf("%s %s", in.connectionID, cmd)

	// _ := commands.NewConnectResult(cmd.Realm(), cmd.ProtocolVersion())
	octetsToSend := packetserializers.ConnectResultToOctets()
	in.sendPacket(octetsToSend)
	return nil
}

func (in *ConnectionHandler) sendPong(echoedTime timestamp.Time) {
	log.Printf("%s sendPong %s", in.connectionID, echoedTime)
	now := timestamp.Now()
	octetsToSend := packetserializers.PongToOctets(now, echoedTime)
	in.sendPacket(octetsToSend)
}

func (in *ConnectionHandler) sendLoginResult(worked bool) {
	log.Printf("%s sendLoginResult %t", in.connectionID, worked)
	octetsToSend := packetserializers.LoginResultToOctets()
	in.sendPacket(octetsToSend)
}

// HandlePing : todo
func (in *ConnectionHandler) HandlePing(cmd commands.Ping) {
	log.Printf("%s %s", in.connectionID, cmd)
	in.sendPong(cmd.SentTime())
}

// HandlePong : todo
func (in *ConnectionHandler) HandlePong(cmd commands.Pong) {
	now := timestamp.Now()
	latency := now.Raw() - cmd.EchoedTime().Raw()
	log.Printf("%s Latency: %d ms", in.connectionID, latency)
}

// HandlePublishStream : todo
func (in *ConnectionHandler) HandlePublishStream(cmd commands.PublishStream) error {
	log.Printf("%s %s", in.connectionID, cmd)
	return nil
}

func (in *ConnectionHandler) StreamChanged(channelID channel.ID) {
	info := in.streamInfos[channelID.Raw()]
	file, fileErr := in.storage.ReadStream(channelID)
	if fileErr != nil {
		return
	}
	file.Seek(info.lastOffsetSent)
	data := make([]byte, 32*1024)
	octetsRead, readErr := file.Read(data)
	if readErr != nil {
		return
	}
	file.Close()
	in.sendStreamData(channelID, uint32(info.lastOffsetSent), data[:octetsRead])
	info.lastOffsetSent += uint64(octetsRead)
}

func (in *ConnectionHandler) sendStreamData(channelID channel.ID, lastOffsetSent uint32, data []byte) {
	log.Printf("%s sendStreamData %s offset:%d", in.connectionID, channelID, lastOffsetSent)
	payload := packetserializers.StreamDataToOctets(channelID, lastOffsetSent, data)
	in.sendPacket(payload)
}

func (in *ConnectionHandler) fetchOrCreateStreamInfo(channelID channel.ID) *StreamInfo {
	infos := in.streamInfos[channelID.Raw()]
	if infos == nil {
		infos = &StreamInfo{}
		in.streamInfos[channelID.Raw()] = infos
	}
	return infos
}

// HandleSubscribeStream : todo
func (in *ConnectionHandler) HandleSubscribeStream(cmd commands.SubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)

	for _, v := range cmd.Infos() {
		readFile, err := in.storage.ReadStream(v.Channel())

		if err != nil {
			return
		}

		buf := make([]byte, 64*1024)
		octetsRead, readErr := readFile.Read(buf)
		if readErr != nil {
			return
		}
		data := buf[:octetsRead]

		octetsToSend := packetserializers.StreamDataToOctets(v.Channel(), 0, data)
		in.sendPacket(octetsToSend)
		infos := in.fetchOrCreateStreamInfo(v.Channel())
		infos.lastOffsetSent = uint64(octetsRead)
		in.subscribers.AddStreamSubscriber(v.Channel(), in)
	}
}

// HandleUnsubscribeStream : todo
func (in *ConnectionHandler) HandleUnsubscribeStream(cmd commands.UnsubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)
}

// HandleCreateStream : todo
func (in *ConnectionHandler) HandleCreateStream(cmd commands.CreateStream) (channel.ID, error) {
	log.Printf("%s %s", in.connectionID, cmd)
	return channel.ID{}, nil
}

func (in *ConnectionHandler) fetchOrAssoicateChunkStream(channelID channel.ID) *chunk.Stream {
	stream := in.chunkStreams[channelID.Raw()]
	if stream == nil {
		stream = &chunk.Stream{}
		in.chunkStreams[channelID.Raw()] = stream
	}
	return stream
}

func (in *ConnectionHandler) publishMasterStream(channel channel.ID, payload []byte) {
	fakeClient := authorization.AdminClient{}
	cmd := commands.NewPublishStream(channel, payload)
	in.masterHandler.HandlePublishStream(fakeClient, cmd)
}

// HandleStreamData : todo
func (in *ConnectionHandler) HandleStreamData(cmd commands.StreamData) {
	log.Printf("%s %s", in.connectionID, cmd)
	chunkStream := in.fetchOrAssoicateChunkStream(cmd.Channel())
	chunkStream.Feed(cmd.Data())
	foundChunk, fetchErr := chunkStream.FetchChunk()
	if fetchErr != nil {
		_, isNotDoneError := fetchErr.(*chunk.NotDoneError)
		if isNotDoneError {
		} else {
			log.Printf("Fetcherror:%s", fetchErr)
		}
	} else {
		in.publishMasterStream(cmd.Channel(), foundChunk.Payload())
	}
}

// HandleLogin : todo
func (in *ConnectionHandler) HandleLogin(cmd commands.Login) error {
	log.Printf("%s", cmd)
	in.sendLoginResult(true)
	return nil
}

func (in *ConnectionHandler) sendPacket(octets []byte) {
	payloadLength := uint16(len(octets))
	hexPayload := hex.Dump(octets)
	lengthBuf, lengthErr := serializer.SmallLengthToOctets(payloadLength)
	if lengthErr != nil {
		log.Printf("We couldn't write length")
		return
	}
	log.Printf("%s Sending packet (size %d) %s", in.connectionID, payloadLength, hexPayload)
	(*in.conn).Write(lengthBuf)
	(*in.conn).Write(octets)
}

// HandleTransportDisconnect : todo
func (in *ConnectionHandler) HandleTransportDisconnect() {
	log.Printf("%s Transport disconnect", in.connectionID)
}
<commit_msg>Creating chunk streams<commit_after>package connection

import (
	"encoding/hex"
	"log"
	"net"

	"github.com/piot/hasty-protocol/channel"
	"github.com/piot/hasty-protocol/chunk"
	"github.com/piot/hasty-protocol/commands"
	"github.com/piot/hasty-protocol/packet"
	"github.com/piot/hasty-protocol/packetserializers"
	"github.com/piot/hasty-protocol/serializer"
	"github.com/piot/hasty-protocol/timestamp"
	"github.com/piot/hasty-server/authorization"
	"github.com/piot/hasty-server/master"
	"github.com/piot/hasty-server/storage"
	"github.com/piot/hasty-server/subscribers"
)

type StreamInfo struct {
	lastOffsetSent uint64
}

type ConnectionHandler struct {
	conn          *net.Conn
	storage       *filestorage.StreamStorage
	subscribers   *subscribers.Subscribers
	streamInfos   map[uint32]*StreamInfo
	connectionID  packet.ConnectionID
	masterHandler *master.MasterCommandHandler
	chunkStreams  map[uint32]*chunk.Stream
}

// NewConnectionHandler : todo
func NewConnectionHandler(connection *net.Conn, masterHandler *master.MasterCommandHandler, storage *filestorage.StreamStorage, subs *subscribers.Subscribers, connectionID packet.ConnectionID) *ConnectionHandler {
	return &ConnectionHandler{connectionID: connectionID, masterHandler: masterHandler, conn: connection, storage: storage, subscribers: subs, streamInfos: map[uint32]*StreamInfo{}, chunkStreams: map[uint32]*chunk.Stream{}}
}

// HandleConnect : todo
func (in *ConnectionHandler) HandleConnect(cmd commands.Connect) error {
	log.Printf("%s %s", in.connectionID, cmd)

	// _ := commands.NewConnectResult(cmd.Realm(), cmd.ProtocolVersion())
	octetsToSend := packetserializers.ConnectResultToOctets()
	in.sendPacket(octetsToSend)
	return nil
}

func (in *ConnectionHandler) sendPong(echoedTime timestamp.Time) {
	log.Printf("%s sendPong %s", in.connectionID, echoedTime)
	now := timestamp.Now()
	octetsToSend := packetserializers.PongToOctets(now, echoedTime)
	in.sendPacket(octetsToSend)
}

func (in *ConnectionHandler) sendLoginResult(worked bool) {
	log.Printf("%s sendLoginResult %t", in.connectionID, worked)
	octetsToSend := packetserializers.LoginResultToOctets()
	in.sendPacket(octetsToSend)
}

// HandlePing : todo
func (in *ConnectionHandler) HandlePing(cmd commands.Ping) {
	log.Printf("%s %s", in.connectionID, cmd)
	in.sendPong(cmd.SentTime())
}

// HandlePong : todo
func (in *ConnectionHandler) HandlePong(cmd commands.Pong) {
	now := timestamp.Now()
	latency := now.Raw() - cmd.EchoedTime().Raw()
	log.Printf("%s Latency: %d ms", in.connectionID, latency)
}

// HandlePublishStream : todo
func (in *ConnectionHandler) HandlePublishStream(cmd commands.PublishStream) error {
	log.Printf("%s %s", in.connectionID, cmd)
	return nil
}

func (in *ConnectionHandler) StreamChanged(channelID channel.ID) {
	info := in.streamInfos[channelID.Raw()]
	file, fileErr := in.storage.ReadStream(channelID)
	if fileErr != nil {
		return
	}
	file.Seek(info.lastOffsetSent)
	data := make([]byte, 32*1024)
	octetsRead, readErr := file.Read(data)
	if readErr != nil {
		return
	}
	file.Close()
	in.sendStreamData(channelID, uint32(info.lastOffsetSent), data[:octetsRead])
	info.lastOffsetSent += uint64(octetsRead)
}

func (in *ConnectionHandler) sendStreamData(channelID channel.ID, lastOffsetSent uint32, data []byte) {
	log.Printf("%s sendStreamData %s offset:%d", in.connectionID, channelID, lastOffsetSent)
	payload := packetserializers.StreamDataToOctets(channelID, lastOffsetSent, data)
	in.sendPacket(payload)
}

func (in *ConnectionHandler) fetchOrCreateStreamInfo(channelID channel.ID) *StreamInfo {
	infos := in.streamInfos[channelID.Raw()]
	if infos == nil {
		infos = &StreamInfo{}
		in.streamInfos[channelID.Raw()] = infos
	}
	return infos
}

// HandleSubscribeStream : todo
func (in *ConnectionHandler) HandleSubscribeStream(cmd commands.SubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)

	for _, v := range cmd.Infos() {
		readFile, err := in.storage.ReadStream(v.Channel())

		if err != nil {
			return
		}

		buf := make([]byte, 64*1024)
		octetsRead, readErr := readFile.Read(buf)
		if readErr != nil {
			return
		}
		data := buf[:octetsRead]

		octetsToSend := packetserializers.StreamDataToOctets(v.Channel(), 0, data)
		in.sendPacket(octetsToSend)
		infos := in.fetchOrCreateStreamInfo(v.Channel())
		infos.lastOffsetSent = uint64(octetsRead)
		in.subscribers.AddStreamSubscriber(v.Channel(), in)
	}
}

// HandleUnsubscribeStream : todo
func (in *ConnectionHandler) HandleUnsubscribeStream(cmd commands.UnsubscribeStream) {
	log.Printf("%s %s", in.connectionID, cmd)
}

// HandleCreateStream : todo
func (in *ConnectionHandler) HandleCreateStream(cmd commands.CreateStream) (channel.ID, error) {
	log.Printf("%s %s", in.connectionID, cmd)
	return channel.ID{}, nil
}

func (in *ConnectionHandler) fetchOrAssoicateChunkStream(channelID channel.ID) *chunk.Stream {
	stream := in.chunkStreams[channelID.Raw()]
	if stream == nil {
		stream = chunk.NewChunkStream(in.connectionID, channelID)
		in.chunkStreams[channelID.Raw()] = stream
	}
	return stream
}

func (in *ConnectionHandler) publishMasterStream(channel channel.ID, payload []byte) {
	fakeClient := authorization.AdminClient{}
	cmd := commands.NewPublishStream(channel, payload)
	in.masterHandler.HandlePublishStream(fakeClient, cmd)
}

// HandleStreamData : todo
func (in *ConnectionHandler) HandleStreamData(cmd commands.StreamData) {
	log.Printf("%s %s", in.connectionID, cmd)
	chunkStream := in.fetchOrAssoicateChunkStream(cmd.Channel())
	chunkStream.Feed(cmd.Data())
	foundChunk, fetchErr := chunkStream.FetchChunk()
	if fetchErr != nil {
		_, isNotDoneError := fetchErr.(*chunk.NotDoneError)
		if isNotDoneError {
		} else {
			log.Printf("Fetcherror:%s", fetchErr)
		}
	} else {
		in.publishMasterStream(cmd.Channel(), foundChunk.Payload())
	}
}

// HandleLogin : todo
func (in *ConnectionHandler) HandleLogin(cmd commands.Login) error {
	log.Printf("%s", cmd)
	in.sendLoginResult(true)
	return nil
}

func (in *ConnectionHandler) sendPacket(octets []byte) {
	payloadLength := uint16(len(octets))
	hexPayload := hex.Dump(octets)
	lengthBuf, lengthErr := serializer.SmallLengthToOctets(payloadLength)
	if lengthErr != nil {
		log.Printf("We couldn't write length")
		return
	}
	log.Printf("%s Sending packet (size %d) %s", in.connectionID, payloadLength, hexPayload)
	(*in.conn).Write(lengthBuf)
	(*in.conn).Write(octets)
}

// HandleTransportDisconnect : todo
func (in *ConnectionHandler) HandleTransportDisconnect() {
	log.Printf("%s Transport disconnect", in.connectionID)
}
<|endoftext|>