<commit_before>package box

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
)

// TODO(ttacon): reconcile this with Folder for one common struct?
type File struct {
	ID                string          `json:"id,omitempty"`
	FolderUploadEmail *AccessEmail    `json:"folder_upload_email,omitempty"`
	Parent            *Item           `json:"parent,omitempty"`
	ItemStatus        string          `json:"item_status"`
	ItemCollection    *ItemCollection `json:"item_collection"`
	Type              string          `json:"type"` // TODO(ttacon): enum
	Description       string          `json:"description"`
	Size              int             `json:"size"`
	CreateBy          *Item           `json:"created_by"`
	ModifiedBy        *Item           `json:"modified_by"`
	TrashedAt         *string         `json:"trashed_at"`          // TODO(ttacon): change to time.Time
	ContentModifiedAt *string         `json:"content_modified_at"` // TODO(ttacon): change to time.Time
	PurgedAt          *string         `json:"purged_at"`           // TODO(ttacon): change to time.Time, this field isn't documented but I keep getting it back...
	SharedLinkg       *string         `json:"shared_link"`
	SequenceId        string          `json:"sequence_id"`
	ETag              *string         `json:"etag"`
	Name              string          `json:"name"`
	CreatedAt         *string         `json:"created_at"` // TODO(ttacon): change to time.Time
	OwnedBy           *Item           `json:"owned_by"`
	ModifiedAt        *string         `json:"modified_at"`        // TODO(ttacon): change to time.Time
	ContentCreatedAt  *string         `json:"content_created_at"` // TODO(ttacon): change to time.Time
	PathCollection    *PathCollection `json:"path_collection"`    // TODO(ttacon): make sure this is the correct kind of struct(ure)
	SharedLink        *Link           `json:"shared_link"`

	SHA1 string `json:"sha1"`
}

type FileCollection struct {
	TotalCount int     `json:"total_count"`
	Entries    []*File `json:"entries"`
}

// Documentation: https://developer.box.com/docs/#files-get
func (c *Client) GetFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation https://developer.box.com/docs/#files-upload-a-file
// TODO(ttacon): deal with handling SHA1 headers
func (c *Client) UploadFile(filePath, parentId string) (*http.Response, *FileCollection, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	fileContents, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, nil, err
	}

	fi, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	var (
		body   = &bytes.Buffer{}
		writer = multipart.NewWriter(body)
	)

	// write the file
	part, err := writer.CreateFormFile("file", fi.Name())
	if err != nil {
		return nil, nil, err
	}
	part.Write(fileContents)

	// write the other form fields we need
	err = writer.WriteField("filename", fi.Name())
	if err != nil {
		return nil, nil, err
	}

	err = writer.WriteField("parent_id", parentId)
	if err != nil {
		return nil, nil, err
	}

	// TODO(ttacon): add in content_created_at, content_modified_at

	err = writer.Close()
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("https://upload.box.com/api/2.0/files/content"),
		body,
	)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-delete-a-file
func (c *Client) DeleteFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"DELETE",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-copy-a-file
func (c *Client) CopyFile(fileId, parent, name string) (*http.Response, *File, error) {
	var bodyData = map[string]interface{}{
		"parent": map[string]string{
			"id": parent,
		},
		"name": name,
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("/files/%s/copy", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we return the http.Response as Box may return a 202 if there is not
// yet a download link, or a 302 with the link - this allows the user to
// decide what to do.
// Documentation: https://developers.box.com/docs/#files-download-a-file
func (c *Client) DownloadFile(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/content", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-view-versions-of-a-file
// TODO(ttacon): don't use file collection, make actual structs specific to file versions
func (c *Client) ViewVersionsOfFile(fileId string) (*http.Response, *FileCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/versions", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// NOTE: we only return the response as there are many possible responses that we
// feel the user should have control over
// Documentation: https://developers.box.com/docs/#files-get-a-thumbnail-for-a-file
func (c *Client) GetThumbnail(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/thumbnail.extension", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-create-a-shared-link-for-a-file
func (c *Client) CreateSharedLinkForFile(fileId, access, unsharedAt string, canDownload, canPreview bool) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(access) > 0 {
		dataMap["access"] = access
	}
	// TODO(ttacon): support unshared_at as time.Time
	// TODO(ttacon): validate access is open or company before add permissions
	if canDownload {
		dataMap["permissions"] = map[string]bool{
			"can_download": canDownload,
		}
	}
	if canPreview {
		if m, ok := dataMap["permissions"]; ok {
			mVal, _ := m.(map[string]bool)
			mVal["can_preview"] = canPreview
		} else {
			dataMap["permissions"] = map[string]bool{
				"can_preview": canPreview,
			}
		}
	}

	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"PUT",
		fmt.Sprintf("%s/files/%s", BASE_URL, fileId),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-a-trashed-file
func (c *Client) GetTrashedFile(fileId string) (*http.Response, *File, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-restore-a-trashed-item
func (c *Client) RestoreTrashedItem(name, parentId string) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(name) > 0 {
		dataMap["name"] = name
	}
	if len(parentId) > 0 {
		dataMap["parent"] = map[string]string{
			"id": parentId,
		}
	}
	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf(),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-permanently-delete-a-trashed-file
func (c *Client) PermanentlyDeleteTrashedFile(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"DELETE",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-view-the-comments-on-a-file
func (c *Client) ViewCommentsOnFile(fileId string) (*http.Response, *CommentCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/comments", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data CommentCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-the-tasks-for-a-file
func (c *Client) GetTasksForFile(fileId string) (*http.Response, *TaskCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/tasks", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data TaskCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}
<commit_msg>refactor download file<commit_after>package box

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
)

// TODO(ttacon): reconcile this with Folder for one common struct?
type File struct {
	ID                string          `json:"id,omitempty"`
	FolderUploadEmail *AccessEmail    `json:"folder_upload_email,omitempty"`
	Parent            *Item           `json:"parent,omitempty"`
	ItemStatus        string          `json:"item_status"`
	ItemCollection    *ItemCollection `json:"item_collection"`
	Type              string          `json:"type"` // TODO(ttacon): enum
	Description       string          `json:"description"`
	Size              int             `json:"size"`
	CreateBy          *Item           `json:"created_by"`
	ModifiedBy        *Item           `json:"modified_by"`
	TrashedAt         *string         `json:"trashed_at"`          // TODO(ttacon): change to time.Time
	ContentModifiedAt *string         `json:"content_modified_at"` // TODO(ttacon): change to time.Time
	PurgedAt          *string         `json:"purged_at"`           // TODO(ttacon): change to time.Time, this field isn't documented but I keep getting it back...
	SharedLinkg       *string         `json:"shared_link"`
	SequenceId        string          `json:"sequence_id"`
	ETag              *string         `json:"etag"`
	Name              string          `json:"name"`
	CreatedAt         *string         `json:"created_at"` // TODO(ttacon): change to time.Time
	OwnedBy           *Item           `json:"owned_by"`
	ModifiedAt        *string         `json:"modified_at"`        // TODO(ttacon): change to time.Time
	ContentCreatedAt  *string         `json:"content_created_at"` // TODO(ttacon): change to time.Time
	PathCollection    *PathCollection `json:"path_collection"`    // TODO(ttacon): make sure this is the correct kind of struct(ure)
	SharedLink        *Link           `json:"shared_link"`

	SHA1 string `json:"sha1"`
}

type FileCollection struct {
	TotalCount int     `json:"total_count"`
	Entries    []*File `json:"entries"`
}

// Documentation: https://developer.box.com/docs/#files-get
func (c *Client) GetFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation https://developer.box.com/docs/#files-upload-a-file
// TODO(ttacon): deal with handling SHA1 headers
func (c *Client) UploadFile(filePath, parentId string) (*http.Response, *FileCollection, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	fileContents, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, nil, err
	}

	fi, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	var (
		body   = &bytes.Buffer{}
		writer = multipart.NewWriter(body)
	)

	// write the file
	part, err := writer.CreateFormFile("file", fi.Name())
	if err != nil {
		return nil, nil, err
	}
	part.Write(fileContents)

	// write the other form fields we need
	err = writer.WriteField("filename", fi.Name())
	if err != nil {
		return nil, nil, err
	}

	err = writer.WriteField("parent_id", parentId)
	if err != nil {
		return nil, nil, err
	}

	// TODO(ttacon): add in content_created_at, content_modified_at

	err = writer.Close()
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("https://upload.box.com/api/2.0/files/content"),
		body,
	)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-delete-a-file
func (c *Client) DeleteFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"DELETE",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-copy-a-file
func (c *Client) CopyFile(fileId, parent, name string) (*http.Response, *File, error) {
	var bodyData = map[string]interface{}{
		"parent": map[string]string{
			"id": parent,
		},
		"name": name,
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("/files/%s/copy", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we return the http.Response as Box may return a 202 if there is not
// yet a download link, or a 302 with the link - this allows the user to
// decide what to do.
// Documentation: https://developers.box.com/docs/#files-download-a-file
func (c *Client) DownloadFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/content", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-view-versions-of-a-file
// TODO(ttacon): don't use file collection, make actual structs specific to file versions
func (c *Client) ViewVersionsOfFile(fileId string) (*http.Response, *FileCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/versions", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// NOTE: we only return the response as there are many possible responses that we
// feel the user should have control over
// Documentation: https://developers.box.com/docs/#files-get-a-thumbnail-for-a-file
func (c *Client) GetThumbnail(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/thumbnail.extension", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-create-a-shared-link-for-a-file
func (c *Client) CreateSharedLinkForFile(fileId, access, unsharedAt string, canDownload, canPreview bool) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(access) > 0 {
		dataMap["access"] = access
	}
	// TODO(ttacon): support unshared_at as time.Time
	// TODO(ttacon): validate access is open or company before add permissions
	if canDownload {
		dataMap["permissions"] = map[string]bool{
			"can_download": canDownload,
		}
	}
	if canPreview {
		if m, ok := dataMap["permissions"]; ok {
			mVal, _ := m.(map[string]bool)
			mVal["can_preview"] = canPreview
		} else {
			dataMap["permissions"] = map[string]bool{
				"can_preview": canPreview,
			}
		}
	}

	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"PUT",
		fmt.Sprintf("%s/files/%s", BASE_URL, fileId),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-a-trashed-file
func (c *Client) GetTrashedFile(fileId string) (*http.Response, *File, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-restore-a-trashed-item
func (c *Client) RestoreTrashedItem(name, parentId string) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(name) > 0 {
		dataMap["name"] = name
	}
	if len(parentId) > 0 {
		dataMap["parent"] = map[string]string{
			"id": parentId,
		}
	}
	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf(),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-permanently-delete-a-trashed-file
func (c *Client) PermanentlyDeleteTrashedFile(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"DELETE",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-view-the-comments-on-a-file
func (c *Client) ViewCommentsOnFile(fileId string) (*http.Response, *CommentCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/comments", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data CommentCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-the-tasks-for-a-file
func (c *Client) GetTasksForFile(fileId string) (*http.Response, *TaskCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/tasks", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data TaskCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"os"
	"time"

	mountpkg "github.com/googlecloudplatform/gcsfuse/mount"
	"github.com/jgeewax/cli"
)

func getApp() (app *cli.App) {
	app = &cli.App{
		Name:          "gcsfuse",
		Usage:         "Mount a GCS bucket locally",
		ArgumentUsage: "bucket mountpoint",
		HideHelp:      true,
		Version:       "0.1.0",
		Writer:        os.Stderr,
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:        "dir-mode, d",
				Usage:       "Permissions bits for directories. (default: 0755)",
				HideDefault: true,
			},
			cli.IntFlag{
				Name:        "file-mode, f",
				Value:       0644,
				Usage:       "Permission bits for files (default: 0644)",
				HideDefault: true,
			},
			cli.IntFlag{
				Name:  "gcs-chunk-size",
				Value: 1 << 24,
				Usage: "Max chunk size for loading GCS objects.",
			},
			cli.IntFlag{
				Name:        "gid",
				Value:       -1,
				HideDefault: true,
				Usage:       "GID owner of all inodes.",
			},
			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See" +
					"docs/semantics.md",
			},
			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},
			cli.StringFlag{
				Name:        "key-file",
				Value:       "",
				HideDefault: true,
				Usage: "Path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},
			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: 5.0,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},
			cli.StringFlag{
				Name:        "mount-options, o",
				HideDefault: true,
				Usage:       "Additional system-specific mount options. Be careful!",
			},
			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results from GCS.",
			},
			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},
			cli.StringFlag{
				Name:        "temp-dir, t",
				Value:       "",
				HideDefault: true,
				Usage: "Temporary directory for local GCS object copies. " +
					"(default: system default, likely /tmp)",
			},
			cli.IntFlag{
				Name:  "temp-dir-bytes",
				Value: 1 << 31,
				Usage: "Size limit of the temporary directory.",
			},
			cli.IntFlag{
				Name:        "uid",
				Value:       -1,
				HideDefault: true,
				Usage:       "UID owner of all inodes.",
			},
			cli.BoolFlag{
				Name:  "help, h",
				Usage: "print this help text",
			},
		},
	}

	return
}

type flagStorage struct {
	MountOptions                       map[string]string
	Uid                                int64
	Gid                                int64
	FileMode                           uint
	DirMode                            uint
	TempDir                            string
	TempDirLimit                       int64
	GCSChunkSize                       uint64
	ImplicitDirs                       bool
	StatCacheTTL                       time.Duration
	TypeCacheTTL                       time.Duration
	OpRateLimitHz                      float64
	EgressBandwidthLimitBytesPerSecond float64
	KeyFile                            string
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	flags = new(flagStorage)
	flags.MountOptions = make(map[string]string)
	mountpkg.OptionValue(flags.MountOptions).Set(c.String("mount-options"))
	flags.Uid = int64(c.Int("uid"))
	flags.Gid = int64(c.Int("gid"))
	flags.FileMode = uint(c.Int("file-mode"))
	flags.DirMode = uint(c.Int("dir-mode"))
	flags.TempDir = c.String("temp-dir")
	flags.TempDirLimit = int64(c.Int("temp-dir-bytes"))
	flags.GCSChunkSize = uint64(c.Int("gcs-chunk-size"))
	flags.ImplicitDirs = c.Bool("implicit-dirs")
	flags.StatCacheTTL = c.Duration("stat-cache-ttl")
	flags.TypeCacheTTL = c.Duration("type-cache-ttl")
	flags.OpRateLimitHz = c.Float64("limit-ops-per-sec")
	flags.EgressBandwidthLimitBytesPerSecond = c.Float64("limit-bytes-per-sec")
	flags.KeyFile = c.String("key-file")
	return
}
<commit_msg>Don't include an (already out of date) version.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"os"
	"time"

	mountpkg "github.com/googlecloudplatform/gcsfuse/mount"
	"github.com/jgeewax/cli"
)

func getApp() (app *cli.App) {
	app = &cli.App{
		Name:          "gcsfuse",
		Usage:         "Mount a GCS bucket locally",
		ArgumentUsage: "bucket mountpoint",
		HideHelp:      true,
		HideVersion:   true,
		Writer:        os.Stderr,
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:        "dir-mode, d",
				Usage:       "Permissions bits for directories. (default: 0755)",
				HideDefault: true,
			},
			cli.IntFlag{
				Name:        "file-mode, f",
				Value:       0644,
				Usage:       "Permission bits for files (default: 0644)",
				HideDefault: true,
			},
			cli.IntFlag{
				Name:  "gcs-chunk-size",
				Value: 1 << 24,
				Usage: "Max chunk size for loading GCS objects.",
			},
			cli.IntFlag{
				Name:        "gid",
				Value:       -1,
				HideDefault: true,
				Usage:       "GID owner of all inodes.",
			},
			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See" +
					"docs/semantics.md",
			},
			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},
			cli.StringFlag{
				Name:        "key-file",
				Value:       "",
				HideDefault: true,
				Usage: "Path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},
			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: 5.0,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},
			cli.StringFlag{
				Name:        "mount-options, o",
				HideDefault: true,
				Usage:       "Additional system-specific mount options. Be careful!",
			},
			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results from GCS.",
			},
			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},
			cli.StringFlag{
				Name:        "temp-dir, t",
				Value:       "",
				HideDefault: true,
				Usage: "Temporary directory for local GCS object copies. " +
					"(default: system default, likely /tmp)",
			},
			cli.IntFlag{
				Name:  "temp-dir-bytes",
				Value: 1 << 31,
				Usage: "Size limit of the temporary directory.",
			},
			cli.IntFlag{
				Name:        "uid",
				Value:       -1,
				HideDefault: true,
				Usage:       "UID owner of all inodes.",
			},
			cli.BoolFlag{
				Name:  "help, h",
				Usage: "print this help text",
			},
		},
	}

	return
}

type flagStorage struct {
	MountOptions                       map[string]string
	Uid                                int64
	Gid                                int64
	FileMode                           uint
	DirMode                            uint
	TempDir                            string
	TempDirLimit                       int64
	GCSChunkSize                       uint64
	ImplicitDirs                       bool
	StatCacheTTL                       time.Duration
	TypeCacheTTL                       time.Duration
	OpRateLimitHz                      float64
	EgressBandwidthLimitBytesPerSecond float64
	KeyFile                            string
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	flags = new(flagStorage)
	flags.MountOptions = make(map[string]string)
	mountpkg.OptionValue(flags.MountOptions).Set(c.String("mount-options"))
	flags.Uid = int64(c.Int("uid"))
	flags.Gid = int64(c.Int("gid"))
	flags.FileMode = uint(c.Int("file-mode"))
	flags.DirMode = uint(c.Int("dir-mode"))
	flags.TempDir = c.String("temp-dir")
	flags.TempDirLimit = int64(c.Int("temp-dir-bytes"))
	flags.GCSChunkSize = uint64(c.Int("gcs-chunk-size"))
	flags.ImplicitDirs = c.Bool("implicit-dirs")
	flags.StatCacheTTL = c.Duration("stat-cache-ttl")
	flags.TypeCacheTTL = c.Duration("type-cache-ttl")
	flags.OpRateLimitHz = c.Float64("limit-ops-per-sec")
	flags.EgressBandwidthLimitBytesPerSecond = c.Float64("limit-bytes-per-sec")
	flags.KeyFile = c.String("key-file")
	return
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	sdk "github.com/dysolution/espsdk"
	"github.com/icrowley/fake"
)

var bullets = make(map[string]Bullet)
var bombs = make(map[string]Bomb)

func bullet(name string, method string, url string, payload sdk.RESTObject) {
	bullets[name] = Bullet{&client, name, method, url, payload}
}

func makeBomb(name string, bullets ...Bullet) {
	var bs []Bullet
	for _, bullet := range bullets {
		bs = append(bs, bullet)
	}
	bombs[name] = Bomb{
		Name:    name,
		Bullets: bs,
	}
}

func lastBatch() sdk.Batch {
	return client.Get(sdk.Batches).Last()
}

func defineBullets() {
	bullet("get_batches", "GET", sdk.Batches, nil)

	bullet("create_batch", "POST", sdk.Batches, sdk.Batch{
		SubmissionName: appID + ": " + fake.FullName(),
		SubmissionType: "getty_creative_video",
	})

	bullet("delete_last_batch", "DELETE", "", lastBatch())

	badBatch := sdk.Batch{ID: -1}
	bullet("get_invalid_batch", "GET", badBatch.Path(), badBatch)

	edPhoto := sdk.Contribution{
		SubmissionBatchID:    86102,
		CameraShotDate:       time.Now().Format("01/02/2006"),
		ContentProviderName:  "provider",
		ContentProviderTitle: "Contributor",
		CountryOfShoot:       fake.Country(),
		CreditLine:           fake.FullName(),
		FileName:             fake.Word() + ".jpg",
		Headline:             fake.Sentence(),
		IPTCCategory:         "S",
		SiteDestination:      []string{"Editorial", "WireImage.com"},
		Source:               "AFP",
	}
	bullet("create_photo", "POST", edPhoto.Path(), edPhoto)

	edBatch := sdk.Batch{ID: 86103}
	bullet("get_photos", "GET", edBatch.Path(), edBatch)

	release := sdk.Release{
		SubmissionBatchID: 86103,
		FileName:          "some_property.jpg",
		ReleaseType:       "Property",
		FilePath:          "submission/releases/batch_86103/24780225369200015_some_property.jpg",
		MimeType:          "image/jpeg",
	}
	bullet("create_release", "POST", release.Path(), release)
}

// ExampleConfig returns an example of a complete configuration for the app.
// When marshaled into JSON, this can be used as the contents of the config
// file.
func ExampleConfig() Raid {
	defineBullets()

	makeBomb("create_batch",
		bullets["create_batch"],
	)
	makeBomb("delete_last_batch",
		bullets["delete_last_batch"],
	)
	makeBomb("create_and_confirm_batch",
		bullets["get_batches"],
		bullets["create_batch"],
		bullets["get_batches"],
	)
	makeBomb("create_and_delete_batch",
		bullets["create_batch"],
		bullets["delete_last_batch"],
	)
	makeBomb("get_invalid_batches",
		bullets["get_invalid_batch"],
	)
	makeBomb("create_and_confirm_photo",
		bullets["create_photo"],
		bullets["get_photos"],
	)
	makeBomb("upload_a_release",
		bullets["create_release"],
	)

	return NewRaid(
		bombs["create_batch"],
		bombs["delete_last_batch"],
	)
}
<commit_msg>add workflows<commit_after>package main

import (
	"time"

	sdk "github.com/dysolution/espsdk"
	"github.com/icrowley/fake"
)

var bullets = make(map[string]Bullet)
var bombs = make(map[string]Bomb)

func bullet(name string, method string, url string, payload sdk.RESTObject) {
	bullets[name] = Bullet{&client, name, method, url, payload}
}

func makeBomb(name string, bullets ...Bullet) {
	var bs []Bullet
	for _, bullet := range bullets {
		bs = append(bs, bullet)
	}
	bombs[name] = Bomb{
		Name:    name,
		Bullets: bs,
	}
}

func lastBatch() sdk.Batch {
	return client.Get(sdk.Batches).Last()
}

func defineBullets() {
	bullet("get_batches", "GET", sdk.Batches, nil)

	bullet("create_batch", "POST", sdk.Batches, sdk.Batch{
		SubmissionName: appID + ": " + fake.FullName(),
		SubmissionType: "getty_creative_video",
	})

	bullet("delete_last_batch", "DELETE", "", lastBatch())

	badBatch := sdk.Batch{ID: -1}
	bullet("get_invalid_batch", "GET", badBatch.Path(), badBatch)

	edPhoto := sdk.Contribution{
		SubmissionBatchID:    86102,
		CameraShotDate:       time.Now().Format("01/02/2006"),
		ContentProviderName:  "provider",
		ContentProviderTitle: "Contributor",
		CountryOfShoot:       fake.Country(),
		CreditLine:           fake.FullName(),
		FileName:             fake.Word() + ".jpg",
		Headline:             fake.Sentence(),
		IPTCCategory:         "S",
		SiteDestination:      []string{"Editorial", "WireImage.com"},
		Source:               "AFP",
	}
	bullet("create_photo", "POST", edPhoto.Path(), edPhoto)

	edBatch := sdk.Batch{ID: 86103}
	bullet("get_photos", "GET", edBatch.Path(), edBatch)

	release := sdk.Release{
		SubmissionBatchID: 86103,
		FileName:          "some_property.jpg",
		ReleaseType:       "Property",
		FilePath:          "submission/releases/batch_86103/24780225369200015_some_property.jpg",
		MimeType:          "image/jpeg",
	}
	bullet("create_release", "POST", release.Path(), release)
}

// ExampleConfig returns an example of a complete configuration for the app.
// When marshaled into JSON, this can be used as the contents of the config
// file.
func ExampleConfig() Raid {
	defineBullets()

	makeBomb("create_batch",
		bullets["create_batch"],
	)
	makeBomb("delete_last_batch",
		bullets["delete_last_batch"],
	)
	makeBomb("create_and_confirm_batch",
		bullets["get_batches"],
		bullets["create_batch"],
		bullets["get_batches"],
	)
	makeBomb("create_and_delete_batch",
		bullets["create_batch"],
		bullets["delete_last_batch"],
	)
	makeBomb("get_invalid_batches",
		bullets["get_invalid_batch"],
	)
	makeBomb("create_and_confirm_photo",
		bullets["create_photo"],
		bullets["get_photos"],
	)
	makeBomb("upload_a_release",
		bullets["create_release"],
	)

	return NewRaid(
		bombs["create_batch"],
		bombs["delete_last_batch"],
		// bombs["create_and_confirm_batch"],
		// bombs["create_and_delete_batch"],
		bombs["get_invalid_batches"],
		bombs["create_and_confirm_photo"],
		bombs["upload_a_release"],
	)
}
<|endoftext|>


<commit_before>package newsGetter

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"
	"web_apps/news_crawlers/modules/database"
)

// GoogleNews interface for google news
type GoogleNews map[string]interface{}

// GoogleNewsResponseData response struct
type GoogleNewsResponseData struct {
	ResponseData struct {
		Results []GoogleNewsResults
	}
}

// ResponseData response struct
type ResponseData struct {
	Results []GoogleNewsResults
}

var (
	googleNewsProvider = "https://news.google.com/"
	googleNewsName     = "GoogleNews"
)

// TopicsList return a list of topics/categories
func TopicsList() Topics {
	topics := Topics{
		"society":       TopicIdentity{"y", "社会"},
		"international": TopicIdentity{"w", "国際"},
		"business":      TopicIdentity{"b", "ビジネス"},
		"politics":      TopicIdentity{"p", "政治"},
		"entertainment": TopicIdentity{"e", "エンタメ"},
		"sports":        TopicIdentity{"s", "スポーツ"},
		"technology":    TopicIdentity{"t", "テクノロジー"},
		"pickup":        TopicIdentity{"ir", "ピックアップ"},
	}

	return topics
}

// StartGoogleNews start collecting google news
func StartGoogleNews(googleLoopCounterDelay int) {
	fmt.Println("startgoogle news launched!")

	for t := range time.Tick(time.Duration(googleLoopCounterDelay) * time.Second) {
		_ = t

		fmt.Println("loop will start")
		var wsg sync.WaitGroup
		n := make(chan GoogleNewsResponseData)
		for _, v := range TopicsList() {
			wsg.Add(1)
			go func(v TopicIdentity) {
				go GoogleNewsRequester(googleURLConstructor(v.Initial), v, n, &wsg)

				result := <-n
				GoogleNewsRW(result, v)
			}(v)
		}
		wsg.Wait()
		close(n)
	}
}

// GoogleNewsRequester google news http getter
func GoogleNewsRequester(url string, topic TopicIdentity, c chan GoogleNewsResponseData, wg *sync.WaitGroup) {
	defer wg.Done()
	var googleNews GoogleNewsResponseData
	response, err := httpGet(url)
	if err != nil {
		fmt.Println("got error google!")
		return
	}

	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &googleNews); err != nil {
		//return id_containers
		fmt.Println(err)
		c <- googleNews
		return
	}
	c <- googleNews
}

// GoogleNewsRW read and write data from google news
func GoogleNewsRW(gn GoogleNewsResponseData, topic TopicIdentity) {
	var wg sync.WaitGroup
	for _, g := range gn.ResponseData.Results {
		// set news item category
		g.Category = topic
		wg.Add(1)
		GoogleNewsDataSetter(g, &wg)
	}
	wg.Wait()
}

// GoogleNewsDataSetter builds and construct data for insertion
func GoogleNewsDataSetter(googleNews GoogleNewsResults, wg *sync.WaitGroup) {
	defer wg.Done()
	jsonNews := &jsonNewsBody{
		Title:          googleNews.Title,
		By:             "GoogleNews",
		Score:          0,
		Time:           int(time.Now().Unix()),
		Url:            googleNews.URL,
		ProviderName:   googleNewsName,
		RelatedStories: googleNews.RelatedStories,
		CreatedAt:      fmt.Sprintf("%v", time.Now().Local()),
		Category:       googleNews.Category,
		Image:          googleNews.Image,
	}

	// check if data exists already, need refactoring though
	saved := database.GoogleNewsInsert(jsonNews, googleNews.Title)
	if saved {
		fmt.Println("saved!! google news!")
		return
	}

	fmt.Println("did not save!")
}

//googleUrlConstructor return url string
func googleURLConstructor(v string) string {
	url := fmt.Sprintf("https://ajax.googleapis.com/ajax/services/search/news?v=1.0&topic=%s&ned=jp&userip=192.168.0.1", v)
	return url
}
<commit_msg>changed fmt print to log<commit_after>package newsGetter

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"
	"time"
	"web_apps/news_crawlers/modules/database"
)

// GoogleNews interface for google news
type GoogleNews map[string]interface{}

// GoogleNewsResponseData response struct
type GoogleNewsResponseData struct {
	ResponseData struct {
		Results []GoogleNewsResults
	}
}

// ResponseData response struct
type ResponseData struct {
	Results []GoogleNewsResults
}

var (
	googleNewsProvider = "https://news.google.com/"
	googleNewsName     = "GoogleNews"
)

// TopicsList return a list of topics/categories
func TopicsList() Topics {
	topics := Topics{
		"society":       TopicIdentity{"y", "社会"},
		"international": TopicIdentity{"w", "国際"},
		"business":      TopicIdentity{"b", "ビジネス"},
		"politics":      TopicIdentity{"p", "政治"},
		"entertainment": TopicIdentity{"e", "エンタメ"},
		"sports":        TopicIdentity{"s", "スポーツ"},
		"technology":    TopicIdentity{"t", "テクノロジー"},
		"pickup":        TopicIdentity{"ir", "ピックアップ"},
	}

	return topics
}

// StartGoogleNews start collecting google news
func StartGoogleNews(googleLoopCounterDelay int) {
	fmt.Println("startgoogle news launched!")

	for t := range time.Tick(time.Duration(googleLoopCounterDelay) * time.Second) {
		_ = t

		log.Println("loop will start")
		var wsg sync.WaitGroup
		n := make(chan GoogleNewsResponseData)
		for _, v := range TopicsList() {
			wsg.Add(1)
			go func(v TopicIdentity) {
				go GoogleNewsRequester(googleURLConstructor(v.Initial), v, n, &wsg)

				result := <-n
				GoogleNewsRW(result, v)
			}(v)
		}
		wsg.Wait()
		close(n)
	}
}

// GoogleNewsRequester google news http getter
func GoogleNewsRequester(url string, topic TopicIdentity, c chan GoogleNewsResponseData, wg *sync.WaitGroup) {
	defer wg.Done()
	var googleNews GoogleNewsResponseData
	response, err := httpGet(url)
	if err != nil {
		fmt.Println("got error google!")
		return
	}

	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &googleNews); err != nil {
		//return id_containers
		fmt.Println(err)
		c <- googleNews
		return
	}
	c <- googleNews
}

// GoogleNewsRW read and write data from google news
func GoogleNewsRW(gn GoogleNewsResponseData, topic TopicIdentity) {
	var wg sync.WaitGroup
	for _, g := range gn.ResponseData.Results {
		// set news item category
		g.Category = topic
		wg.Add(1)
		GoogleNewsDataSetter(g, &wg)
	}
	wg.Wait()
}

// GoogleNewsDataSetter builds and construct data for insertion
func GoogleNewsDataSetter(googleNews GoogleNewsResults, wg *sync.WaitGroup) {
	defer wg.Done()
	jsonNews := &jsonNewsBody{
		Title:          googleNews.Title,
		By:             "GoogleNews",
		Score:          0,
		Time:           int(time.Now().Unix()),
		Url:            googleNews.URL,
		ProviderName:   googleNewsName,
		RelatedStories: googleNews.RelatedStories,
		CreatedAt:      fmt.Sprintf("%v", time.Now().Local()),
		Category:       googleNews.Category,
		Image:          googleNews.Image,
	}

	// check if data exists already, need refactoring though
	saved := database.GoogleNewsInsert(jsonNews, googleNews.Title)
	if saved {
		log.Println("saved!! google news!")
		return
	}

	log.Println("did not save!")
}

//googleUrlConstructor return url string
func googleURLConstructor(v string) string {
	url := fmt.Sprintf("https://ajax.googleapis.com/ajax/services/search/news?v=1.0&topic=%s&ned=jp&userip=192.168.0.1", v)
	return url
}
<|endoftext|>


<commit_before>package idols

import (
	"fmt"

	"github.com/Seklfreak/Robyul2/cache"

	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/bwmarrin/discordgo"
)

// migrateIdols will migrate old idol records where each image was saved
// seporatly to the new table
func migrateIdols(msg *discordgo.Message, content string) {

	// check if the new table already exists and has records. if it does block the migration again
	/*
		count, err := helpers.GetMDb().C(models.IdolTable.String()).Count()
		if err != nil {
			cache.GetLogger().Errorln(err.Error())
			return
		}
		if count > 0 {
			helpers.SendMessage(msg.ChannelID, fmt.Sprintf("Migration has already been run. If something went wrong please drop table **%s** and run this again.", models.IdolTable))
			return
		}
	*/

	// refresh the idols from mongo again to make sure the data is up to date
	helpers.SendMessage(msg.ChannelID, "Refreshing Idols from old table....")
	refreshIdolsFromOld(true)
	helpers.SendMessage(msg.ChannelID, "Idols refreshed")
	helpers.SendMessage(msg.ChannelID, fmt.Sprintf("Idols Found: %d", len(GetAllIdols())))

	// convert current in memory idol records to the new tables idol entries
	helpers.SendMessage(msg.ChannelID, "Migrating idols to new table...")
	for _, idol := range GetAllIdols() {
		newIdolEntry := models.IdolEntry{
			ID:        "",
			Name:      idol.Name,
			GroupName: idol.GroupName,
			Gender:    idol.Gender,
		}

		// convert current idol images to new table images entries
		for _, img := range idol.Images {
			newImageEntry := models.IdolImageEntry{
				HashString: img.HashString,
				ObjectName: img.ObjectName,
			}
			newIdolEntry.Images = append(newIdolEntry.Images, newImageEntry)
		}
		if len(newIdolEntry.Images) == 0 {
			helpers.SendMessage(msg.ChannelID, fmt.Sprintf("Could not migrate %s %s due to missing images", newIdolEntry.GroupName, newIdolEntry.Name))
			continue
		}
		_, err := helpers.MDbInsert(models.IdolTable, newIdolEntry)
		if err != nil {
			cache.GetLogger().Errorln(err.Error())
		}
	}

	refreshIdols(true)

	helpers.SendMessage(msg.ChannelID, "Migration done.")
}
<commit_msg>[idol] oof<commit_after>package idols

import (
	"fmt"

	"github.com/Seklfreak/Robyul2/cache"

	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/bwmarrin/discordgo"
)

// migrateIdols will migrate old idol records where each image was saved
// seporatly to the new table
func migrateIdols(msg *discordgo.Message, content string) {

	// check if the new table already exists and has records. if it does block the migration again
	count, err := helpers.GetMDb().C(models.IdolTable.String()).Count()
	if err != nil {
		cache.GetLogger().Errorln(err.Error())
		return
	}
	if count > 0 {
		helpers.SendMessage(msg.ChannelID, fmt.Sprintf("Migration has already been run. If something went wrong please drop table **%s** and run this again.", models.IdolTable))
		return
	}

	// refresh the idols from mongo again to make sure the data is up to date
	helpers.SendMessage(msg.ChannelID, "Refreshing Idols from old table....")
	refreshIdolsFromOld(true)
	helpers.SendMessage(msg.ChannelID, "Idols refreshed")
	helpers.SendMessage(msg.ChannelID, fmt.Sprintf("Idols Found: %d", len(GetAllIdols())))

	// convert current in memory idol records to the new tables idol entries
	helpers.SendMessage(msg.ChannelID, "Migrating idols to new table...")
	for _, idol := range GetAllIdols() {
		newIdolEntry := models.IdolEntry{
			ID:        "",
			Name:      idol.Name,
			GroupName: idol.GroupName,
			Gender:    idol.Gender,
		}

		// convert current idol images to new table images entries
		for _, img := range idol.Images {
			newImageEntry := models.IdolImageEntry{
				HashString: img.HashString,
				ObjectName: img.ObjectName,
			}
			newIdolEntry.Images = append(newIdolEntry.Images, newImageEntry)
		}
		if len(newIdolEntry.Images) == 0 {
			helpers.SendMessage(msg.ChannelID, fmt.Sprintf("Could not migrate %s %s due to missing images", newIdolEntry.GroupName, newIdolEntry.Name))
			continue
		}
		_, err := helpers.MDbInsert(models.IdolTable, newIdolEntry)
		if err != nil {
			cache.GetLogger().Errorln(err.Error())
		}
	}

	refreshIdols(true)

	helpers.SendMessage(msg.ChannelID, "Migration done.")
}
<|endoftext|>


<commit_before>package exp

// Boolean

type Bool bool

func (b Bool) Eval(p Params) bool {
	return bool(b)
}

var (
	// True is an expression that always evaluates to true.
	True = Bool(true)
	// False is an expression that always evaluates to false.
	False = Bool(false)
)
<commit_msg>commend on Bool<commit_after>package exp

// Boolean

// Bool is a wrapper for the native bool type which provides an Eval function so
// that it satisfies the Exp interface.
type Bool bool

// Eval will always return the boolean value of b and disregard p.
func (b Bool) Eval(p Params) bool {
	return bool(b)
}

var (
	// True is an expression that always evaluates to true.
	True = Bool(true)
	// False is an expression that always evaluates to false.
	False = Bool(false)
)
<|endoftext|>


<commit_before>// Package sqlc helps you combine bits of arbitrary bits of SQL
package sqlc

import "strings"

type component struct {
	partial string
	args    []interface{}
}

// From the psql documentation:
// SELECT [ ALL | DISTINCT [ ON ( expression [, ...] ) ] ]
//     * | expression [ [ AS ] output_name ] [, ...]
//     [ FROM from_item [, ...] ]
//     [ WHERE condition ]
//     [ GROUP BY expression [, ...] ]
//     [ HAVING condition [, ...] ]
//     [ WINDOW window_name AS ( window_definition ) [, ...] ]
//     [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ]
//     [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ]
//     [ LIMIT { count | ALL } ]
//     [ OFFSET start [ ROW | ROWS ] ]
//     [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ]
//     [ FOR { UPDATE | NO KEY UPDATE | SHARE | KEY SHARE } [ OF table_name [, ...] ] [ NOWAIT ] [...] ]

type Statement struct {
	selects []component
	froms   []component
	joins   []component
	wheres  []component
	groups  []component
	orders  []component
	limit   component
}

func (s Statement) Select(partial string, args ...interface{}) Statement {
	s.selects = append(s.selects, component{partial, args})
	return s
}

func (s Statement) From(partial string, args ...interface{}) Statement {
	s.froms = append(s.froms, component{partial, args})
	return s
}

func (s Statement) Join(partial string, args ...interface{}) Statement {
	s.joins = append(s.joins, component{partial, args})
	return s
}

func (s Statement) Where(partial string, args ...interface{}) Statement {
	s.wheres = append(s.wheres, component{"(" + partial + ")", args})
	return s
}

func (s Statement) Group(partial string, args ...interface{}) Statement {
	s.groups = append(s.groups, component{partial, args})
	return s
}

func (s Statement) Order(partial string, args ...interface{}) Statement {
	s.orders = append(s.orders, component{partial, args})
	return s
}

func (s Statement) Limit(partial string, args ...interface{}) Statement {
	s.limit = component{partial, args}
	return s
}

func (s Statement) ToSQL() (string, []interface{}) {
	parts := make([]string, 0)
	args := make([]interface{}, 0)

	if len(s.selects) > 0 {
		parts = append(parts, "SELECT "+joinParts(s.selects, ", ", &args))
	}
	if len(s.froms) > 0 {
		parts = append(parts, "FROM "+joinParts(s.froms, " ,", &args))
	}
	if len(s.joins) > 0 {
		parts = append(parts, "JOIN "+joinParts(s.joins, " ,", &args))
	}
	if len(s.wheres) > 0 {
		parts = append(parts, "WHERE "+joinParts(s.wheres, " AND ", &args))
	}
	if len(s.groups) > 0 {
		parts = append(parts, "GROUP BY "+joinParts(s.groups, " ,", &args))
	}
	if len(s.orders) > 0 {
		parts = append(parts, "ORDER BY "+joinParts(s.orders, " ,", &args))
	}
	if s.limit.partial != "" {
		parts = append(parts, "LIMIT "+addPart(s.limit, &args))
	}

	return strings.Join(parts, "\n"), args
}

func joinParts(components []component, joiner string, args *[]interface{}) string {
	partials := make([]string, 0)
	for _, component := range components {
		partials = append(partials, addPart(component, args))
	}
	return strings.Join(partials, joiner)
}

func addPart(component component, args *[]interface{}) string {
	*args = append(*args, component.args...)
	return component.partial
}
<commit_msg>Remove Join(), part of From()<commit_after>// Package sqlc helps you combine bits of arbitrary bits of SQL
package sqlc

import "strings"

type component struct {
	partial string
	args    []interface{}
}

// From the psql documentation:
// SELECT [ ALL | DISTINCT [ ON ( expression [, ...] ) ] ]
//     * | expression [ [ AS ] output_name ] [, ...]
//     [ FROM from_item [, ...] ]
//     [ WHERE condition ]
//     [ GROUP BY expression [, ...] ]
//     [ HAVING condition [, ...] ]
//     [ WINDOW window_name AS ( window_definition ) [, ...] ]
//     [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ]
//     [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ]
//     [ LIMIT { count | ALL } ]
//     [ OFFSET start [ ROW | ROWS ] ]
//     [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ]
//     [ FOR { UPDATE | NO KEY UPDATE | SHARE | KEY SHARE } [ OF table_name [, ...] ] [ NOWAIT ] [...] ]

type Statement struct {
	selects []component
	froms   []component
	wheres  []component
	groups  []component
	orders  []component
	limit   component
}

func (s Statement) Select(partial string, args ...interface{}) Statement {
	s.selects = append(s.selects, component{partial, args})
	return s
}

func (s Statement) From(partial string, args ...interface{}) Statement {
	s.froms = append(s.froms, component{partial, args})
	return s
}

func (s Statement) Where(partial string, args ...interface{}) Statement {
	s.wheres = append(s.wheres, component{"(" + partial + ")", args})
	return s
}

func (s Statement) Group(partial string, args ...interface{}) Statement {
	s.groups = append(s.groups, component{partial, args})
	return s
}

func (s Statement) Order(partial string, args ...interface{}) Statement {
	s.orders = append(s.orders, component{partial, args})
	return s
}

func (s Statement) Limit(partial string, args ...interface{}) Statement {
	s.limit = component{partial, args}
	return s
}

func (s Statement) ToSQL() (string, []interface{}) {
	parts := make([]string, 0)
	args := make([]interface{}, 0)

	if len(s.selects) > 0 {
		parts = append(parts, "SELECT "+joinParts(s.selects, ", ", &args))
	}
	if len(s.froms) > 0 {
		parts = append(parts, "FROM "+joinParts(s.froms, " ,", &args))
	}
	if len(s.wheres) > 0 {
		parts = append(parts, "WHERE "+joinParts(s.wheres, " AND ", &args))
	}
	if len(s.groups) > 0 {
		parts = append(parts, "GROUP BY "+joinParts(s.groups, " ,", &args))
	}
	if len(s.orders) > 0 {
		parts = append(parts, "ORDER BY "+joinParts(s.orders, " ,", &args))
	}
	if s.limit.partial != "" {
		parts = append(parts, "LIMIT "+addPart(s.limit, &args))
	}

	return strings.Join(parts, "\n"), args
}

func joinParts(components []component, joiner string, args *[]interface{}) string {
	partials := make([]string, 0)
	for _, component := range components {
		partials = append(partials, addPart(component, args))
	}
	return strings.Join(partials, joiner)
}

func addPart(component component, args *[]interface{}) string {
	*args = append(*args, component.args...)
	return component.partial
}
<|endoftext|>


<commit_before>package main

import (
	"math"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/mt2d2/forum/model"
)

func (app *app) handleIndex(w http.ResponseWriter, req *http.Request) {
	forums, err := model.FindForums(app.db)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	results := make(map[string]interface{})
	results["forums"] = forums

	app.renderTemplate(w, req, "index", results)
}

func (app *app) handleForum(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	id := vars["id"]
	pageOffset := 0
	if page, ok := vars["page"]; ok {
		if val, err := strconv.Atoi(page); err == nil {
			pageOffset = val - 1
		}
	}

	forum, err := model.FindOneForum(app.db, id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	numberOfPages := int(math.Ceil(float64(forum.TopicCount) / float64(limitTopics)))
	pageIndicies := make([]int, numberOfPages)
	for i := 0; i < numberOfPages; i++ {
		pageIndicies[i] = i + 1
	}

	topics, err := model.FindTopics(app.db, id, limitTopics, pageOffset*limitTopics)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	app.addBreadCrumb("/forum/"+strconv.Itoa(forum.Id), forum.Title)

	results := make(map[string]interface{})
	results["forum"] = forum
	results["topics"] = topics
	results["pageIndicies"] = pageIndicies
	results["currentPage"] = int(pageOffset + 1)

	app.renderTemplate(w, req, "forum", results)
}
<commit_msg>breadcrump forum page numbers<commit_after>package main

import (
	"math"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/mt2d2/forum/model"
)

func (app *app) handleIndex(w http.ResponseWriter, req *http.Request) {
	forums, err := model.FindForums(app.db)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	results := make(map[string]interface{})
	results["forums"] = forums

	app.renderTemplate(w, req, "index", results)
}

func (app *app) handleForum(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	id := vars["id"]
	pageOffset := 0
	if page, ok := vars["page"]; ok {
		if val, err := strconv.Atoi(page); err == nil {
			pageOffset = val - 1
		}
	}

	forum, err := model.FindOneForum(app.db, id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	numberOfPages := int(math.Ceil(float64(forum.TopicCount) / float64(limitTopics)))
	pageIndicies := make([]int, numberOfPages)
	for i := 0; i < numberOfPages; i++ {
		pageIndicies[i] = i + 1
	}
	currentPage := int(pageOffset + 1)

	topics, err := model.FindTopics(app.db, id, limitTopics, pageOffset*limitTopics)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	app.addBreadCrumb("/forum/"+strconv.Itoa(forum.Id), forum.Title)
	if currentPage > 1 {
		app.addBreadCrumb("forum/"+strconv.Itoa(forum.Id)+"/page/"+strconv.Itoa(currentPage), "page "+strconv.Itoa(currentPage))
	}

	results := make(map[string]interface{})
	results["forum"] = forum
	results["topics"] = topics
	results["pageIndicies"] = pageIndicies
	results["currentPage"] = currentPage

	app.renderTemplate(w, req, "forum", results)
}
<|endoftext|>


<commit_before>package main

const (
	VERSION         = "1.36.1"
	DATABASE_DIR    = "database"
	RELEASE_URL     = "https://github.com/Seklfreak/discord-image-downloader-go/releases/latest"
	RELEASE_API_URL = "https://api.github.com/repos/Seklfreak/discord-image-downloader-go/releases/latest"
	IMGUR_CLIENT_ID = "a39473314df3f59"

	DEFAULT_CONFIG_FILE = "config.ini"

	discordEmojiBaseUrl = "https://cdn.discordapp.com/emojis/"
)
<commit_msg>bumps version<commit_after>package main

const (
	VERSION         = "1.36.2"
	DATABASE_DIR    = "database"
	RELEASE_URL     = "https://github.com/Seklfreak/discord-image-downloader-go/releases/latest"
	RELEASE_API_URL = "https://api.github.com/repos/Seklfreak/discord-image-downloader-go/releases/latest"
	IMGUR_CLIENT_ID = "a39473314df3f59"

	DEFAULT_CONFIG_FILE = "config.ini"

	discordEmojiBaseUrl = "https://cdn.discordapp.com/emojis/"
)
<|endoftext|>


<commit_before>package main

import "regexp"

var (
	numWorkers    int64 = 5
	timeout       int64 = 10000
	serverPort          = 19091
	useFUP              = false
	logErrors           = false
	useFind             = false
	useCheck            = false
	useCheckAll         = false
	useMyIPCheck        = false
	useServer           = false
	useDebug            = false
	useAddLink          = false
	useNoAddLinks       = false
	useTestLink         = false
	useFile             = ""
	testLink            = ""
	addLink             = ""

	myIP       string
	reRemoteIP = regexp.MustCompile(`<p>RemoteAddr: (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):\d{1,5}<\/p>`)
	reMyIP     = regexp.MustCompile(`<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})<\/td>`)

	// siteList = []string{
	// 	`https://hidester.com/proxydata/php/data.php?mykey=data&offset=0&limit=1000&orderBy=latest_check&sortOrder=DESC&country=&port=&type=undefined&anonymity=undefined&ping=undefined&gproxy=2`,
	// 	`http://gatherproxy.com/embed/`,
	// 	`http://txt.proxyspy.net/proxy.txt`,
	// 	`http://webanetlabs.net/publ/24`,
	// 	`http://awmproxy.com/freeproxy.php`,
	// 	`http://www.samair.ru/proxy/type-01.htm`,
	// 	`https://www.us-proxy.org/`,
	// 	`http://free-proxy-list.net/`,
	// 	`http://www.proxynova.com/proxy-server-list/`,
	// 	`http://proxyserverlist-24.blogspot.ru/`,
	// 	`http://gatherproxy.com/`,
	// 	`https://hidemy.name/ru/proxy-list/`,
	// 	`https://hidemy.name/en/proxy-list/?type=hs&anon=34#list`,
	// 	`https://free-proxy-list.com`,
	// 	`https://free-proxy-list.com/?search=1&page=&port=&type%5B%5D=http&type%5B%5D=https&level%5B%5D=anonymous&level%5B%5D=high-anonymous&speed%5B%5D=2&speed%5B%5D=3&connect_time%5B%5D=2&connect_time%5B%5D=3&up_time=40&search=Search`,
	// 	`http://www.idcloak.com/proxylist/free-proxy-servers-list.html`,
	// 	`https://premproxy.com/list/`,
	// 	`https://proxy-list.org/english/index.php`,

	// 	`https://www.sslproxies.org/`,

	// 	// `https://best-proxies.ru/proxylist/free/`,
	// 	// `http://spys.ru/proxies/`,

	// 	// `http://www.freeproxylists.com/elite.html`,
	// 	// `http://www.freeproxylists.com/anonymous.html`,
	// 	// `http://www.xroxy.com/proxylist.php?port=Standard&type=All_http&ssl=&country=&latency=1000&reliability=9000#table`,
	// 	// `http://www.prime-speed.ru/proxy/free-proxy-list/all-working-proxies.php`,
	// 	// `http://proxyservers.pro/`,
	// }

	reURL = []string{
		`href=(?:'|")/publ/(\d{1,3}-\d{1,3})(?:'|")\s`,
		`href=(?:'|")/(freeproxylist/proxylist.*?\.txt)(?:'|")`,
		`value=(?:'|")http://awmproxy.com/(freeproxy_\d{3,12}\.txt)(?:'|")`,
		`<a href=(?:'|")/(proxies/\d{1,3}/)(?:'|")>`,
		`<a class=(?:'|")page(?:'|") href=(?:'|")(type-\d{1,3}.htm)(?:'|")>`,
		`<a href=(?:'|")/(proxy/list/order/updated/order_dir/desc/page/\d{1,3})(?:'|")>`,
		`<a href='http://proxyserverlist-24.blogspot.ru/(\d{4}/\d{1,2}/\d{1,2}-\d{1,2}-\d{1,2}-free-proxy-server-list-\d{1,6}.html#more)'`,
		`<a href=(?:'|")((?:anon|elite)/\d{1,12}.html)(?:'|")>(?:anon|elite)`,
		`href=(?:'|")/(proxylist\.php\?.+?\#table)`,
		`href=(?:'|")/(\w\w/proxy-list/\?.*?#list)`,
		`<a class=(?:'|")page(?:'|") href=(?:'|")(\d{1,3}.htm)(?:'|")>`,
		`href=(?:'|")\./(index\.php\?p=\d{1,3})(?:'|")>`,
		`<a href=(?:'|")(\d{1,3}\.htm)(?:'|")>`,
		`<a href=(?:'|")/(\w*?/\w*?/\d{1,4}/)(?:'|")>`,
		`http://www\.proxyserverlist24\.top/(\d{4}/\d{2}/\S*?\.html)`,
	}
	reIP        = `((?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))`
	reCommaList = []string{
		`</td><td>`,
		`:`,
		`.*?(?:'|")Proxy Port \d{2,5}(?:'|")>`,
		`(?:'|")\,(?:'|")PORT(?:'|"):`,
		`(?:'|") data\-port=(?:'|")`,
	}
	rePort  = `(\d{2,5})`
	replace = [][]string{
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")1F90(?:'|")`, ":8080"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")C38(?:'|")`, ":3128"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")1FB6(?:'|")`, ":8118"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")22B8(?:'|")`, ":8888"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")270F(?:'|")`, ":9999"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")50(?:'|")`, ":80"},
		{`<script type=(?:'|")text/javascript(?:'|")>document\.write\((?:'|")<font class=spy2>`, ""},
		{`<span style=(?:'|")display:none(?:'|")>\d{1,3}</span>`, ""},
		{`</font>(?:'|")`, ""},
		{`<span>`, ""},
		{`</span>`, ""},
		{`\n`, ""},
		{`</td>.+?<td.+?>`, "	"},
	}
	baseDecode = []string{
		`text/javascript(?:'|")>Proxy\((?:'|")(.*?)(?:'|")\)</script>`,
	}
	base16 = []string{
		`PROXY_IP(?:'|"):(?:'|")((?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))(?:'|"),.*?,(?:'|")PROXY_PORT(?:'|"):(?:'|")([a-fA-F0-9]{1,4})(?:'|")`,
	}
	headers = []string{
		"HTTP_VIA",
		"HTTP_X_FORWARDED_FOR",
		"HTTP_FORWARDED_FOR",
		"HTTP_X_FORWARDED",
		"HTTP_FORWARDED",
		"HTTP_CLIENT_IP",
		"HTTP_FORWARDED_FOR_IP",
		"VIA",
		"X_FORWARDED_FOR",
		"FORWARDED_FOR",
		"X_FORWARDED",
		"FORWARDED",
		"CLIENT_IP",
		"FORWARDED_FOR_IP",
		"HTTP_PROXY_CONNECTION",
	}
)
<commit_msg>increase default timeout<commit_after>package main

import "regexp"

var (
	numWorkers    int64 = 5
	timeout       int64 = 15000
	serverPort          = 19091
	useFUP              = false
	logErrors           = false
	useFind             = false
	useCheck            = false
	useCheckAll         = false
	useMyIPCheck        = false
	useServer           = false
	useDebug            = false
	useAddLink          = false
	useNoAddLinks       = false
	useTestLink         = false
	useFile             = ""
	testLink            = ""
	addLink             = ""

	myIP       string
	reRemoteIP = regexp.MustCompile(`<p>RemoteAddr: (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):\d{1,5}<\/p>`)
	reMyIP     = regexp.MustCompile(`<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})<\/td>`)

	// siteList = []string{
	// 	`https://hidester.com/proxydata/php/data.php?mykey=data&offset=0&limit=1000&orderBy=latest_check&sortOrder=DESC&country=&port=&type=undefined&anonymity=undefined&ping=undefined&gproxy=2`,
	// 	`http://gatherproxy.com/embed/`,
	// 	`http://txt.proxyspy.net/proxy.txt`,
	// 	`http://webanetlabs.net/publ/24`,
	// 	`http://awmproxy.com/freeproxy.php`,
	// 	`http://www.samair.ru/proxy/type-01.htm`,
	// 	`https://www.us-proxy.org/`,
	// 	`http://free-proxy-list.net/`,
	// 	`http://www.proxynova.com/proxy-server-list/`,
	// 	`http://proxyserverlist-24.blogspot.ru/`,
	// 	`http://gatherproxy.com/`,
	// 	`https://hidemy.name/ru/proxy-list/`,
	// 	`https://hidemy.name/en/proxy-list/?type=hs&anon=34#list`,
	// 	`https://free-proxy-list.com`,
	// 	`https://free-proxy-list.com/?search=1&page=&port=&type%5B%5D=http&type%5B%5D=https&level%5B%5D=anonymous&level%5B%5D=high-anonymous&speed%5B%5D=2&speed%5B%5D=3&connect_time%5B%5D=2&connect_time%5B%5D=3&up_time=40&search=Search`,
	// 	`http://www.idcloak.com/proxylist/free-proxy-servers-list.html`,
	// 	`https://premproxy.com/list/`,
	// 	`https://proxy-list.org/english/index.php`,

	// 	`https://www.sslproxies.org/`,

	// 	// `https://best-proxies.ru/proxylist/free/`,
	// 	// `http://spys.ru/proxies/`,

	// 	// `http://www.freeproxylists.com/elite.html`,
	// 	// `http://www.freeproxylists.com/anonymous.html`,
	// 	// `http://www.xroxy.com/proxylist.php?port=Standard&type=All_http&ssl=&country=&latency=1000&reliability=9000#table`,
	// 	// `http://www.prime-speed.ru/proxy/free-proxy-list/all-working-proxies.php`,
	// 	// `http://proxyservers.pro/`,
	// }

	reURL = []string{
		`href=(?:'|")/publ/(\d{1,3}-\d{1,3})(?:'|")\s`,
		`href=(?:'|")/(freeproxylist/proxylist.*?\.txt)(?:'|")`,
		`value=(?:'|")http://awmproxy.com/(freeproxy_\d{3,12}\.txt)(?:'|")`,
		`<a href=(?:'|")/(proxies/\d{1,3}/)(?:'|")>`,
		`<a class=(?:'|")page(?:'|") href=(?:'|")(type-\d{1,3}.htm)(?:'|")>`,
		`<a href=(?:'|")/(proxy/list/order/updated/order_dir/desc/page/\d{1,3})(?:'|")>`,
		`<a href='http://proxyserverlist-24.blogspot.ru/(\d{4}/\d{1,2}/\d{1,2}-\d{1,2}-\d{1,2}-free-proxy-server-list-\d{1,6}.html#more)'`,
		`<a href=(?:'|")((?:anon|elite)/\d{1,12}.html)(?:'|")>(?:anon|elite)`,
		`href=(?:'|")/(proxylist\.php\?.+?\#table)`,
		`href=(?:'|")/(\w\w/proxy-list/\?.*?#list)`,
		`<a class=(?:'|")page(?:'|") href=(?:'|")(\d{1,3}.htm)(?:'|")>`,
		`href=(?:'|")\./(index\.php\?p=\d{1,3})(?:'|")>`,
		`<a href=(?:'|")(\d{1,3}\.htm)(?:'|")>`,
		`<a href=(?:'|")/(\w*?/\w*?/\d{1,4}/)(?:'|")>`,
		`http://www\.proxyserverlist24\.top/(\d{4}/\d{2}/\S*?\.html)`,
	}
	reIP        = `((?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))`
	reCommaList = []string{
		`</td><td>`,
		`:`,
		`.*?(?:'|")Proxy Port \d{2,5}(?:'|")>`,
		`(?:'|")\,(?:'|")PORT(?:'|"):`,
		`(?:'|") data\-port=(?:'|")`,
	}
	rePort  = `(\d{2,5})`
	replace = [][]string{
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")1F90(?:'|")`, ":8080"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")C38(?:'|")`, ":3128"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")1FB6(?:'|")`, ":8118"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")22B8(?:'|")`, ":8888"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")270F(?:'|")`, ":9999"},
		{`(?:'|")\,(?:'|")PROXY_LAST_UPDATE(?:'|"):(?:'|")\d{1,3} \d{1,3}(?:'|"),(?:'|")PROXY_PORT(?:'|"):(?:'|")50(?:'|")`, ":80"},
		{`<script type=(?:'|")text/javascript(?:'|")>document\.write\((?:'|")<font class=spy2>`, ""},
		{`<span style=(?:'|")display:none(?:'|")>\d{1,3}</span>`, ""},
		{`</font>(?:'|")`, ""},
		{`<span>`, ""},
		{`</span>`, ""},
		{`\n`, ""},
		{`</td>.+?<td.+?>`, "	"},
	}
	baseDecode = []string{
		`text/javascript(?:'|")>Proxy\((?:'|")(.*?)(?:'|")\)</script>`,
	}
	base16 = []string{
		`PROXY_IP(?:'|"):(?:'|")((?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))(?:'|"),.*?,(?:'|")PROXY_PORT(?:'|"):(?:'|")([a-fA-F0-9]{1,4})(?:'|")`,
	}
	headers = []string{
		"HTTP_VIA",
		"HTTP_X_FORWARDED_FOR",
		"HTTP_FORWARDED_FOR",
		"HTTP_X_FORWARDED",
		"HTTP_FORWARDED",
		"HTTP_CLIENT_IP",
		"HTTP_FORWARDED_FOR_IP",
		"VIA",
		"X_FORWARDED_FOR",
		"FORWARDED_FOR",
		"X_FORWARDED",
		"FORWARDED",
		"CLIENT_IP",
		"FORWARDED_FOR_IP",
		"HTTP_PROXY_CONNECTION",
	}
)
<|endoftext|>


<commit_before>package dht

import (
	"math/rand"
	"testing"

	crand "crypto/rand"

	inet "github.com/jbenet/go-ipfs/net"
	mocknet "github.com/jbenet/go-ipfs/net/mock"
	peer "github.com/jbenet/go-ipfs/peer"
	routing "github.com/jbenet/go-ipfs/routing"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	u "github.com/jbenet/go-ipfs/util"
	testutil "github.com/jbenet/go-ipfs/util/testutil"

	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"

	"time"
)

func TestGetFailures(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	ctx := context.Background()
	peerstore := peer.NewPeerstore()
	local := makePeerString(t, "")
	peers := []peer.Peer{local, testutil.RandPeer()}

	nets, err := mocknet.MakeNetworks(ctx, peers)
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(ctx, peers[0], peerstore, nets[0], ds.NewMapDatastore())
	d.Update(ctx, peers[1])

	// This one should time out
	// u.POut("Timout Test\n")
	ctx1, _ := context.WithTimeout(context.Background(), time.Second)
	if _, err := d.GetValue(ctx1, u.Key("test")); err != nil {
		if err != context.DeadlineExceeded {
			t.Fatal("Got different error than we expected", err)
		}
	} else {
		t.Fatal("Did not get expected error!")
	}

	msgs := make(chan *pb.Message, 100)

	// u.POut("NotFound Test\n")
	// Reply with failures to every message
	nets[1].SetHandler(inet.ProtocolDHT, func(s inet.Stream) {
		defer s.Close()

		pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
		pbw := ggio.NewDelimitedWriter(s)

		pmes := new(pb.Message)
		if err := pbr.ReadMsg(pmes); err != nil {
			panic(err)
		}

		resp := &pb.Message{
			Type: pmes.Type,
		}
		if err := pbw.WriteMsg(resp); err != nil {
			panic(err)
		}

		msgs <- resp
	})

	// This one should fail with NotFound
	ctx2, _ := context.WithTimeout(context.Background(), time.Second)
	_, err = d.GetValue(ctx2, u.Key("test"))
	if err != nil {
		if err != routing.ErrNotFound {
			t.Fatalf("Expected ErrNotFound, got: %s", err)
		}
	} else {
		t.Fatal("expected error, got none.")
	}

	// Now we test this DHT's handleGetValue failure
	{
		typ := pb.Message_GET_VALUE
		str := "hello"
		rec, err := d.makePutRecord(u.Key(str), []byte("blah"))
		if err != nil {
			t.Fatal(err)
		}
		req := pb.Message{
			Type:   &typ,
			Key:    &str,
			Record: rec,
		}

		// u.POut("handleGetValue Test\n")
		s, err := nets[1].NewStream(inet.ProtocolDHT, peers[0])
		if err != nil {
			t.Fatal(err)
		}
		defer s.Close()

		pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
		pbw := ggio.NewDelimitedWriter(s)

		if err := pbw.WriteMsg(&req); err != nil {
			t.Fatal(err)
		}

		pmes := new(pb.Message)
		if err := pbr.ReadMsg(pmes); err != nil {
			t.Fatal(err)
		}
		if pmes.GetRecord() != nil {
			t.Fatal("shouldnt have value")
		}
		if pmes.GetProviderPeers() != nil {
			t.Fatal("shouldnt have provider peers")
		}
	}
}

// TODO: Maybe put these in some sort of "ipfs_testutil" package
func _randPeer() peer.Peer {
	id := make(peer.ID, 16)
	crand.Read(id)
	p := testutil.NewPeerWithID(id)
	return p
}

func TestNotFound(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	ctx := context.Background()
	peerstore := peer.NewPeerstore()

	var peers []peer.Peer
	for i := 0; i < 16; i++ {
		peers = append(peers, testutil.RandPeer())
	}

	nets, err := mocknet.MakeNetworks(ctx, peers)
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(ctx, peers[0], peerstore, nets[0], ds.NewMapDatastore())

	for _, p := range peers {
		d.Update(ctx, p)
	}

	// Reply with random peers to every message
	for _, neti := range nets {
		neti.SetHandler(inet.ProtocolDHT, func(s inet.Stream) {
			defer s.Close()

			pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
			pbw := ggio.NewDelimitedWriter(s)

			pmes := new(pb.Message)
			if err := pbr.ReadMsg(pmes); err != nil {
				panic(err)
			}

			switch pmes.GetType() {
			case pb.Message_GET_VALUE:
				resp := &pb.Message{Type: pmes.Type}

				ps := []peer.Peer{}
				for i := 0; i < 7; i++ {
					ps = append(ps, peers[rand.Intn(len(peers))])
				}

				resp.CloserPeers = pb.PeersToPBPeers(d.network, peers)
				if err := pbw.WriteMsg(resp); err != nil {
					panic(err)
				}

			default:
				panic("Shouldnt recieve this.")
			}
		})
	}

	ctx, _ = context.WithTimeout(ctx, time.Second*5)
	v, err := d.GetValue(ctx, u.Key("hello"))
	log.Debugf("get value got %v", v)
	if err != nil {
		switch err {
		case routing.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}

// If less than K nodes are in the entire network, it should fail when we make
// a GET rpc and nobody has the value
func TestLessThanKResponses(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	ctx := context.Background()
	peerstore := peer.NewPeerstore()

	var peers []peer.Peer
	for i := 0; i < 6; i++ {
		peers = append(peers, testutil.RandPeer())
	}

	nets, err := mocknet.MakeNetworks(ctx, peers)
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(ctx, peers[0], peerstore, nets[0], ds.NewMapDatastore())

	for i := 1; i < 5; i++ {
		d.Update(ctx, peers[i])
	}

	// Reply with random peers to every message
	for _, neti := range nets {
		neti.SetHandler(inet.ProtocolDHT, func(s inet.Stream) {
			defer s.Close()

			pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
			pbw := ggio.NewDelimitedWriter(s)

			pmes := new(pb.Message)
			if err := pbr.ReadMsg(pmes); err != nil {
				panic(err)
			}

			switch pmes.GetType() {
			case pb.Message_GET_VALUE:
				resp := &pb.Message{
					Type:        pmes.Type,
					CloserPeers: pb.PeersToPBPeers(d.network, []peer.Peer{peers[1]}),
				}

				if err := pbw.WriteMsg(resp); err != nil {
					panic(err)
				}
			default:
				panic("Shouldnt recieve this.")
			}

		})
	}

	ctx, _ = context.WithTimeout(ctx, time.Second*30)
	if _, err := d.GetValue(ctx, u.Key("hello")); err != nil {
		switch err {
		case routing.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}
<commit_msg>transition dht to mock2<commit_after>package dht

import (
	"math/rand"
	"testing"

	crand "crypto/rand"

	inet "github.com/jbenet/go-ipfs/net"
	mocknet "github.com/jbenet/go-ipfs/net/mock2"
	peer "github.com/jbenet/go-ipfs/peer"
	routing "github.com/jbenet/go-ipfs/routing"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	u "github.com/jbenet/go-ipfs/util"
	testutil "github.com/jbenet/go-ipfs/util/testutil"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"

	"time"
)

func TestGetFailures(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	ctx := context.Background()
	mn, err := mocknet.FullMeshConnected(ctx, 2)
	if err != nil {
		t.Fatal(err)
	}
	nets := mn.Nets()
	peers := mn.Peers()

	ps := peer.NewPeerstore()
	d := NewDHT(ctx, peers[0], ps, nets[0], ds.NewMapDatastore())
	d.Update(ctx, peers[1])

	// This one should time out
	// u.POut("Timout Test\n")
	ctx1, _ := context.WithTimeout(context.Background(), time.Second)
	if _, err := d.GetValue(ctx1, u.Key("test")); err != nil {
		if err != context.DeadlineExceeded {
			t.Fatal("Got different error than we expected", err)
		}
	} else {
		t.Fatal("Did not get expected error!")
	}

	msgs := make(chan *pb.Message, 100)

	// u.POut("NotFound Test\n")
	// Reply with failures to every message
	nets[1].SetHandler(inet.ProtocolDHT, func(s inet.Stream) {
		defer s.Close()

		pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
		pbw := ggio.NewDelimitedWriter(s)

		pmes := new(pb.Message)
		if err := pbr.ReadMsg(pmes); err != nil {
			panic(err)
		}

		resp := &pb.Message{
			Type: pmes.Type,
		}
		if err := pbw.WriteMsg(resp); err != nil {
			panic(err)
		}

		msgs <- resp
	})

	// This one should fail with NotFound
	ctx2, _ := context.WithTimeout(context.Background(), time.Second)
	_, err = d.GetValue(ctx2, u.Key("test"))
	if err != nil {
		if err != routing.ErrNotFound {
			t.Fatalf("Expected ErrNotFound, got: %s", err)
		}
	} else {
		t.Fatal("expected error, got none.")
	}

	// Now we test this DHT's handleGetValue failure
	{
		typ := pb.Message_GET_VALUE
		str := "hello"
		rec, err := d.makePutRecord(u.Key(str), []byte("blah"))
		if err != nil {
			t.Fatal(err)
		}
		req := pb.Message{
			Type:   &typ,
			Key:    &str,
			Record: rec,
		}

		// u.POut("handleGetValue Test\n")
		s, err := nets[1].NewStream(inet.ProtocolDHT, peers[0])
		if err != nil {
			t.Fatal(err)
		}
		defer s.Close()

		pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
		pbw := ggio.NewDelimitedWriter(s)

		if err := pbw.WriteMsg(&req); err != nil {
			t.Fatal(err)
		}

		pmes := new(pb.Message)
		if err := pbr.ReadMsg(pmes); err != nil {
			t.Fatal(err)
		}
		if pmes.GetRecord() != nil {
			t.Fatal("shouldnt have value")
		}
		if pmes.GetProviderPeers() != nil {
			t.Fatal("shouldnt have provider peers")
		}
	}
}

// TODO: Maybe put these in some sort of "ipfs_testutil" package
func _randPeer() peer.Peer {
	id := make(peer.ID, 16)
	crand.Read(id)
	p := testutil.NewPeerWithID(id)
	return p
}

func TestNotFound(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	ctx := context.Background()
	mn, err := mocknet.FullMeshConnected(ctx, 16)
	if err != nil {
		t.Fatal(err)
	}
	nets := mn.Nets()
	peers := mn.Peers()
	peerstore := peer.NewPeerstore()

	d := NewDHT(ctx, peers[0], peerstore, nets[0], ds.NewMapDatastore())

	for _, p := range peers {
		d.Update(ctx, p)
	}

	// Reply with random peers to every message
	for _, neti := range nets {
		neti.SetHandler(inet.ProtocolDHT, func(s inet.Stream) {
			defer s.Close()

			pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
			pbw := ggio.NewDelimitedWriter(s)

			pmes := new(pb.Message)
			if err := pbr.ReadMsg(pmes); err != nil {
				panic(err)
			}

			switch pmes.GetType() {
			case pb.Message_GET_VALUE:
				resp := &pb.Message{Type: pmes.Type}

				ps := []peer.Peer{}
				for i := 0; i < 7; i++ {
					ps = append(ps, peers[rand.Intn(len(peers))])
				}

				resp.CloserPeers = pb.PeersToPBPeers(d.network, peers)
				if err := pbw.WriteMsg(resp); err != nil {
					panic(err)
				}

			default:
				panic("Shouldnt recieve this.")
			}
		})
	}

	ctx, _ = context.WithTimeout(ctx, time.Second*5)
	v, err := d.GetValue(ctx, u.Key("hello"))
	log.Debugf("get value got %v", v)
	if err != nil {
		switch err {
		case routing.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}

// If less than K nodes are in the entire network, it should fail when we make
// a GET rpc and nobody has the value
func TestLessThanKResponses(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	ctx := context.Background()
	mn, err := mocknet.FullMeshConnected(ctx, 6)
	if err != nil {
		t.Fatal(err)
	}
	nets := mn.Nets()
	peers := mn.Peers()
	peerstore := peer.NewPeerstore()

	d := NewDHT(ctx, peers[0], peerstore, nets[0], ds.NewMapDatastore())

	for i := 1; i < 5; i++ {
		d.Update(ctx, peers[i])
	}

	// Reply with random peers to every message
	for _, neti := range nets {
		neti.SetHandler(inet.ProtocolDHT, func(s inet.Stream) {
			defer s.Close()

			pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
			pbw := ggio.NewDelimitedWriter(s)

			pmes := new(pb.Message)
			if err := pbr.ReadMsg(pmes); err != nil {
				panic(err)
			}

			switch pmes.GetType() {
			case pb.Message_GET_VALUE:
				resp := &pb.Message{
					Type:        pmes.Type,
					CloserPeers: pb.PeersToPBPeers(d.network, []peer.Peer{peers[1]}),
				}

				if err := pbw.WriteMsg(resp); err != nil {
					panic(err)
				}
			default:
				panic("Shouldnt recieve this.")
			}

		})
	}

	ctx, _ = context.WithTimeout(ctx, time.Second*30)
	if _, err := d.GetValue(ctx, u.Key("hello")); err != nil {
		switch err {
		case routing.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/camd67/moebot/moebot_bot/bot/permissions"
	"github.com/camd67/moebot/moebot_bot/util/db"
)

const (
	maxWrites     = 4 // Number of times to write out the time
	writeInterval = 5 // Number of seconds between each write
)

type TimerCommand struct {
	chTimers SyncChannelTimerMap
	Checker  permissions.PermissionChecker
}

func NewTimerCommand() *TimerCommand {
	tc := &TimerCommand{}
	tc.chTimers = SyncChannelTimerMap{
		RWMutex: sync.RWMutex{},
		M:       make(map[string]*ChannelTimer),
	}
	return tc
}

func (tc *TimerCommand) Execute(pack *CommPackage) {
	channelID := pack.message.ChannelID
	if len(pack.params) > 0 && strings.EqualFold(pack.params[0], "start") {
		// Make sure the user has at least mod-level permissions before starting the timer
		if tc.Checker.HasPermission(pack.message.Author.ID, pack.member.Roles, pack.guild, db.PermMod) {
			tc.chTimers.Lock()

			// If this channel timer is currently writing out, tell it to stop
			if chTimer, ok := tc.chTimers.M[channelID]; ok {
				chTimer.Lock()
				if chTimer.isWriting {
					close(chTimer.requestCh)
				}
				chTimer.Unlock()
			}

			// Create a new timer
			tc.chTimers.M[channelID] = &ChannelTimer{
				time:      time.Now(),
				writes:    0,
				isWriting: false,
				requestCh: make(chan string, 10),
			}

			tc.chTimers.Unlock()
			pack.session.ChannelMessageSend(pack.message.ChannelID, "Timer started!")
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, pack.message.Author.Mention()+", you... you don't have permission to do that!")
		}
	} else {
		tc.chTimers.RLock()
		if chTimer, ok := tc.chTimers.M[channelID]; ok {
			chTimer.Lock()
			// Reset the number of writes
			chTimer.writes = 0

			// If the time is not writing, start it
			if !chTimer.isWriting {
				go chTimer.writeTimes(pack)
				chTimer.isWriting = true
			}
			chTimer.Unlock()
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, "No timer started for this channel...")
		}
		tc.chTimers.RUnlock()
	}
}

func (ct *ChannelTimer) writeTimes(pack *CommPackage) {
	duration := time.Since(ct.time)

	// Write the time once right away
	pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(duration))
	ct.writes = 1

TimerWriting:
	for {
		select {
		case msg, chOpen := <-ct.requestCh:
			// Break out of this loop if the channel was closed (probably shouldn't be closed since there are multiple writeres) or a "stop" was issued
			if !chOpen || msg == "stop" {
				break TimerWriting
			}

		case <-time.After(time.Second * writeInterval):
			// Increment the duration and write time to the channel
			duration += (time.Second * time.Duration(writeInterval))
			go func() {
				pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(duration))
			}()

			// Exit once we've reached the max write count
			ct.Lock()
			ct.writes++
			if ct.writes >= maxWrites {
				ct.isWriting = false
				ct.Unlock()
				break TimerWriting
			}
			ct.Unlock()
		}
	}
}

// fmtDuration formats a duration into a hh:mm:ss format
func fmtDuration(dur time.Duration) string {
	remainingDur := dur.Round(time.Second)
	hours := remainingDur / time.Hour
	remainingDur -= hours * time.Hour
	minutes := remainingDur / time.Minute
	remainingDur -= minutes * time.Minute
	seconds := remainingDur / time.Second
	return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
}

func (tc *TimerCommand) GetPermLevel() db.Permission {
	return db.PermAll
}

func (tc *TimerCommand) GetCommandKeys() []string {
	return []string{"TIMER"}
}

func (tc *TimerCommand) GetCommandHelp(commPrefix string) string {
	return fmt.Sprintf("`%[1]s timer` - Checks the timestamp. Moderators may provide the `--start` option to begin start or restart the timer.", commPrefix)
}

type SyncChannelTimerMap struct {
	sync.RWMutex
	M map[string]*ChannelTimer
}

type ChannelTimer struct {
	sync.Mutex
	time      time.Time
	writes    int
	isWriting bool
	requestCh chan string
}
<commit_msg>Replace using a label to break from for loop with returns<commit_after>package commands

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/camd67/moebot/moebot_bot/bot/permissions"
	"github.com/camd67/moebot/moebot_bot/util/db"
)

const (
	maxWrites     = 4 // Number of times to write out the time
	writeInterval = 5 // Number of seconds between each write
)

type TimerCommand struct {
	chTimers SyncChannelTimerMap
	Checker  permissions.PermissionChecker
}

func NewTimerCommand() *TimerCommand {
	tc := &TimerCommand{}
	tc.chTimers = SyncChannelTimerMap{
		RWMutex: sync.RWMutex{},
		M:       make(map[string]*ChannelTimer),
	}
	return tc
}

func (tc *TimerCommand) Execute(pack *CommPackage) {
	channelID := pack.message.ChannelID
	if len(pack.params) > 0 && strings.EqualFold(pack.params[0], "start") {
		// Make sure the user has at least mod-level permissions before starting the timer
		if tc.Checker.HasPermission(pack.message.Author.ID, pack.member.Roles, pack.guild, db.PermMod) {
			tc.chTimers.Lock()

			// If this channel timer is currently writing out, tell it to stop
			if chTimer, ok := tc.chTimers.M[channelID]; ok {
				chTimer.Lock()
				if chTimer.isWriting {
					close(chTimer.requestCh)
				}
				chTimer.Unlock()
			}

			// Create a new timer
			tc.chTimers.M[channelID] = &ChannelTimer{
				time:      time.Now(),
				writes:    0,
				isWriting: false,
				requestCh: make(chan string, 10),
			}

			tc.chTimers.Unlock()
			pack.session.ChannelMessageSend(pack.message.ChannelID, "Timer started!")
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, pack.message.Author.Mention()+", you... you don't have permission to do that!")
		}
	} else {
		tc.chTimers.RLock()
		if chTimer, ok := tc.chTimers.M[channelID]; ok {
			chTimer.Lock()
			// Reset the number of writes
			chTimer.writes = 0

			// If the time is not writing, start it
			if !chTimer.isWriting {
				go chTimer.writeTimes(pack)
				chTimer.isWriting = true
			}
			chTimer.Unlock()
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, "No timer started for this channel...")
		}
		tc.chTimers.RUnlock()
	}
}

func (ct *ChannelTimer) writeTimes(pack *CommPackage) {
	duration := time.Since(ct.time)

	// Write the time once right away
	pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(duration))
	ct.writes = 1

	for {
		select {
		case msg, chOpen := <-ct.requestCh:
			// Break out of this loop if the channel was closed (probably shouldn't be closed since there are multiple writeres) or a "stop" was issued
			if !chOpen || msg == "stop" {
				return
			}

		case <-time.After(time.Second * writeInterval):
			// Increment the duration and write time to the channel
			duration += (time.Second * time.Duration(writeInterval))
			go func() {
				pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(duration))
			}()

			// Exit once we've reached the max write count
			ct.Lock()
			ct.writes++
			if ct.writes >= maxWrites {
				ct.isWriting = false
				ct.Unlock()
				return
			}
			ct.Unlock()
		}
	}
}

// fmtDuration formats a duration into a hh:mm:ss format
func fmtDuration(dur time.Duration) string {
	remainingDur := dur.Round(time.Second)
	hours := remainingDur / time.Hour
	remainingDur -= hours * time.Hour
	minutes := remainingDur / time.Minute
	remainingDur -= minutes * time.Minute
	seconds := remainingDur / time.Second
	return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
}

func (tc *TimerCommand) GetPermLevel() db.Permission {
	return db.PermAll
}

func (tc *TimerCommand) GetCommandKeys() []string {
	return []string{"TIMER"}
}

func (tc *TimerCommand) GetCommandHelp(commPrefix string) string {
	return fmt.Sprintf("`%[1]s timer` - Checks the timestamp. Moderators may provide the `--start` option to begin start or restart the timer.", commPrefix)
}

type SyncChannelTimerMap struct {
	sync.RWMutex
	M map[string]*ChannelTimer
}

type ChannelTimer struct {
	sync.Mutex
	time      time.Time
	writes    int
	isWriting bool
	requestCh chan string
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"ember/http/rpc"
	"ember/cli"
	"os"
	"strings"
)

func main() {
	args := os.Args[1:]
	c := NewEmberClient(args, "/")
	if len(args) > 1 {
		c.Call(args)
	} else {
		c.List(args)
	}
}

func (p *EmberClient) List([]string) {
	fns, err := p.client.Builtin.List()
	cli.Check(err)
	p.help(fns)
}

func (p *EmberClient) Call(args []string) {
	if len(args) == 0 {
		return
	}
	if len(args) == 2 {
		err := p.client.SimpleCall(args[0], []string{})
		cli.Check(err)
		return
	}

	if len(args) > 2 {
		for _, arg := range args[1:] {
			if !strings.Contains(arg, "=") {
				fmt.Println("usage: <bin> func [arg1]=\"\" [arg2]=\"\" ...")
				break
			}
		}
	}

	err := p.client.SimpleCall(args[0], args[1:])
	cli.Check(err)
}

func (p *EmberClient) help(fns []rpc.FnProto) {
	types := func(names []string, types []string, lb, rb string) string {
		str := lb
		for i, name := range names {
			str += types[i] + " " + name
			if i + 1 != len(names) {
				str += ", "
			}
		}
		return str + rb
	}

	for _, fn := range fns {
		fmt.Printf("  %s%v => %v\n",
			fn.Name,
			types(fn.ArgNames, fn.ArgTypes, "(", ")"),
			types(fn.ReturnNames, fn.ReturnTypes, "(", ")"))
	}
}

func NewEmberClient(args []string, path string) (p *EmberClient) {
	host, args := cli.PopArg("host", "127.0.0.1", args)
	portStr, args := cli.PopArg("port", "", args)
	var addr string
	if portStr != "" {
		addr = host + path
	} else {
		addr = host + ":" + portStr + path
	}

	client := rpc.NewClient(addr)
	p = &EmberClient{client, path, args}
	return
}

type EmberClient struct {
	client *rpc.Client
	path string
	args []string
}
<commit_msg>Fixed http code<commit_after>package main

import (
	"fmt"
	"ember/http/rpc"
	"ember/cli"
	"os"
	"strings"
)

func main() {
	args := os.Args[1:]
	c := NewEmberClient(args, "/")
	if len(args) > 1 {
		c.Call(args)
	} else {
		c.List(args)
	}
}

func (p *EmberClient) List([]string) {
	fns, err := p.client.Builtin.List()
	cli.Check(err)
	p.help(fns)
}

func (p *EmberClient) Call(args []string) {
	if len(args) == 0 {
		return
	}
	if len(args) == 2 {
		err := p.client.SimpleCall(args[0], []string{})
		cli.Check(err)
		return
	}

	if len(args) > 2 {
		for _, arg := range args[1:] {
			if !strings.Contains(arg, "=") {
				fmt.Println("usage: <bin> func [arg1]=\"\" [arg2]=\"\" ...")
				break
			}
		}
	}

	err := p.client.SimpleCall(args[0], args[1:])
	cli.Check(err)
}

func (p *EmberClient) help(fns []rpc.FnProto) {
	types := func(names []string, types []string, lb, rb string) string {
		str := lb
		for i, name := range names {
			str += types[i] + " " + name
			if i + 1 != len(names) {
				str += ", "
			}
		}
		return str + rb
	}

	for _, fn := range fns {
		fmt.Printf("  %s%v => %v\n",
			fn.Name,
			types(fn.ArgNames, fn.ArgTypes, "(", ")"),
			types(fn.ReturnNames, fn.ReturnTypes, "(", ")"))
	}
}

func NewEmberClient(args []string, path string) (p *EmberClient) {
	host, args := cli.PopArg("host", "127.0.0.1", args)
	portStr, args := cli.PopArg("port", "", args)
	var addr string
	if portStr != "" {
		addr = host + path
	} else {
		addr = host + ":" + portStr + path
	}

	client := rpc.NewClient(addr)
	p = &EmberClient{client, path, args}
	return
}

type EmberClient struct {
	client *rpc.Client
	path string
	args []string
}
<|endoftext|>


<commit_before>package torrent

import (
	"errors"
	"io"
	"os"
)

// Accesses torrent data via a client.
type Reader struct {
	t          *Torrent
	pos        int64
	responsive bool
	readahead  int64
}

var _ io.ReadCloser = &Reader{}

// Don't wait for pieces to complete and be verified. Read calls return as
// soon as they can when the underlying chunks become available.
func (r *Reader) SetResponsive() {
	r.responsive = true
}

// Configure the number of bytes ahead of a read that should also be
// prioritized in preparation for further reads.
func (r *Reader) SetReadahead(readahead int64) {
	r.readahead = readahead
}

func (r *Reader) raisePriorities(off int64, n int) {
	if r.responsive {
		r.t.cl.addUrgentRequests(r.t.torrent, off, n)
	}
	if !r.responsive || r.readahead != 0 {
		r.t.cl.readRaisePiecePriorities(r.t.torrent, off, int64(n)+r.readahead)
	}
}

func (r *Reader) readable(off int64) (ret bool) {
	// log.Println("readable", off)
	// defer func() {
	// 	log.Println("readable", ret)
	// }()
	req, ok := r.t.offsetRequest(off)
	if !ok {
		panic(off)
	}
	if r.responsive {
		return r.t.haveChunk(req)
	}
	return r.t.pieceComplete(int(req.Index))
}

// How many bytes are available to read. Max is the most we could require.
func (r *Reader) available(off, max int64) (ret int64) {
	for max > 0 {
		req, ok := r.t.offsetRequest(off)
		if !ok {
			break
		}
		if !r.t.haveChunk(req) {
			break
		}
		len1 := int64(req.Length) - (off - r.t.requestOffset(req))
		max -= len1
		ret += len1
		off += len1
	}
	// Ensure that ret hasn't exceeded our original max.
	if max < 0 {
		ret += max
	}
	return
}

func (r *Reader) waitReadable(off int64) {
	r.t.Pieces[off/int64(r.t.usualPieceSize())].Event.Wait()
}

func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
	for {
		var n1 int
		n1, err = r.readAt(b, off)
		n += n1
		b = b[n1:]
		off += int64(n1)
		if err != nil || len(b) == 0 || n1 == 0 {
			return
		}
	}
}

func (r *Reader) Read(b []byte) (n int, err error) {
	n, err = r.readAt(b, r.pos)
	r.pos += int64(n)
	return
}

// Must only return EOF at the end of the torrent.
func (r *Reader) readAt(b []byte, pos int64) (n int, err error) {
	// defer func() {
	// 	log.Println(pos, n, err)
	// }()
	r.t.cl.mu.Lock()
	defer r.t.cl.mu.Unlock()
	maxLen := r.t.torrent.Info.TotalLength() - pos
	if maxLen <= 0 {
		err = io.EOF
		return
	}
	if int64(len(b)) > maxLen {
		b = b[:maxLen]
	}
again:
	r.raisePriorities(pos, len(b))
	for !r.readable(pos) {
		r.raisePriorities(pos, len(b))
		r.waitReadable(pos)
	}
	avail := r.available(pos, int64(len(b)))
	// log.Println("available", avail)
	b1 := b[:avail]
	pi := int(pos / r.t.Info().PieceLength)
	tp := r.t.torrent.Pieces[pi]
	ip := r.t.Info().Piece(pi)
	po := pos % ip.Length()
	if int64(len(b1)) > ip.Length()-po {
		b1 = b1[:ip.Length()-po]
	}
	tp.pendingWrites.Wait()
	n, err = dataReadAt(r.t.data, b1, pos)
	if n != 0 {
		err = nil
		return
	}
	if err == io.ErrUnexpectedEOF {
		goto again
	}
	return
}

func (r *Reader) Close() error {
	r.t = nil
	return nil
}

func (r *Reader) Seek(off int64, whence int) (ret int64, err error) {
	switch whence {
	case os.SEEK_SET:
		r.pos = off
	case os.SEEK_CUR:
		r.pos += off
	case os.SEEK_END:
		r.pos = r.t.torrent.Info.TotalLength() + off
	default:
		err = errors.New("bad whence")
	}
	ret = r.pos
	return
}
<commit_msg>Hold client lock as little as possible in Reader.readAt<commit_after>package torrent

import (
	"errors"
	"io"
	"os"
)

// Accesses torrent data via a client.
type Reader struct {
	t          *Torrent
	pos        int64
	responsive bool
	readahead  int64
}

var _ io.ReadCloser = &Reader{}

// Don't wait for pieces to complete and be verified. Read calls return as
// soon as they can when the underlying chunks become available.
func (r *Reader) SetResponsive() {
	r.responsive = true
}

// Configure the number of bytes ahead of a read that should also be
// prioritized in preparation for further reads.
func (r *Reader) SetReadahead(readahead int64) {
	r.readahead = readahead
}

func (r *Reader) raisePriorities(off int64, n int) {
	if r.responsive {
		r.t.cl.addUrgentRequests(r.t.torrent, off, n)
	}
	if !r.responsive || r.readahead != 0 {
		r.t.cl.readRaisePiecePriorities(r.t.torrent, off, int64(n)+r.readahead)
	}
}

func (r *Reader) readable(off int64) (ret bool) {
	// log.Println("readable", off)
	// defer func() {
	// 	log.Println("readable", ret)
	// }()
	req, ok := r.t.offsetRequest(off)
	if !ok {
		panic(off)
	}
	if r.responsive {
		return r.t.haveChunk(req)
	}
	return r.t.pieceComplete(int(req.Index))
}

// How many bytes are available to read. Max is the most we could require.
func (r *Reader) available(off, max int64) (ret int64) {
	for max > 0 {
		req, ok := r.t.offsetRequest(off)
		if !ok {
			break
		}
		if !r.t.haveChunk(req) {
			break
		}
		len1 := int64(req.Length) - (off - r.t.requestOffset(req))
		max -= len1
		ret += len1
		off += len1
	}
	// Ensure that ret hasn't exceeded our original max.
	if max < 0 {
		ret += max
	}
	return
}

func (r *Reader) waitReadable(off int64) {
	r.t.Pieces[off/int64(r.t.usualPieceSize())].Event.Wait()
}

func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
	for {
		var n1 int
		n1, err = r.readAt(b, off)
		n += n1
		b = b[n1:]
		off += int64(n1)
		if err != nil || len(b) == 0 || n1 == 0 {
			return
		}
	}
}

func (r *Reader) Read(b []byte) (n int, err error) {
	n, err = r.readAt(b, r.pos)
	r.pos += int64(n)
	return
}

// Must only return EOF at the end of the torrent.
func (r *Reader) readAt(b []byte, pos int64) (n int, err error) {
	// defer func() {
	// 	log.Println(pos, n, err)
	// }()
	maxLen := r.t.torrent.Info.TotalLength() - pos
	if maxLen <= 0 {
		err = io.EOF
		return
	}
	if int64(len(b)) > maxLen {
		b = b[:maxLen]
	}
again:
	r.t.cl.mu.Lock()
	r.raisePriorities(pos, len(b))
	for !r.readable(pos) {
		r.raisePriorities(pos, len(b))
		r.waitReadable(pos)
	}
	avail := r.available(pos, int64(len(b)))
	// log.Println("available", avail)
	r.t.cl.mu.Unlock()
	b1 := b[:avail]
	pi := int(pos / r.t.Info().PieceLength)
	tp := r.t.torrent.Pieces[pi]
	ip := r.t.Info().Piece(pi)
	po := pos % ip.Length()
	if int64(len(b1)) > ip.Length()-po {
		b1 = b1[:ip.Length()-po]
	}
	tp.pendingWrites.Wait()
	n, err = dataReadAt(r.t.data, b1, pos)
	if n != 0 {
		err = nil
		return
	}
	if err == io.ErrUnexpectedEOF {
		goto again
	}
	return
}

func (r *Reader) Close() error {
	r.t = nil
	return nil
}

func (r *Reader) Seek(off int64, whence int) (ret int64, err error) {
	switch whence {
	case os.SEEK_SET:
		r.pos = off
	case os.SEEK_CUR:
		r.pos += off
	case os.SEEK_END:
		r.pos = r.t.torrent.Info.TotalLength() + off
	default:
		err = errors.New("bad whence")
	}
	ret = r.pos
	return
}
<|endoftext|>