<commit_before>//
// Copyright 2020, Eric Stevens
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// GroupHook represents a GitLab group hook.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/groups.html#list-group-hooks
type GroupHook struct {
	ID                       int        `json:"id"`
	URL                      string     `json:"url"`
	GroupID                  int        `json:"group_id"`
	PushEvents               bool       `json:"push_events"`
	IssuesEvents             bool       `json:"issues_events"`
	ConfidentialIssuesEvents bool       `json:"confidential_issues_events"`
	ConfidentialNoteEvents   bool       `json:"confidential_note_events"`
	MergeRequestsEvents      bool       `json:"merge_requests_events"`
	TagPushEvents            bool       `json:"tag_push_events"`
	NoteEvents               bool       `json:"note_events"`
	JobEvents                bool       `json:"job_events"`
	PipelineEvents           bool       `json:"pipeline_events"`
	WikiPageEvents           bool       `json:"wiki_page_events"`
	EnableSSLVerification    bool       `json:"enable_ssl_verification"`
	CreatedAt                *time.Time `json:"created_at"`
}

// ListGroupHooks gets a list of group hooks.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/groups.html#list-group-hooks
func (s *GroupsService) ListGroupHooks(gid interface{}) ([]*GroupHook, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks", pathEscape(group))

	req, err := s.client.NewRequest("GET", u, nil, nil)
	if err != nil {
		return nil, nil, err
	}
	var gh []*GroupHook
	resp, err := s.client.Do(req, &gh)
	if err != nil {
		return nil, resp, err
	}

	return gh, resp, err
}

// GetGroupHook gets a specific hook for a group.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/groups.html#get-group-hook
func (s *GroupsService) GetGroupHook(pid interface{}, hook int, options ...OptionFunc) (*GroupHook, *Response, error) {
	group, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks/%d", pathEscape(group), hook)

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	gh := new(GroupHook)
	resp, err := s.client.Do(req, gh)
	if err != nil {
		return nil, resp, err
	}

	return gh, resp, err
}

// AddGroupHookOptions represents the available AddGroupHook() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/groups.html#add-group-hook
type AddGroupHookOptions struct {
	URL                      *string `url:"url,omitempty" json:"url,omitempty"`
	PushEvents               *bool   `url:"push_events,omitempty"  json:"push_events,omitempty"`
	IssuesEvents             *bool   `url:"issues_events,omitempty"  json:"issues_events,omitempty"`
	ConfidentialIssuesEvents *bool   `url:"confidential_issues_events,omitempty"  json:"confidential_issues_events,omitempty"`
	ConfidentialNoteEvents   *bool   `url:"confidential_note_events,omitempty"  json:"confidential_note_events,omitempty"`
	MergeRequestsEvents      *bool   `url:"merge_requests_events,omitempty"  json:"merge_requests_events,omitempty"`
	TagPushEvents            *bool   `url:"tag_push_events,omitempty"  json:"tag_push_events,omitempty"`
	NoteEvents               *bool   `url:"note_events,omitempty"  json:"note_events,omitempty"`
	JobEvents                *bool   `url:"job_events,omitempty"  json:"job_events,omitempty"`
	PipelineEvents           *bool   `url:"pipeline_events,omitempty"  json:"pipeline_events,omitempty"`
	WikiPageEvents           *bool   `url:"wiki_page_events,omitempty"  json:"wiki_page_events,omitempty"`
	EnableSSLVerification    *bool   `url:"enable_ssl_verification,omitempty"  json:"enable_ssl_verification,omitempty"`
	Token                    *string `url:"token,omitempty" json:"token,omitempty"`
}

// AddGroupHook create a new group scoped webhook.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/groups.html#add-group-hook
func (s *GroupsService) AddGroupHook(gid interface{}, opt *AddGroupHookOptions, options ...OptionFunc) (*GroupHook, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks", pathEscape(group))

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	gh := new(GroupHook)
	resp, err := s.client.Do(req, gh)
	if err != nil {
		return nil, resp, err
	}

	return gh, resp, err
}

// EditGroupHookOptions represents the available EditGroupHook() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/groups.html#edit-group-hook
type EditGroupHookOptions struct {
	URL                      *string `url:"url,omitempty" json:"url,omitempty"`
	PushEvents               *bool   `url:"push_events,omitempty" json:"push_events,omitempty"`
	IssuesEvents             *bool   `url:"issues_events,omitempty" json:"issues_events,omitempty"`
	ConfidentialIssuesEvents *bool   `url:"confidential_issues_events,omitempty" json:"confidential_issues_events,omitempty"`
	ConfidentialNoteEvents   *bool   `url:"confidential_note_events,omitempty" json:"confidential_note_events,omitempty"`
	MergeRequestsEvents      *bool   `url:"merge_requests_events,omitempty" json:"merge_requests_events,omitempty"`
	TagPushEvents            *bool   `url:"tag_push_events,omitempty" json:"tag_push_events,omitempty"`
	NoteEvents               *bool   `url:"note_events,omitempty" json:"note_events,omitempty"`
	JobEvents                *bool   `url:"job_events,omitempty" json:"job_events,omitempty"`
	PipelineEvents           *bool   `url:"pipeline_events,omitempty" json:"pipeline_events,omitempty"`
	WikiPageEvents           *bool   `url:"wiki_page_events,omitempty" json:"wiki_page_events,omitempty"`
	EnableSSLVerification    *bool   `url:"enable_ssl_verification,omitempty" json:"enable_ssl_verification,omitempty"`
	Token                    *string `url:"token,omitempty" json:"token,omitempty"`
}

// EditGroupHook edits a hook for a specified group.
//
// Gitlab API docs:
// https://gitlab.com/help/api/groups.md#edit-group-hook
func (s *GroupsService) EditGroupHook(pid interface{}, hook int, opt *EditGroupHookOptions, options ...OptionFunc) (*GroupHook, *Response, error) {
	group, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks/%d", pathEscape(group), hook)

	req, err := s.client.NewRequest("PUT", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	ph := &GroupHook{}
	resp, err := s.client.Do(req, ph)
	if err != nil {
		return nil, resp, err
	}

	return ph, resp, err
}

// DeleteGroupHook removes a hook from a group. This is an idempotent
// method and can be called multiple times. Either the hook is available or not.
//
// GitLab API docs:
// https://gitlab.com/help/api/groups.md#delete-group-hook
func (s *GroupsService) DeleteGroupHook(pid interface{}, hook int, options ...OptionFunc) (*Response, error) {
	group, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks/%d", pathEscape(group), hook)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<commit_msg>Update group_hooks.go<commit_after>//
// Copyright 2020, Eric Stevens
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// GroupHook represents a GitLab group hook.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/groups.html#list-group-hooks
type GroupHook struct {
	ID                       int        `json:"id"`
	URL                      string     `json:"url"`
	GroupID                  int        `json:"group_id"`
	PushEvents               bool       `json:"push_events"`
	IssuesEvents             bool       `json:"issues_events"`
	ConfidentialIssuesEvents bool       `json:"confidential_issues_events"`
	ConfidentialNoteEvents   bool       `json:"confidential_note_events"`
	MergeRequestsEvents      bool       `json:"merge_requests_events"`
	TagPushEvents            bool       `json:"tag_push_events"`
	NoteEvents               bool       `json:"note_events"`
	JobEvents                bool       `json:"job_events"`
	PipelineEvents           bool       `json:"pipeline_events"`
	WikiPageEvents           bool       `json:"wiki_page_events"`
	EnableSSLVerification    bool       `json:"enable_ssl_verification"`
	CreatedAt                *time.Time `json:"created_at"`
}

// ListGroupHooks gets a list of group hooks.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/groups.html#list-group-hooks
func (s *GroupsService) ListGroupHooks(gid interface{}) ([]*GroupHook, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks", pathEscape(group))

	req, err := s.client.NewRequest("GET", u, nil, nil)
	if err != nil {
		return nil, nil, err
	}
	var gh []*GroupHook
	resp, err := s.client.Do(req, &gh)
	if err != nil {
		return nil, resp, err
	}

	return gh, resp, err
}

// GetGroupHook gets a specific hook for a group.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/groups.html#get-group-hook
func (s *GroupsService) GetGroupHook(pid interface{}, hook int, options ...OptionFunc) (*GroupHook, *Response, error) {
	group, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks/%d", pathEscape(group), hook)

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	gh := new(GroupHook)
	resp, err := s.client.Do(req, gh)
	if err != nil {
		return nil, resp, err
	}

	return gh, resp, err
}

// AddGroupHookOptions represents the available AddGroupHook() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/groups.html#add-group-hook
type AddGroupHookOptions struct {
	URL                      *string `url:"url,omitempty" json:"url,omitempty"`
	PushEvents               *bool   `url:"push_events,omitempty"  json:"push_events,omitempty"`
	IssuesEvents             *bool   `url:"issues_events,omitempty"  json:"issues_events,omitempty"`
	ConfidentialIssuesEvents *bool   `url:"confidential_issues_events,omitempty"  json:"confidential_issues_events,omitempty"`
	ConfidentialNoteEvents   *bool   `url:"confidential_note_events,omitempty"  json:"confidential_note_events,omitempty"`
	MergeRequestsEvents      *bool   `url:"merge_requests_events,omitempty"  json:"merge_requests_events,omitempty"`
	TagPushEvents            *bool   `url:"tag_push_events,omitempty"  json:"tag_push_events,omitempty"`
	NoteEvents               *bool   `url:"note_events,omitempty"  json:"note_events,omitempty"`
	JobEvents                *bool   `url:"job_events,omitempty"  json:"job_events,omitempty"`
	PipelineEvents           *bool   `url:"pipeline_events,omitempty"  json:"pipeline_events,omitempty"`
	WikiPageEvents           *bool   `url:"wiki_page_events,omitempty"  json:"wiki_page_events,omitempty"`
	EnableSSLVerification    *bool   `url:"enable_ssl_verification,omitempty"  json:"enable_ssl_verification,omitempty"`
	Token                    *string `url:"token,omitempty" json:"token,omitempty"`
}

// AddGroupHook create a new group scoped webhook.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/groups.html#add-group-hook
func (s *GroupsService) AddGroupHook(gid interface{}, opt *AddGroupHookOptions, options ...OptionFunc) (*GroupHook, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks", pathEscape(group))

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	gh := new(GroupHook)
	resp, err := s.client.Do(req, gh)
	if err != nil {
		return nil, resp, err
	}

	return gh, resp, err
}

// EditGroupHookOptions represents the available EditGroupHook() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/groups.html#edit-group-hook
type EditGroupHookOptions struct {
	URL                      *string `url:"url,omitempty" json:"url,omitempty"`
	PushEvents               *bool   `url:"push_events,omitempty" json:"push_events,omitempty"`
	IssuesEvents             *bool   `url:"issues_events,omitempty" json:"issues_events,omitempty"`
	ConfidentialIssuesEvents *bool   `url:"confidential_issues_events,omitempty" json:"confidential_issues_events,omitempty"`
	ConfidentialNoteEvents   *bool   `url:"confidential_note_events,omitempty" json:"confidential_note_events,omitempty"`
	MergeRequestsEvents      *bool   `url:"merge_requests_events,omitempty" json:"merge_requests_events,omitempty"`
	TagPushEvents            *bool   `url:"tag_push_events,omitempty" json:"tag_push_events,omitempty"`
	NoteEvents               *bool   `url:"note_events,omitempty" json:"note_events,omitempty"`
	JobEvents                *bool   `url:"job_events,omitempty" json:"job_events,omitempty"`
	PipelineEvents           *bool   `url:"pipeline_events,omitempty" json:"pipeline_events,omitempty"`
	WikiPageEvents           *bool   `url:"wiki_page_events,omitempty" json:"wiki_page_events,omitempty"`
	EnableSSLVerification    *bool   `url:"enable_ssl_verification,omitempty" json:"enable_ssl_verification,omitempty"`
	Token                    *string `url:"token,omitempty" json:"token,omitempty"`
}

// EditGroupHook edits a hook for a specified group.
//
// Gitlab API docs:
// https://docs.gitlab.com/ce/api/groups.html#edit-group-hook
func (s *GroupsService) EditGroupHook(pid interface{}, hook int, opt *EditGroupHookOptions, options ...OptionFunc) (*GroupHook, *Response, error) {
	group, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks/%d", pathEscape(group), hook)

	req, err := s.client.NewRequest("PUT", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	ph := &GroupHook{}
	resp, err := s.client.Do(req, ph)
	if err != nil {
		return nil, resp, err
	}

	return ph, resp, err
}

// DeleteGroupHook removes a hook from a group. This is an idempotent
// method and can be called multiple times. Either the hook is available or not.
//
// GitLab API docs:
// https://gitlab.com/help/api/groups.md#delete-group-hook
func (s *GroupsService) DeleteGroupHook(pid interface{}, hook int, options ...OptionFunc) (*Response, error) {
	group, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("groups/%s/hooks/%d", pathEscape(group), hook)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<|endoftext|>


<commit_before>package rotatelogs

import (
  "fmt"
  "io/ioutil"
  "log"
  "os"
  "path/filepath"
  "strings"
  "testing"
  "time"
)

func TestGenFilename(t *testing.T) {
  // Mock time
  ts := []time.Time {
    time.Time {},
    (time.Time {}).Add(24 * time.Hour),
  }

  var old = CurrentTime
  defer func() { CurrentTime = old }()
  for _, xt := range ts {
    CurrentTime = func() (time.Time) { return xt }
    rl := NewRotateLogs("/path/to/%Y/%m/%d")

    fn, err := rl.GenFilename()
    if err != nil {
      t.Errorf("Failed to generate filename: %s", err)
    }

    expected := fmt.Sprintf("/path/to/%04d/%02d/%02d",
      xt.Year(),
      xt.Month(),
      xt.Day(),
    )

    if fn != expected {
      t.Errorf("Failed to match fn (%s)", fn)
    }
    t.Logf("fn = %s", fn)
  }
}

func TestLogFilePattern (t *testing.T) {
  rl := NewRotateLogs("/path/to/%Y/%m/%d")
  pattern := rl.LogFilePattern()
  if pattern != "/path/to/*/*/*" {
    t.Errorf("Failed to match pattern (%s)", pattern)
  }
}

func TestLogRotate (t *testing.T) {
  dir, err := ioutil.TempDir("", "file-rotatelogs-test")
  if err != nil {
    t.Errorf("Failed to create temporary directory: %s", err)
  }
  defer os.RemoveAll(dir)

  // Change current time, so we can safely purge old logs
  old := CurrentTime
  dummyTime := time.Now().Add(-7 * 86400 * time.Second)
  dummyTime  = dummyTime.Add(time.Duration(-1 * dummyTime.Nanosecond()))
  defer func() { CurrentTime = old }()
  CurrentTime = func() (time.Time) { return dummyTime }

  rl := NewRotateLogs(filepath.Join(dir, "log%Y%m%d%H%M%S"))
  rl.MaxAge = 86400 * time.Second
  rl.LinkName = filepath.Join(dir, "log")

  str := "Hello, World"
  n, err := rl.Write([]byte(str))
  if n != len(str) {
    t.Errorf("Could not write %d bytes (wrote %d bytes)", len(str), n)
  }

  if err != nil {
    t.Errorf("Failed to Write() to log: %s", err)
  }

  fn := rl.CurrentFileName()
  if fn == "" {
    t.Errorf("Could not get filename %s", fn)
  }

  content, err := ioutil.ReadFile(fn)
  if err != nil {
    t.Errorf("Failed to read file %s: %s", fn, err)
  }

  if string(content) != str {
    t.Errorf(`File content does not match (was "%s")`,  content)
  }

  err = os.Chtimes(fn, dummyTime, dummyTime)
  if err != nil {
    t.Errorf("Failed to change access/modification times for %s: %s", fn, err)
  }

  fi, err := os.Stat(fn)
  if err != nil {
    t.Errorf("Failed to stat %s: %s", fn, err)
  }

  if ! fi.ModTime().Equal(dummyTime) {
    t.Errorf("Failed to chtime for %s (expected %s, got %s)", fn, fi.ModTime(), dummyTime)
  }

  CurrentTime = old

  rl.Write([]byte(str))
  newfn := rl.CurrentFileName()
  if newfn == fn {
    t.Errorf(`New file name and old file name should not match ("%s" != "%s")`, fn, newfn)
  }

  content, err = ioutil.ReadFile(newfn)
  if err != nil {
    t.Errorf("Failed to read file %s: %s", newfn, err)
  }

  if string(content) != str {
    t.Errorf(`File content does not match (was "%s")`,  content)
  }

  // fn was declared above, before mocking CurrentTime
  // Old files should have been unlinked
  _, err = os.Stat(fn)
  if err == nil {
    t.Errorf("Stat succeeded (should have failed) %s: %s", fn, err)
  }

  linkDest, err := os.Readlink(rl.LinkName)
  if err != nil {
    t.Errorf("Failed to readlink %s: %s", rl.LinkName, err)
  }

  if linkDest != newfn {
    t.Errorf(`Symlink destination does not match expected filename ("%s" != "%s")`, newfn, linkDest)
  }
}

func TestLogSetOutput (t *testing.T) {
  dir, err := ioutil.TempDir("", "file-rotatelogs-test")
  if err != nil {
    t.Errorf("Failed to create temporary directory: %s", err)
  }
  defer os.RemoveAll(dir)

  rl := NewRotateLogs(filepath.Join(dir, "log%Y%m%d%H%M%S"))

  log.SetOutput(rl)
  defer log.SetOutput(os.Stderr)

  str := "Hello, World"
  log.Print(str)

  fn := rl.CurrentFileName()
  if fn == "" {
    t.Errorf("Could not get filename %s", fn)
  }

  content, err := ioutil.ReadFile(fn)
  if err != nil {
    t.Errorf("Failed to read file %s: %s", fn, err)
  }

  if ! strings.Contains(string(content), str) {
    t.Errorf(`File content does not contain "%s" (was "%s")`, str, content)
  }
}<commit_msg>see if sleeping will do any change<commit_after>package rotatelogs

import (
  "fmt"
  "io/ioutil"
  "log"
  "os"
  "path/filepath"
  "strings"
  "testing"
  "time"
)

func TestGenFilename(t *testing.T) {
  // Mock time
  ts := []time.Time {
    time.Time {},
    (time.Time {}).Add(24 * time.Hour),
  }

  var old = CurrentTime
  defer func() { CurrentTime = old }()
  for _, xt := range ts {
    CurrentTime = func() (time.Time) { return xt }
    rl := NewRotateLogs("/path/to/%Y/%m/%d")

    fn, err := rl.GenFilename()
    if err != nil {
      t.Errorf("Failed to generate filename: %s", err)
    }

    expected := fmt.Sprintf("/path/to/%04d/%02d/%02d",
      xt.Year(),
      xt.Month(),
      xt.Day(),
    )

    if fn != expected {
      t.Errorf("Failed to match fn (%s)", fn)
    }
    t.Logf("fn = %s", fn)
  }
}

func TestLogFilePattern (t *testing.T) {
  rl := NewRotateLogs("/path/to/%Y/%m/%d")
  pattern := rl.LogFilePattern()
  if pattern != "/path/to/*/*/*" {
    t.Errorf("Failed to match pattern (%s)", pattern)
  }
}

func TestLogRotate (t *testing.T) {
  dir, err := ioutil.TempDir("", "file-rotatelogs-test")
  if err != nil {
    t.Errorf("Failed to create temporary directory: %s", err)
  }
  defer os.RemoveAll(dir)

  // Change current time, so we can safely purge old logs
  old := CurrentTime
  dummyTime := time.Now().Add(-7 * 86400 * time.Second)
  dummyTime  = dummyTime.Add(time.Duration(-1 * dummyTime.Nanosecond()))
  defer func() { CurrentTime = old }()
  CurrentTime = func() (time.Time) { return dummyTime }

  rl := NewRotateLogs(filepath.Join(dir, "log%Y%m%d%H%M%S"))
  rl.MaxAge = 86400 * time.Second
  rl.LinkName = filepath.Join(dir, "log")

  str := "Hello, World"
  n, err := rl.Write([]byte(str))
  if n != len(str) {
    t.Errorf("Could not write %d bytes (wrote %d bytes)", len(str), n)
  }

  if err != nil {
    t.Errorf("Failed to Write() to log: %s", err)
  }

  fn := rl.CurrentFileName()
  if fn == "" {
    t.Errorf("Could not get filename %s", fn)
  }

  content, err := ioutil.ReadFile(fn)
  if err != nil {
    t.Errorf("Failed to read file %s: %s", fn, err)
  }

  if string(content) != str {
    t.Errorf(`File content does not match (was "%s")`,  content)
  }

  err = os.Chtimes(fn, dummyTime, dummyTime)
  if err != nil {
    t.Errorf("Failed to change access/modification times for %s: %s", fn, err)
  }

  fi, err := os.Stat(fn)
  if err != nil {
    t.Errorf("Failed to stat %s: %s", fn, err)
  }

  if ! fi.ModTime().Equal(dummyTime) {
    t.Errorf("Failed to chtime for %s (expected %s, got %s)", fn, fi.ModTime(), dummyTime)
  }

  CurrentTime = old

  // This next Write() should trigger Rotate()
  rl.Write([]byte(str))
  newfn := rl.CurrentFileName()
  if newfn == fn {
    t.Errorf(`New file name and old file name should not match ("%s" != "%s")`, fn, newfn)
  }

  content, err = ioutil.ReadFile(newfn)
  if err != nil {
    t.Errorf("Failed to read file %s: %s", newfn, err)
  }

  if string(content) != str {
    t.Errorf(`File content does not match (was "%s")`,  content)
  }

  time.Sleep(1 * time.Second)

  // fn was declared above, before mocking CurrentTime
  // Old files should have been unlinked
  _, err = os.Stat(fn)
  if err == nil {
    t.Errorf("Stat succeeded (should have failed) %s: %s", fn, err)
  }

  linkDest, err := os.Readlink(rl.LinkName)
  if err != nil {
    t.Errorf("Failed to readlink %s: %s", rl.LinkName, err)
  }

  if linkDest != newfn {
    t.Errorf(`Symlink destination does not match expected filename ("%s" != "%s")`, newfn, linkDest)
  }
}

func TestLogSetOutput (t *testing.T) {
  dir, err := ioutil.TempDir("", "file-rotatelogs-test")
  if err != nil {
    t.Errorf("Failed to create temporary directory: %s", err)
  }
  defer os.RemoveAll(dir)

  rl := NewRotateLogs(filepath.Join(dir, "log%Y%m%d%H%M%S"))

  log.SetOutput(rl)
  defer log.SetOutput(os.Stderr)

  str := "Hello, World"
  log.Print(str)

  fn := rl.CurrentFileName()
  if fn == "" {
    t.Errorf("Could not get filename %s", fn)
  }

  content, err := ioutil.ReadFile(fn)
  if err != nil {
    t.Errorf("Failed to read file %s: %s", fn, err)
  }

  if ! strings.Contains(string(content), str) {
    t.Errorf(`File content does not contain "%s" (was "%s")`, str, content)
  }
}<|endoftext|>


<commit_before>//go:build linux && cgo
// +build linux,cgo

package netutils

import (
	"fmt"
	"io"
	"net"
	"os"
	"strings"
	"unsafe"

	"github.com/lxc/lxd/shared/api"
)

/*
#include "unixfd.h"
#include "netns_getifaddrs.c"
*/
import "C"

// Allow the caller to set expectations.

// UnixFdsAcceptExact will only succeed if the exact amount of fds has been
// received  (unless combined with UNIX_FDS_ACCEPT_NONE).
const UnixFdsAcceptExact uint = C.UNIX_FDS_ACCEPT_EXACT

// UnixFdsAcceptLess will also succeed if less than the requested number of fd
// has been received.
// If the UNIX_FDS_ACCEPT_NONE flag is not raised than at least one fd must be
// received.
const UnixFdsAcceptLess uint = C.UNIX_FDS_ACCEPT_LESS

// UnixFdsAcceptMore will also succeed if more than the requested number of fds
// have been received. Any additional fds will be silently closed.
// If the UNIX_FDS_ACCEPT_NONE flag is not raised than at least one fd must be
// received.
const UnixFdsAcceptMore uint = C.UNIX_FDS_ACCEPT_MORE

// UnixFdsAcceptNone can be specified with any of the above flags and indicates
// that the caller will accept no file descriptors to be received.
const UnixFdsAcceptNone uint = C.UNIX_FDS_ACCEPT_NONE

// UnixFdsAcceptMask is the value of all the above flags or-ed together.
const UnixFdsAcceptMask uint = C.UNIX_FDS_ACCEPT_MASK

// Allow the callee to report back what happened. Only one of those will ever
// be set.

// UnixFdsReceivedExact indicates that the exact number of fds was received.
const UnixFdsReceivedExact uint = C.UNIX_FDS_RECEIVED_EXACT

// UnixFdsReceivedLess indicates that less than the requested number of fd has
// been received.
const UnixFdsReceivedLess uint = C.UNIX_FDS_RECEIVED_LESS

// UnixFdsReceivedMore indicates that more than the requested number of fd has
// been received.
const UnixFdsReceivedMore uint = C.UNIX_FDS_RECEIVED_MORE

// UnixFdsReceivedNone indicates that no fds have been received.
const UnixFdsReceivedNone uint = C.UNIX_FDS_RECEIVED_NONE

// NetnsGetifaddrs returns a map of InstanceStateNetwork for a particular process.
func NetnsGetifaddrs(initPID int32) (map[string]api.InstanceStateNetwork, error) {
	var netnsidAware C.bool
	var ifaddrs *C.struct_netns_ifaddrs
	var netnsID C.__s32

	if initPID > 0 {
		f, err := os.Open(fmt.Sprintf("/proc/%d/ns/net", initPID))
		if err != nil {
			return nil, err
		}
		defer f.Close()

		netnsID = C.netns_get_nsid(C.__s32(f.Fd()))
		if netnsID < 0 {
			return nil, fmt.Errorf("Failed to retrieve network namespace id")
		}
	} else {
		netnsID = -1
	}

	ret := C.netns_getifaddrs(&ifaddrs, netnsID, &netnsidAware)
	if ret < 0 {
		return nil, fmt.Errorf("Failed to retrieve network interfaces and addresses")
	}
	defer C.netns_freeifaddrs(ifaddrs)

	if netnsID >= 0 && !netnsidAware {
		return nil, fmt.Errorf("Netlink requests are not fully network namespace id aware")
	}

	// We're using the interface name as key here but we should really
	// switch to the ifindex at some point to handle ip aliasing correctly.
	networks := map[string]api.InstanceStateNetwork{}

	for addr := ifaddrs; addr != nil; addr = addr.ifa_next {
		var address [C.INET6_ADDRSTRLEN]C.char
		addNetwork, networkExists := networks[C.GoString(addr.ifa_name)]
		if !networkExists {
			addNetwork = api.InstanceStateNetwork{
				Addresses: []api.InstanceStateNetworkAddress{},
				Counters:  api.InstanceStateNetworkCounters{},
			}
		}

		// Interface flags
		netState := "down"
		netType := "unknown"

		if (addr.ifa_flags & C.IFF_BROADCAST) > 0 {
			netType = "broadcast"
		}

		if (addr.ifa_flags & C.IFF_LOOPBACK) > 0 {
			netType = "loopback"
		}

		if (addr.ifa_flags & C.IFF_POINTOPOINT) > 0 {
			netType = "point-to-point"
		}

		if (addr.ifa_flags & C.IFF_UP) > 0 {
			netState = "up"
		}
		addNetwork.State = netState
		addNetwork.Type = netType
		addNetwork.Mtu = int(addr.ifa_mtu)

		if initPID != 0 && int(addr.ifa_ifindex_peer) > 0 {
			hostInterface, err := net.InterfaceByIndex(int(addr.ifa_ifindex_peer))
			if err == nil {
				addNetwork.HostName = hostInterface.Name
			}
		}

		// Addresses
		if addr.ifa_addr != nil && (addr.ifa_addr.sa_family == C.AF_INET || addr.ifa_addr.sa_family == C.AF_INET6) {
			family := "inet"
			if addr.ifa_addr.sa_family == C.AF_INET6 {
				family = "inet6"
			}

			addrPtr := C.get_addr_ptr(addr.ifa_addr)
			if addrPtr == nil {
				return nil, fmt.Errorf("Failed to retrieve valid address pointer")
			}

			addressStr := C.inet_ntop(C.int(addr.ifa_addr.sa_family), addrPtr, &address[0], C.INET6_ADDRSTRLEN)
			if addressStr == nil {
				return nil, fmt.Errorf("Failed to retrieve address string")
			}

			if addNetwork.Addresses == nil {
				addNetwork.Addresses = []api.InstanceStateNetworkAddress{}
			}

			goAddrString := C.GoString(addressStr)
			scope := "global"
			if strings.HasPrefix(goAddrString, "127") {
				scope = "local"
			}

			if goAddrString == "::1" {
				scope = "local"
			}

			if strings.HasPrefix(goAddrString, "169.254") {
				scope = "link"
			}

			if strings.HasPrefix(goAddrString, "fe80:") {
				scope = "link"
			}

			address := api.InstanceStateNetworkAddress{}
			address.Family = family
			address.Address = goAddrString
			address.Netmask = fmt.Sprintf("%d", int(addr.ifa_prefixlen))
			address.Scope = scope

			addNetwork.Addresses = append(addNetwork.Addresses, address)
		} else if addr.ifa_addr != nil && addr.ifa_addr.sa_family == C.AF_PACKET {
			if (addr.ifa_flags & C.IFF_LOOPBACK) == 0 {
				var buf [1024]C.char

				hwaddr := C.get_packet_address(addr.ifa_addr, &buf[0], 1024)
				if hwaddr == nil {
					return nil, fmt.Errorf("Failed to retrieve hardware address")
				}

				addNetwork.Hwaddr = C.GoString(hwaddr)
			}
		}

		if addr.ifa_stats_type == C.IFLA_STATS64 {
			addNetwork.Counters.BytesReceived = int64(addr.ifa_stats64.rx_bytes)
			addNetwork.Counters.BytesSent = int64(addr.ifa_stats64.tx_bytes)
			addNetwork.Counters.PacketsReceived = int64(addr.ifa_stats64.rx_packets)
			addNetwork.Counters.PacketsSent = int64(addr.ifa_stats64.tx_packets)
			addNetwork.Counters.ErrorsReceived = int64(addr.ifa_stats64.rx_errors)
			addNetwork.Counters.ErrorsSent = int64(addr.ifa_stats64.tx_errors)
		}
		ifName := C.GoString(addr.ifa_name)

		networks[ifName] = addNetwork
	}

	return networks, nil
}

// AbstractUnixSendFd sends a Unix file descriptor over a Unix socket.
func AbstractUnixSendFd(sockFD int, sendFD int) error {
	fd := C.int(sendFD)
	skFd := C.int(sockFD)
	ret := C.lxc_abstract_unix_send_fds(skFd, &fd, C.int(1), nil, C.size_t(0))
	if ret < 0 {
		return fmt.Errorf("Failed to send file descriptor via abstract unix socket")
	}

	return nil
}

// AbstractUnixReceiveFd receives a Unix file descriptor from a Unix socket.
func AbstractUnixReceiveFd(sockFD int, flags uint) (*os.File, error) {
	skFd := C.int(sockFD)
	fds := C.struct_unix_fds{}
	fds.fd_count_max = 1
	fds.flags = C.__u32(flags)
	ret := C.lxc_abstract_unix_recv_fds(skFd, &fds, nil, C.size_t(0))
	if ret < 0 {
		return nil, fmt.Errorf("Failed to receive file descriptor via abstract unix socket")
	}

	if fds.fd_count_max != fds.fd_count_ret {
		return nil, fmt.Errorf("Failed to receive file descriptor via abstract unix socket")
	}

	file := os.NewFile(uintptr(fds.fd[0]), "")
	return file, nil
}

// AbstractUnixReceiveFdData is a low level function to receive a file descriptor over a unix socket.
func AbstractUnixReceiveFdData(sockFD int, numFds int, flags uint, iov unsafe.Pointer, iovLen int32) (uint64, []C.int, error) {
	fds := C.struct_unix_fds{}

	if numFds >= C.KERNEL_SCM_MAX_FD {
		return 0, []C.int{-C.EBADF}, fmt.Errorf("Excessive number of file descriptors requested")
	}

	fds.fd_count_max = C.__u32(numFds)
	fds.flags = C.__u32(flags)

	skFd := C.int(sockFD)
	ret, errno := C.lxc_abstract_unix_recv_fds_iov(skFd, &fds, (*C.struct_iovec)(iov), C.size_t(iovLen))
	if ret < 0 {
		return 0, []C.int{-C.EBADF}, fmt.Errorf("Failed to receive file descriptor via abstract unix socket: errno=%d", errno)
	}

	if ret == 0 {
		return 0, []C.int{-C.EBADF}, io.EOF
	}

	if fds.fd_count_ret == 0 {
		return 0, []C.int{-C.EBADF}, io.EOF
	}

	cfd := make([]C.int, numFds)

	// Transfer the file descriptors.
	for i := C.__u32(0); i < fds.fd_count_ret; i++ {
		cfd[i] = fds.fd[i]
	}

	// Make sure that when we received less fds than we intended any
	// additional entries are negative.
	for i := fds.fd_count_ret; i < C.__u32(numFds); i++ {
		cfd[i] = -1
	}

	return uint64(ret), cfd, nil
}
<commit_msg>shared/netutils: Fill Dropped counters<commit_after>//go:build linux && cgo
// +build linux,cgo

package netutils

import (
	"fmt"
	"io"
	"net"
	"os"
	"strings"
	"unsafe"

	"github.com/lxc/lxd/shared/api"
)

/*
#include "unixfd.h"
#include "netns_getifaddrs.c"
*/
import "C"

// Allow the caller to set expectations.

// UnixFdsAcceptExact will only succeed if the exact amount of fds has been
// received  (unless combined with UNIX_FDS_ACCEPT_NONE).
const UnixFdsAcceptExact uint = C.UNIX_FDS_ACCEPT_EXACT

// UnixFdsAcceptLess will also succeed if less than the requested number of fd
// has been received.
// If the UNIX_FDS_ACCEPT_NONE flag is not raised than at least one fd must be
// received.
const UnixFdsAcceptLess uint = C.UNIX_FDS_ACCEPT_LESS

// UnixFdsAcceptMore will also succeed if more than the requested number of fds
// have been received. Any additional fds will be silently closed.
// If the UNIX_FDS_ACCEPT_NONE flag is not raised than at least one fd must be
// received.
const UnixFdsAcceptMore uint = C.UNIX_FDS_ACCEPT_MORE

// UnixFdsAcceptNone can be specified with any of the above flags and indicates
// that the caller will accept no file descriptors to be received.
const UnixFdsAcceptNone uint = C.UNIX_FDS_ACCEPT_NONE

// UnixFdsAcceptMask is the value of all the above flags or-ed together.
const UnixFdsAcceptMask uint = C.UNIX_FDS_ACCEPT_MASK

// Allow the callee to report back what happened. Only one of those will ever
// be set.

// UnixFdsReceivedExact indicates that the exact number of fds was received.
const UnixFdsReceivedExact uint = C.UNIX_FDS_RECEIVED_EXACT

// UnixFdsReceivedLess indicates that less than the requested number of fd has
// been received.
const UnixFdsReceivedLess uint = C.UNIX_FDS_RECEIVED_LESS

// UnixFdsReceivedMore indicates that more than the requested number of fd has
// been received.
const UnixFdsReceivedMore uint = C.UNIX_FDS_RECEIVED_MORE

// UnixFdsReceivedNone indicates that no fds have been received.
const UnixFdsReceivedNone uint = C.UNIX_FDS_RECEIVED_NONE

// NetnsGetifaddrs returns a map of InstanceStateNetwork for a particular process.
func NetnsGetifaddrs(initPID int32) (map[string]api.InstanceStateNetwork, error) {
	var netnsidAware C.bool
	var ifaddrs *C.struct_netns_ifaddrs
	var netnsID C.__s32

	if initPID > 0 {
		f, err := os.Open(fmt.Sprintf("/proc/%d/ns/net", initPID))
		if err != nil {
			return nil, err
		}
		defer f.Close()

		netnsID = C.netns_get_nsid(C.__s32(f.Fd()))
		if netnsID < 0 {
			return nil, fmt.Errorf("Failed to retrieve network namespace id")
		}
	} else {
		netnsID = -1
	}

	ret := C.netns_getifaddrs(&ifaddrs, netnsID, &netnsidAware)
	if ret < 0 {
		return nil, fmt.Errorf("Failed to retrieve network interfaces and addresses")
	}
	defer C.netns_freeifaddrs(ifaddrs)

	if netnsID >= 0 && !netnsidAware {
		return nil, fmt.Errorf("Netlink requests are not fully network namespace id aware")
	}

	// We're using the interface name as key here but we should really
	// switch to the ifindex at some point to handle ip aliasing correctly.
	networks := map[string]api.InstanceStateNetwork{}

	for addr := ifaddrs; addr != nil; addr = addr.ifa_next {
		var address [C.INET6_ADDRSTRLEN]C.char
		addNetwork, networkExists := networks[C.GoString(addr.ifa_name)]
		if !networkExists {
			addNetwork = api.InstanceStateNetwork{
				Addresses: []api.InstanceStateNetworkAddress{},
				Counters:  api.InstanceStateNetworkCounters{},
			}
		}

		// Interface flags
		netState := "down"
		netType := "unknown"

		if (addr.ifa_flags & C.IFF_BROADCAST) > 0 {
			netType = "broadcast"
		}

		if (addr.ifa_flags & C.IFF_LOOPBACK) > 0 {
			netType = "loopback"
		}

		if (addr.ifa_flags & C.IFF_POINTOPOINT) > 0 {
			netType = "point-to-point"
		}

		if (addr.ifa_flags & C.IFF_UP) > 0 {
			netState = "up"
		}
		addNetwork.State = netState
		addNetwork.Type = netType
		addNetwork.Mtu = int(addr.ifa_mtu)

		if initPID != 0 && int(addr.ifa_ifindex_peer) > 0 {
			hostInterface, err := net.InterfaceByIndex(int(addr.ifa_ifindex_peer))
			if err == nil {
				addNetwork.HostName = hostInterface.Name
			}
		}

		// Addresses
		if addr.ifa_addr != nil && (addr.ifa_addr.sa_family == C.AF_INET || addr.ifa_addr.sa_family == C.AF_INET6) {
			family := "inet"
			if addr.ifa_addr.sa_family == C.AF_INET6 {
				family = "inet6"
			}

			addrPtr := C.get_addr_ptr(addr.ifa_addr)
			if addrPtr == nil {
				return nil, fmt.Errorf("Failed to retrieve valid address pointer")
			}

			addressStr := C.inet_ntop(C.int(addr.ifa_addr.sa_family), addrPtr, &address[0], C.INET6_ADDRSTRLEN)
			if addressStr == nil {
				return nil, fmt.Errorf("Failed to retrieve address string")
			}

			if addNetwork.Addresses == nil {
				addNetwork.Addresses = []api.InstanceStateNetworkAddress{}
			}

			goAddrString := C.GoString(addressStr)
			scope := "global"
			if strings.HasPrefix(goAddrString, "127") {
				scope = "local"
			}

			if goAddrString == "::1" {
				scope = "local"
			}

			if strings.HasPrefix(goAddrString, "169.254") {
				scope = "link"
			}

			if strings.HasPrefix(goAddrString, "fe80:") {
				scope = "link"
			}

			address := api.InstanceStateNetworkAddress{}
			address.Family = family
			address.Address = goAddrString
			address.Netmask = fmt.Sprintf("%d", int(addr.ifa_prefixlen))
			address.Scope = scope

			addNetwork.Addresses = append(addNetwork.Addresses, address)
		} else if addr.ifa_addr != nil && addr.ifa_addr.sa_family == C.AF_PACKET {
			if (addr.ifa_flags & C.IFF_LOOPBACK) == 0 {
				var buf [1024]C.char

				hwaddr := C.get_packet_address(addr.ifa_addr, &buf[0], 1024)
				if hwaddr == nil {
					return nil, fmt.Errorf("Failed to retrieve hardware address")
				}

				addNetwork.Hwaddr = C.GoString(hwaddr)
			}
		}

		if addr.ifa_stats_type == C.IFLA_STATS64 {
			addNetwork.Counters.BytesReceived = int64(addr.ifa_stats64.rx_bytes)
			addNetwork.Counters.BytesSent = int64(addr.ifa_stats64.tx_bytes)
			addNetwork.Counters.PacketsReceived = int64(addr.ifa_stats64.rx_packets)
			addNetwork.Counters.PacketsSent = int64(addr.ifa_stats64.tx_packets)
			addNetwork.Counters.ErrorsReceived = int64(addr.ifa_stats64.rx_errors)
			addNetwork.Counters.ErrorsSent = int64(addr.ifa_stats64.tx_errors)
			addNetwork.Counters.PacketsDroppedInbound = int64(addr.ifa_stats64.rx_dropped)
			addNetwork.Counters.PacketsDroppedOutbound = int64(addr.ifa_stats64.tx_dropped)
		}
		ifName := C.GoString(addr.ifa_name)

		networks[ifName] = addNetwork
	}

	return networks, nil
}

// AbstractUnixSendFd sends a Unix file descriptor over a Unix socket.
func AbstractUnixSendFd(sockFD int, sendFD int) error {
	fd := C.int(sendFD)
	skFd := C.int(sockFD)
	ret := C.lxc_abstract_unix_send_fds(skFd, &fd, C.int(1), nil, C.size_t(0))
	if ret < 0 {
		return fmt.Errorf("Failed to send file descriptor via abstract unix socket")
	}

	return nil
}

// AbstractUnixReceiveFd receives a Unix file descriptor from a Unix socket.
func AbstractUnixReceiveFd(sockFD int, flags uint) (*os.File, error) {
	skFd := C.int(sockFD)
	fds := C.struct_unix_fds{}
	fds.fd_count_max = 1
	fds.flags = C.__u32(flags)
	ret := C.lxc_abstract_unix_recv_fds(skFd, &fds, nil, C.size_t(0))
	if ret < 0 {
		return nil, fmt.Errorf("Failed to receive file descriptor via abstract unix socket")
	}

	if fds.fd_count_max != fds.fd_count_ret {
		return nil, fmt.Errorf("Failed to receive file descriptor via abstract unix socket")
	}

	file := os.NewFile(uintptr(fds.fd[0]), "")
	return file, nil
}

// AbstractUnixReceiveFdData is a low level function to receive a file descriptor over a unix socket.
func AbstractUnixReceiveFdData(sockFD int, numFds int, flags uint, iov unsafe.Pointer, iovLen int32) (uint64, []C.int, error) {
	fds := C.struct_unix_fds{}

	if numFds >= C.KERNEL_SCM_MAX_FD {
		return 0, []C.int{-C.EBADF}, fmt.Errorf("Excessive number of file descriptors requested")
	}

	fds.fd_count_max = C.__u32(numFds)
	fds.flags = C.__u32(flags)

	skFd := C.int(sockFD)
	ret, errno := C.lxc_abstract_unix_recv_fds_iov(skFd, &fds, (*C.struct_iovec)(iov), C.size_t(iovLen))
	if ret < 0 {
		return 0, []C.int{-C.EBADF}, fmt.Errorf("Failed to receive file descriptor via abstract unix socket: errno=%d", errno)
	}

	if ret == 0 {
		return 0, []C.int{-C.EBADF}, io.EOF
	}

	if fds.fd_count_ret == 0 {
		return 0, []C.int{-C.EBADF}, io.EOF
	}

	cfd := make([]C.int, numFds)

	// Transfer the file descriptors.
	for i := C.__u32(0); i < fds.fd_count_ret; i++ {
		cfd[i] = fds.fd[i]
	}

	// Make sure that when we received less fds than we intended any
	// additional entries are negative.
	for i := fds.fd_count_ret; i < C.__u32(numFds); i++ {
		cfd[i] = -1
	}

	return uint64(ret), cfd, nil
}
<|endoftext|>


<commit_before>package gdc

import (
	"fmt"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
)

type sortableMetadata []files.IsMetadata

func (slice sortableMetadata) Len() int {
	return len(slice)
}

func (slice sortableMetadata) Less(i, j int) bool {
	m1 := slice[i]
	m2 := slice[j]
	switch m1t := m1.(type) {
	case *files.FolderMetadata:
		switch m2t := m2.(type) {
		case *files.FolderMetadata:
			return strings.Compare(m1t.Name, m2t.Name) < 0
		}
		return true
	case *files.FileMetadata:
		switch m2t := m2.(type) {
		case *files.FileMetadata:
			return strings.Compare(m1t.Name, m2t.Name) < 0
		}
		return false
	}
	return false
}

func (slice sortableMetadata) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// Lister provides access to file listings
type Lister struct {
	Options
	mu    sync.Mutex
	paths map[string]sortableMetadata
	wg    sync.WaitGroup
	dbx   files.Client
}

// NewLister creates a new Lister instance
func NewLister(options *Options) *Lister {
	return &Lister{
		Options: *options,
		paths:   make(map[string]sortableMetadata),
		dbx:     files.New(options.Config),
	}
}

// List files and folders inside the given remote path. This can be recursive depending on the provided Options.
func (l *Lister) List() {
	paths := l.Paths
	if len(paths) == 0 {
		paths = []string{""}
	}
	for _, path := range l.Paths {
		if path != "" && !strings.HasPrefix(path, "/") {
			path = "/" + path
		}

		if l.Verbose {
			fmt.Println("Listing files in", path, "(recursively: ", l.Recursive, ")")
		}

		a := files.NewListFolderArg(path)
		a.Recursive = l.Recursive
		r, err := l.dbx.ListFolder(a)
		if err != nil {
			panic(err)
		} else {
			for len(r.Entries) > 0 {
				l.wg.Add(1)
				go l.processServerResponse(path, r.Entries)
				if !r.HasMore {
					break
				}
				r, err = l.dbx.ListFolderContinue(files.NewListFolderContinueArg(r.Cursor))
				if err != nil {
					panic(err)
				}
			}
			l.wg.Wait()
			l.print()
		}
	}
}

func (l *Lister) processServerResponse(path string, entries []files.IsMetadata) {
	for _, fi := range entries {
		var m *files.Metadata
		switch md := fi.(type) {
		case *files.FileMetadata:
			m = &md.Metadata
		case *files.FolderMetadata:
			m = &md.Metadata

			// Also put the folder itself into the map when listing recursive.
			// In case there are no files in there it would not be listed otherwise
			if l.Recursive {
				l.mu.Lock()
				l.paths[m.PathDisplay] = append(l.paths[m.PathDisplay], nil)
				l.mu.Unlock()
			}
		}
		if path == m.PathDisplay {
			continue
		}
		filePath := l.extractPath(m)
		l.mu.Lock()
		l.paths[filePath] = append(l.paths[filePath], fi)
		l.mu.Unlock()
	}
	l.wg.Done()
}

func (l *Lister) extractPath(md *files.Metadata) string {
	p := path.Dir(md.PathDisplay)
	if p == "." {
		return "/"
	}
	return p
}

func (l *Lister) print() {
	filePaths := make([]string, 0)
	for filePath := range l.paths {
		filePaths = append(filePaths, filePath)
	}
	sort.Strings(filePaths)
	for _, filePath := range filePaths {
		mds := l.paths[filePath]
		sort.Sort(mds)
		if l.Recursive {
			fmt.Println(filePath + ":")
		}
		totalBytes := uint64(0)
		for _, md := range mds {
			switch m := md.(type) {
			case *files.FileMetadata:
				totalBytes += m.Size
			}
		}
		fmt.Println("total", l.convertSize(totalBytes))
		for _, md := range mds {
			switch m := md.(type) {
			case *files.FolderMetadata:
				fmt.Println("[d]\t" + m.PathDisplay)
			case *files.FileMetadata:
				fmt.Println("[f]\t" + l.convertSize(m.Size) + "\t" + m.ServerModified.String() + "\t" + m.PathDisplay)
			}
		}
	}
}

func (l *Lister) convertSize(size uint64) string {
	if l.HumanReadable {
		return HumanReadableBytes(size)
	}
	return strconv.FormatUint(size, 10)
}
<commit_msg>Fix listing root when no path to list is given<commit_after>package gdc

import (
	"fmt"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
)

type sortableMetadata []files.IsMetadata

func (slice sortableMetadata) Len() int {
	return len(slice)
}

func (slice sortableMetadata) Less(i, j int) bool {
	m1 := slice[i]
	m2 := slice[j]
	switch m1t := m1.(type) {
	case *files.FolderMetadata:
		switch m2t := m2.(type) {
		case *files.FolderMetadata:
			return strings.Compare(m1t.Name, m2t.Name) < 0
		}
		return true
	case *files.FileMetadata:
		switch m2t := m2.(type) {
		case *files.FileMetadata:
			return strings.Compare(m1t.Name, m2t.Name) < 0
		}
		return false
	}
	return false
}

func (slice sortableMetadata) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// Lister provides access to file listings
type Lister struct {
	Options
	mu    sync.Mutex
	paths map[string]sortableMetadata
	wg    sync.WaitGroup
	dbx   files.Client
}

// NewLister creates a new Lister instance
func NewLister(options *Options) *Lister {
	return &Lister{
		Options: *options,
		paths:   make(map[string]sortableMetadata),
		dbx:     files.New(options.Config),
	}
}

// List files and folders inside the given remote path. This can be recursive depending on the provided Options.
func (l *Lister) List() {
	paths := l.Paths
	if len(paths) == 0 {
		paths = []string{""}
	}
	for _, path := range paths {
		if path != "" && !strings.HasPrefix(path, "/") {
			path = "/" + path
		}

		if l.Verbose {
			fmt.Println("Listing files in", path, "(recursively: ", l.Recursive, ")")
		}

		a := files.NewListFolderArg(path)
		a.Recursive = l.Recursive
		r, err := l.dbx.ListFolder(a)
		if err != nil {
			panic(err)
		} else {
			for len(r.Entries) > 0 {
				l.wg.Add(1)
				go l.processServerResponse(path, r.Entries)
				if !r.HasMore {
					break
				}
				r, err = l.dbx.ListFolderContinue(files.NewListFolderContinueArg(r.Cursor))
				if err != nil {
					panic(err)
				}
			}
			l.wg.Wait()
			l.print()
		}
	}
}

func (l *Lister) processServerResponse(path string, entries []files.IsMetadata) {
	for _, fi := range entries {
		var m *files.Metadata
		switch md := fi.(type) {
		case *files.FileMetadata:
			m = &md.Metadata
		case *files.FolderMetadata:
			m = &md.Metadata

			// Also put the folder itself into the map when listing recursive.
			// In case there are no files in there it would not be listed otherwise
			if l.Recursive {
				l.mu.Lock()
				l.paths[m.PathDisplay] = append(l.paths[m.PathDisplay], nil)
				l.mu.Unlock()
			}
		}
		if path == m.PathDisplay {
			continue
		}
		filePath := l.extractPath(m)
		l.mu.Lock()
		l.paths[filePath] = append(l.paths[filePath], fi)
		l.mu.Unlock()
	}
	l.wg.Done()
}

func (l *Lister) extractPath(md *files.Metadata) string {
	p := path.Dir(md.PathDisplay)
	if p == "." {
		return "/"
	}
	return p
}

func (l *Lister) print() {
	filePaths := make([]string, 0)
	for filePath := range l.paths {
		filePaths = append(filePaths, filePath)
	}
	sort.Strings(filePaths)
	for _, filePath := range filePaths {
		mds := l.paths[filePath]
		sort.Sort(mds)
		if l.Recursive {
			fmt.Println(filePath + ":")
		}
		totalBytes := uint64(0)
		for _, md := range mds {
			switch m := md.(type) {
			case *files.FileMetadata:
				totalBytes += m.Size
			}
		}
		fmt.Println("total", l.convertSize(totalBytes))
		for _, md := range mds {
			switch m := md.(type) {
			case *files.FolderMetadata:
				fmt.Println("[d]\t" + m.PathDisplay)
			case *files.FileMetadata:
				fmt.Println("[f]\t" + l.convertSize(m.Size) + "\t" + m.ServerModified.String() + "\t" + m.PathDisplay)
			}
		}
	}
}

func (l *Lister) convertSize(size uint64) string {
	if l.HumanReadable {
		return HumanReadableBytes(size)
	}
	return strconv.FormatUint(size, 10)
}
<|endoftext|>


<commit_before>package s3api

import (
	"crypto/md5"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gorilla/mux"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {

	// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html

	bucket, object := getBucketAndObject(r)

	_, err := validateContentMd5(r.Header)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL)
		return
	}

	dataReader := r.Body
	if s3a.iam.isEnabled() {
		rAuthType := getRequestAuthType(r)
		var s3ErrCode s3err.ErrorCode
		switch rAuthType {
		case authTypeStreamingSigned:
			dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
		case authTypeSignedV2, authTypePresignedV2:
			_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
		case authTypePresigned, authTypeSigned:
			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
		}
		if s3ErrCode != s3err.ErrNone {
			writeErrorResponse(w, s3ErrCode, r.URL)
			return
		}
	}
	defer dataReader.Close()

	if strings.HasSuffix(object, "/") {
		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil {
			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
			return
		}
	} else {
		uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

		etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)

		if errCode != s3err.ErrNone {
			writeErrorResponse(w, errCode, r.URL)
			return
		}

		setEtag(w, etag)
	}

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	if strings.HasSuffix(r.URL.Path, "/") {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	response, _ := s3a.listFilerEntries(bucket, object, 1, "", "/")
	if len(response.Contents) != 0 && strings.HasSuffix(r.URL.Path, "/") {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) {
		for k, v := range proxyResponse.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(http.StatusNoContent)
	})
}

// / ObjectIdentifier carries key name for the object to delete.
type ObjectIdentifier struct {
	ObjectName string `xml:"Key"`
}

// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
	// Element to enable quiet mode for the request
	Quiet bool
	// List of objects to be deleted
	Objects []ObjectIdentifier `xml:"Object"`
}

// DeleteError structure.
type DeleteError struct {
	Code    string
	Message string
	Key     string
}

// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`

	// Collection of all deleted objects
	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`

	// Collection of errors deleting certain objects.
	Errors []DeleteError `xml:"Error,omitempty"`
}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	deleteXMLBytes, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	deleteObjects := &DeleteObjectsRequest{}
	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
		writeErrorResponse(w, s3err.ErrMalformedXML, r.URL)
		return
	}

	var deletedObjects []ObjectIdentifier
	var deleteErrors []DeleteError

	s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		for _, object := range deleteObjects.Objects {
			lastSeparator := strings.LastIndex(object.ObjectName, "/")
			parentDirectoryPath, entryName, isDeleteData, isRecursive := "/", object.ObjectName, true, true
			if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) {
				entryName = object.ObjectName[lastSeparator+1:]
				parentDirectoryPath = "/" + object.ObjectName[:lastSeparator]
			}
			parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)

			err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
			if err == nil {
				deletedObjects = append(deletedObjects, object)
			} else {
				deleteErrors = append(deleteErrors, DeleteError{
					Code:    "",
					Message: err.Error(),
					Key:     object.ObjectName,
				})
			}
		}
		return nil
	})

	deleteResp := DeleteObjectsResponse{}
	if !deleteObjects.Quiet {
		deleteResp.DeletedObjects = deletedObjects
	}
	deleteResp.Errors = deleteErrors

	writeSuccessResponseXML(w, encodeResponse(deleteResp))

}

func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter)) {

	glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)

	proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", destUrl, err)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if resp.ContentLength == -1 {
		writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
		return
	}

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	defer util.CloseResponse(resp)

	responseFn(resp, w)

}
func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
	for k, v := range proxyResponse.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(proxyResponse.StatusCode)
	io.Copy(w, proxyResponse.Body)
}

func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) {

	hash := md5.New()
	var body = io.TeeReader(dataReader, hash)

	proxyReq, err := http.NewRequest("PUT", uploadUrl, body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", uploadUrl, err)
		return "", s3err.ErrInternalError
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		return "", s3err.ErrInternalError
	}
	defer resp.Body.Close()

	etag = fmt.Sprintf("%x", hash.Sum(nil))

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.Errorf("upload to filer response read: %v", ra_err)
		return etag, s3err.ErrInternalError
	}
	var ret weed_server.FilerPostResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body))
		return "", s3err.ErrInternalError
	}
	if ret.Error != "" {
		glog.Errorf("upload to filer error: %v", ret.Error)
		return "", s3err.ErrInternalError
	}

	return etag, s3err.ErrNone
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getBucketAndObject(r *http.Request) (bucket, object string) {
	vars := mux.Vars(r)
	bucket = vars["bucket"]
	object = vars["object"]
	if !strings.HasPrefix(object, "/") {
		object = "/" + object
	}

	return
}
<commit_msg>fixed the problem of empty directory when S3 deleted the directory<commit_after>package s3api

import (
	"crypto/md5"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gorilla/mux"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {

	// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html

	bucket, object := getBucketAndObject(r)

	_, err := validateContentMd5(r.Header)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL)
		return
	}

	dataReader := r.Body
	if s3a.iam.isEnabled() {
		rAuthType := getRequestAuthType(r)
		var s3ErrCode s3err.ErrorCode
		switch rAuthType {
		case authTypeStreamingSigned:
			dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
		case authTypeSignedV2, authTypePresignedV2:
			_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
		case authTypePresigned, authTypeSigned:
			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
		}
		if s3ErrCode != s3err.ErrNone {
			writeErrorResponse(w, s3ErrCode, r.URL)
			return
		}
	}
	defer dataReader.Close()

	if strings.HasSuffix(object, "/") {
		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil {
			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
			return
		}
	} else {
		uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

		etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)

		if errCode != s3err.ErrNone {
			writeErrorResponse(w, errCode, r.URL)
			return
		}

		setEtag(w, etag)
	}

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	if strings.HasSuffix(r.URL.Path, "/") {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := getBucketAndObject(r)

	response, _ := s3a.listFilerEntries(bucket, object, 1, "", "/")
	if len(response.Contents) != 0 && strings.HasSuffix(r.URL.Path, "/") {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) {
		for k, v := range proxyResponse.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(http.StatusNoContent)
	})
}

// / ObjectIdentifier carries key name for the object to delete.
type ObjectIdentifier struct {
	ObjectName string `xml:"Key"`
}

// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
	// Element to enable quiet mode for the request
	Quiet bool
	// List of objects to be deleted
	Objects []ObjectIdentifier `xml:"Object"`
}

// DeleteError structure.
type DeleteError struct {
	Code    string
	Message string
	Key     string
}

// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`

	// Collection of all deleted objects
	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`

	// Collection of errors deleting certain objects.
	Errors []DeleteError `xml:"Error,omitempty"`
}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	deleteXMLBytes, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	deleteObjects := &DeleteObjectsRequest{}
	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
		writeErrorResponse(w, s3err.ErrMalformedXML, r.URL)
		return
	}

	var deletedObjects []ObjectIdentifier
	var deleteErrors []DeleteError

	s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		for _, object := range deleteObjects.Objects {
			response, _ := s3a.listFilerEntries(bucket, object.ObjectName, 1, "", "/")
			if len(response.Contents) != 0 && strings.HasSuffix(object.ObjectName, "/") {
				continue
			}

			lastSeparator := strings.LastIndex(object.ObjectName, "/")
			parentDirectoryPath, entryName, isDeleteData, isRecursive := "/", object.ObjectName, true, true
			if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) {
				entryName = object.ObjectName[lastSeparator+1:]
				parentDirectoryPath = "/" + object.ObjectName[:lastSeparator]
			}
			parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)

			err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
			if err == nil {
				deletedObjects = append(deletedObjects, object)
			} else {
				deleteErrors = append(deleteErrors, DeleteError{
					Code:    "",
					Message: err.Error(),
					Key:     object.ObjectName,
				})
			}
		}
		return nil
	})

	deleteResp := DeleteObjectsResponse{}
	if !deleteObjects.Quiet {
		deleteResp.DeletedObjects = deletedObjects
	}
	deleteResp.Errors = deleteErrors

	writeSuccessResponseXML(w, encodeResponse(deleteResp))

}

func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter)) {

	glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)

	proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", destUrl, err)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if resp.ContentLength == -1 {
		writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
		return
	}

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	defer util.CloseResponse(resp)

	responseFn(resp, w)

}
func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
	for k, v := range proxyResponse.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(proxyResponse.StatusCode)
	io.Copy(w, proxyResponse.Body)
}

func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) {

	hash := md5.New()
	var body = io.TeeReader(dataReader, hash)

	proxyReq, err := http.NewRequest("PUT", uploadUrl, body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", uploadUrl, err)
		return "", s3err.ErrInternalError
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		return "", s3err.ErrInternalError
	}
	defer resp.Body.Close()

	etag = fmt.Sprintf("%x", hash.Sum(nil))

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.Errorf("upload to filer response read: %v", ra_err)
		return etag, s3err.ErrInternalError
	}
	var ret weed_server.FilerPostResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body))
		return "", s3err.ErrInternalError
	}
	if ret.Error != "" {
		glog.Errorf("upload to filer error: %v", ret.Error)
		return "", s3err.ErrInternalError
	}

	return etag, s3err.ErrNone
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getBucketAndObject(r *http.Request) (bucket, object string) {
	vars := mux.Vars(r)
	bucket = vars["bucket"]
	object = vars["object"]
	if !strings.HasPrefix(object, "/") {
		object = "/" + object
	}

	return
}
<|endoftext|>


<commit_before>package topology

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/chrislusf/seaweedfs/weed/sequence"
	"github.com/chrislusf/seaweedfs/weed/storage"
)

var topologyLayout = `
{
  "dc1":{
    "rack1":{
      "server111":{
        "volumes":[
          {"id":1, "size":12312},
          {"id":2, "size":12312},
          {"id":3, "size":12312}
        ],
        "limit":3
      },
      "server112":{
        "volumes":[
          {"id":4, "size":12312},
          {"id":5, "size":12312},
          {"id":6, "size":12312}
        ],
        "limit":10
      }
    },
    "rack2":{
      "server121":{
        "volumes":[
          {"id":4, "size":12312},
          {"id":5, "size":12312},
          {"id":6, "size":12312}
        ],
        "limit":4
      },
      "server122":{
        "volumes":[],
        "limit":4
      },
      "server123":{
        "volumes":[
          {"id":2, "size":12312},
          {"id":3, "size":12312},
          {"id":4, "size":12312}
        ],
        "limit":5
      }
    }
  },
  "dc2":{
  },
  "dc3":{
    "rack2":{
      "server321":{
        "volumes":[
          {"id":1, "size":12312},
          {"id":3, "size":12312},
          {"id":5, "size":12312}
        ],
        "limit":4
      }
    }
  }
}
`

func setup(topologyLayout string) *Topology {
	var data interface{}
	err := json.Unmarshal([]byte(topologyLayout), &data)
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Println("data:", data)

	//need to connect all nodes first before server adding volumes
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5)
	mTopology := data.(map[string]interface{})
	for dcKey, dcValue := range mTopology {
		dc := NewDataCenter(dcKey)
		dcMap := dcValue.(map[string]interface{})
		topo.LinkChildNode(dc)
		for rackKey, rackValue := range dcMap {
			rack := NewRack(rackKey)
			rackMap := rackValue.(map[string]interface{})
			dc.LinkChildNode(rack)
			for serverKey, serverValue := range rackMap {
				server := NewDataNode(serverKey)
				serverMap := serverValue.(map[string]interface{})
				rack.LinkChildNode(server)
				for _, v := range serverMap["volumes"].([]interface{}) {
					m := v.(map[string]interface{})
					vi := storage.VolumeInfo{
						Id:      storage.VolumeId(int64(m["id"].(float64))),
						Size:    uint64(m["size"].(float64)),
						Version: storage.CurrentVersion}
					server.AddOrUpdateVolume(vi)
				}
				server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
			}
		}
	}

	return topo
}

func TestFindEmptySlotsForOneVolume(t *testing.T) {
	topo := setup(topologyLayout)
	vg := NewDefaultVolumeGrowth()
	rp, _ := storage.NewReplicaPlacementFromString("002")
	volumeGrowOption := &VolumeGrowOption{
		Collection:       "",
		ReplicaPlacement: rp,
		DataCenter:       "dc1",
		Rack:             "",
		DataNode:         "",
	}
	servers, err := vg.findEmptySlotsForOneVolume(topo, volumeGrowOption)
	if err != nil {
		fmt.Println("finding empty slots error :", err)
		t.Fail()
	}
	for _, server := range servers {
		fmt.Println("assigned node :", server.Id())
	}
}
<commit_msg>fix test<commit_after>package topology

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/chrislusf/seaweedfs/weed/sequence"
	"github.com/chrislusf/seaweedfs/weed/storage"
)

var topologyLayout = `
{
  "dc1":{
    "rack1":{
      "server111":{
        "volumes":[
          {"id":1, "size":12312},
          {"id":2, "size":12312},
          {"id":3, "size":12312}
        ],
        "limit":3
      },
      "server112":{
        "volumes":[
          {"id":4, "size":12312},
          {"id":5, "size":12312},
          {"id":6, "size":12312}
        ],
        "limit":10
      }
    },
    "rack2":{
      "server121":{
        "volumes":[
          {"id":4, "size":12312},
          {"id":5, "size":12312},
          {"id":6, "size":12312}
        ],
        "limit":4
      },
      "server122":{
        "volumes":[],
        "limit":4
      },
      "server123":{
        "volumes":[
          {"id":2, "size":12312},
          {"id":3, "size":12312},
          {"id":4, "size":12312}
        ],
        "limit":5
      }
    }
  },
  "dc2":{
  },
  "dc3":{
    "rack2":{
      "server321":{
        "volumes":[
          {"id":1, "size":12312},
          {"id":3, "size":12312},
          {"id":5, "size":12312}
        ],
        "limit":4
      }
    }
  }
}
`

func setup(topologyLayout string) *Topology {
	var data interface{}
	err := json.Unmarshal([]byte(topologyLayout), &data)
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Println("data:", data)

	//need to connect all nodes first before server adding volumes
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5)
	mTopology := data.(map[string]interface{})
	for dcKey, dcValue := range mTopology {
		dc := NewDataCenter(dcKey)
		dcMap := dcValue.(map[string]interface{})
		topo.LinkChildNode(dc)
		for rackKey, rackValue := range dcMap {
			rack := NewRack(rackKey)
			rackMap := rackValue.(map[string]interface{})
			dc.LinkChildNode(rack)
			for serverKey, serverValue := range rackMap {
				server := NewDataNode(serverKey)
				serverMap := serverValue.(map[string]interface{})
				rack.LinkChildNode(server)
				for _, v := range serverMap["volumes"].([]interface{}) {
					m := v.(map[string]interface{})
					vi := storage.VolumeInfo{
						Id:      storage.VolumeId(int64(m["id"].(float64))),
						Size:    uint64(m["size"].(float64)),
						Version: storage.CurrentVersion}
					server.AddOrUpdateVolume(vi)
				}
				server.UpAdjustMaxVolumeCountDelta(int64(serverMap["limit"].(float64)))
			}
		}
	}

	return topo
}

func TestFindEmptySlotsForOneVolume(t *testing.T) {
	topo := setup(topologyLayout)
	vg := NewDefaultVolumeGrowth()
	rp, _ := storage.NewReplicaPlacementFromString("002")
	volumeGrowOption := &VolumeGrowOption{
		Collection:       "",
		ReplicaPlacement: rp,
		DataCenter:       "dc1",
		Rack:             "",
		DataNode:         "",
	}
	servers, err := vg.findEmptySlotsForOneVolume(topo, volumeGrowOption)
	if err != nil {
		fmt.Println("finding empty slots error :", err)
		t.Fail()
	}
	for _, server := range servers {
		fmt.Println("assigned node :", server.Id())
	}
}
<|endoftext|>


<commit_before>package namespace

import (
	"encoding/json"
	"time"

	"veyron.io/veyron/veyron2"
	"veyron.io/veyron/veyron2/context"
	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/verror2"

	"veyron.io/wspr/veyron/services/wsprd/lib"
)

// request struct represents a request to call a method on the namespace client
type request struct {
	Method namespaceMethod
	Args   json.RawMessage
	Roots  []string
}

type namespaceMethod int

// enumerates the methods available to be called on the namespace client
const (
	methodGlob            namespaceMethod = 0
	methodMount                           = 1
	methodUnmount                         = 2
	methodResolve                         = 3
	methodResolveToMt                     = 4
	methodFlushCacheEntry                 = 5
	methodDisableCache                    = 6
	methodRoots                           = 7
)

// globArgs defines the args for the glob method
type globArgs struct {
	Pattern string
}

// mountArgs defines the args for the mount method
type mountArgs struct {
	Name         string
	Server       string
	Ttl          time.Duration
	replaceMount bool
}

// unmountArgs defines the args for the unmount method
type unmountArgs struct {
	Name   string
	Server string
}

// resolveArgs defines the args for the resolve method
type resolveArgs struct {
	Name string
}

// resolveToMtArgs defines the args for the resolveToMt method
type resolveToMtArgs struct {
	Name string
}

// flushCacheEntryArgs defines the args for the flushCacheEntry method
type flushCacheEntryArgs struct {
	Name string
}

// disableCacheArgs defines the args for the disableCache method
type disableCacheArgs struct {
	Disable bool
}

// handleRequest uses the namespace client to respond to namespace specific requests such as glob
func HandleRequest(ctx context.T, rt veyron2.Runtime, data string, w lib.ClientWriter) {
	// Decode the request
	var req request
	if err := json.Unmarshal([]byte(data), &req); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	// Create a namespace and set roots if provided
	var ns = rt.Namespace()
	if len(req.Roots) > 0 {
		ns.SetRoots(req.Roots...)
	}

	switch req.Method {
	case methodGlob:
		glob(ctx, ns, w, req.Args)
	case methodMount:
		mount(ctx, ns, w, req.Args)
	case methodUnmount:
		unmount(ctx, ns, w, req.Args)
	case methodResolve:
		resolve(ctx, ns, w, req.Args)
	case methodResolveToMt:
		resolveToMt(ctx, ns, w, req.Args)
	case methodFlushCacheEntry:
		flushCacheEntry(ctx, ns, w, req.Args)
	case methodDisableCache:
		disableCache(ctx, ns, w, req.Args)
	case methodRoots:
		roots(ctx, ns, w)
	default:
		w.Error(verror2.Make(verror2.NoExist, ctx, req.Method))
	}
}

func glob(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args globArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	// Call Glob on the namespace client instance
	ch, err := ns.Glob(ctx, args.Pattern)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	for name := range ch {
		if err := w.Send(lib.ResponseStream, name); err != nil {
			w.Error(verror2.Make(verror2.Internal, ctx, name))
			return
		}
	}

	if err := w.Send(lib.ResponseStreamClose, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseStreamClose"))
	}
}

func mount(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args mountArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	rmOpt := naming.ReplaceMountOpt(args.replaceMount)
	err := ns.Mount(ctx, args.Name, args.Server, args.Ttl, rmOpt)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func unmount(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args unmountArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	err := ns.Unmount(ctx, args.Name, args.Server)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func resolve(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args resolveArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	addresses, err := ns.Resolve(ctx, args.Name)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, addresses); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func resolveToMt(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args resolveToMtArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	addresses, err := ns.ResolveToMountTable(ctx, args.Name)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, addresses); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func flushCacheEntry(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args flushCacheEntryArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	flushed := ns.FlushCacheEntry(args.Name)

	if err := w.Send(lib.ResponseFinal, flushed); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func disableCache(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args disableCacheArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	disableCacheCtl := naming.DisableCache(args.Disable)
	_ = ns.CacheCtl(disableCacheCtl)

	if err := w.Send(lib.ResponseFinal, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func roots(ctx context.T, ns naming.Namespace, w lib.ClientWriter) {
	roots := ns.Roots()

	if err := w.Send(lib.ResponseFinal, roots); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}
<commit_msg>Fix broken tests.<commit_after>package namespace

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"time"

	"veyron.io/veyron/veyron2"
	"veyron.io/veyron/veyron2/context"
	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/verror2"
	"veyron.io/veyron/veyron2/vom2"

	"veyron.io/wspr/veyron/services/wsprd/lib"
)

// request struct represents a request to call a method on the namespace client
type request struct {
	Method namespaceMethod
	Args   json.RawMessage
	Roots  []string
}

type namespaceMethod int

// enumerates the methods available to be called on the namespace client
const (
	methodGlob            namespaceMethod = 0
	methodMount                           = 1
	methodUnmount                         = 2
	methodResolve                         = 3
	methodResolveToMt                     = 4
	methodFlushCacheEntry                 = 5
	methodDisableCache                    = 6
	methodRoots                           = 7
)

// globArgs defines the args for the glob method
type globArgs struct {
	Pattern string
}

// mountArgs defines the args for the mount method
type mountArgs struct {
	Name         string
	Server       string
	Ttl          time.Duration
	replaceMount bool
}

// unmountArgs defines the args for the unmount method
type unmountArgs struct {
	Name   string
	Server string
}

// resolveArgs defines the args for the resolve method
type resolveArgs struct {
	Name string
}

// resolveToMtArgs defines the args for the resolveToMt method
type resolveToMtArgs struct {
	Name string
}

// flushCacheEntryArgs defines the args for the flushCacheEntry method
type flushCacheEntryArgs struct {
	Name string
}

// disableCacheArgs defines the args for the disableCache method
type disableCacheArgs struct {
	Disable bool
}

// handleRequest uses the namespace client to respond to namespace specific requests such as glob
func HandleRequest(ctx context.T, rt veyron2.Runtime, data string, w lib.ClientWriter) {
	// Decode the request
	var req request
	if err := json.Unmarshal([]byte(data), &req); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	// Create a namespace and set roots if provided
	var ns = rt.Namespace()
	if len(req.Roots) > 0 {
		ns.SetRoots(req.Roots...)
	}

	switch req.Method {
	case methodGlob:
		glob(ctx, ns, w, req.Args)
	case methodMount:
		mount(ctx, ns, w, req.Args)
	case methodUnmount:
		unmount(ctx, ns, w, req.Args)
	case methodResolve:
		resolve(ctx, ns, w, req.Args)
	case methodResolveToMt:
		resolveToMt(ctx, ns, w, req.Args)
	case methodFlushCacheEntry:
		flushCacheEntry(ctx, ns, w, req.Args)
	case methodDisableCache:
		disableCache(ctx, ns, w, req.Args)
	case methodRoots:
		roots(ctx, ns, w)
	default:
		w.Error(verror2.Make(verror2.NoExist, ctx, req.Method))
	}
}

func encodeVom2(value interface{}) (string, error) {
	var buf bytes.Buffer
	encoder, err := vom2.NewBinaryEncoder(&buf)
	if err != nil {
		return "", err
	}

	if err := encoder.Encode(value); err != nil {
		return "", err
	}
	return hex.EncodeToString(buf.Bytes()), nil

}

func convertToVDLEntry(value naming.MountEntry) naming.VDLMountEntry {
	result := naming.VDLMountEntry{
		Name: value.Name,
		MT:   false,
	}
	for _, s := range value.Servers {
		result.Servers = append(result.Servers,
			naming.VDLMountedServer{
				Server: s.Server,
				TTL:    uint32(s.Expires.Sub(time.Now())),
			})
	}
	return result
}

func glob(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args globArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	// Call Glob on the namespace client instance
	ch, err := ns.Glob(ctx, args.Pattern)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	for name := range ch {
		val, err := encodeVom2(convertToVDLEntry(name))
		if err != nil {
			w.Error(verror2.Make(verror2.Internal, ctx, err))
			return
		}
		if err := w.Send(lib.ResponseStream, val); err != nil {
			w.Error(verror2.Make(verror2.Internal, ctx, name))
			return
		}
	}

	if err := w.Send(lib.ResponseStreamClose, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseStreamClose"))
	}
}

func mount(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args mountArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	rmOpt := naming.ReplaceMountOpt(args.replaceMount)
	err := ns.Mount(ctx, args.Name, args.Server, args.Ttl, rmOpt)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func unmount(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args unmountArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	err := ns.Unmount(ctx, args.Name, args.Server)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func resolve(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args resolveArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	addresses, err := ns.Resolve(ctx, args.Name)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, addresses); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func resolveToMt(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args resolveToMtArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	addresses, err := ns.ResolveToMountTable(ctx, args.Name)

	if err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	if err := w.Send(lib.ResponseFinal, addresses); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func flushCacheEntry(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args flushCacheEntryArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	flushed := ns.FlushCacheEntry(args.Name)

	if err := w.Send(lib.ResponseFinal, flushed); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func disableCache(ctx context.T, ns naming.Namespace, w lib.ClientWriter, rawArgs json.RawMessage) {
	var args disableCacheArgs
	if err := json.Unmarshal([]byte(rawArgs), &args); err != nil {
		w.Error(verror2.Convert(verror2.Internal, ctx, err))
		return
	}

	disableCacheCtl := naming.DisableCache(args.Disable)
	_ = ns.CacheCtl(disableCacheCtl)

	if err := w.Send(lib.ResponseFinal, nil); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}

func roots(ctx context.T, ns naming.Namespace, w lib.ClientWriter) {
	roots := ns.Roots()

	if err := w.Send(lib.ResponseFinal, roots); err != nil {
		w.Error(verror2.Make(verror2.Internal, ctx, "ResponseFinal"))
	}
}
<|endoftext|>


<commit_before>package runtime

import (
    "log"
    "net/url"

    "github.com/ChimeraCoder/anaconda"

    "github.com/remeh/wcie/db"
)

// Our crawler.
type Crawler struct {
    App *App
}

func NewCrawler(app *App) *Crawler {
    return &Crawler{App: app}
}

// TODO cache tweet by ids.
// TODO complex queries support
func (c *Crawler) Crawl() {
    // Api twitter provided by ChimeraCoder !
    anaconda.SetConsumerKey(c.App.Config.TwitterApiKey)
    anaconda.SetConsumerSecret(c.App.Config.TwitterSecret)
    api := anaconda.NewTwitterApi(c.App.Config.TwitterAccessToken, c.App.Config.TwitterAccessTokenSecret)
    defer api.Close()

    c.Search(api, "je mange un")
    c.Search(api, "je mange une")
    c.Search(api, "je mange du")
    c.Search(api, "je mange des")
}

// Calls Twitter to execute the given query search.
// Stores the retrieved tweets into MongoDB, deduplicating
// them using their ID. Retweets aren't stored.
// Returns how many tweets were actually stored.
func (c *Crawler) Search(api *anaconda.TwitterApi, query string) int {
    // DAO
    tweetDao := db.NewTweetDAO(c.App.Mongo)

    // Maximum value for the search
    params := url.Values{}
    params.Set("count", "100")          // Amount of tweet possible in one query
    params.Set("result_type", "recent") // We want the more recent tweets
    searchResult, err := api.GetSearch("\""+query+"\"", params)

    // Error, end of job for this time.
    if err != nil {
        log.Printf("An error ocurred during the search on Twitter : %s\n", err.Error())
        return 0
    }

    i := 0;

    // Saves every tweet into MongoDB for further analysis
    for _ , tweet := range searchResult {
        // Don't insert retweet
        if tweet.RetweetedStatus != nil {
            continue
        }

        // Don't insert it if we already have it (retweets)
        existing, err := tweetDao.FindByTweetId(tweet.Id)
        // Look for existing
        if len(existing) == 0 {
            err = tweetDao.Upsert(db.NewTweetFromApiTweet(&tweet, query))
            if err == nil {
                i++;
            }
        }
    }

    log.Printf("[info] %d tweets saved for \"%s\".\n", i, query);
    return i
}
<commit_msg>Added some search queries<commit_after>package runtime

import (
    "log"
    "net/url"

    "github.com/ChimeraCoder/anaconda"

    "github.com/remeh/wcie/db"
)

// Our crawler.
type Crawler struct {
    App *App
}

func NewCrawler(app *App) *Crawler {
    return &Crawler{App: app}
}

// TODO cache tweet by ids.
// TODO complex queries support
func (c *Crawler) Crawl() {
    // Api twitter provided by ChimeraCoder !
    anaconda.SetConsumerKey(c.App.Config.TwitterApiKey)
    anaconda.SetConsumerSecret(c.App.Config.TwitterSecret)
    api := anaconda.NewTwitterApi(c.App.Config.TwitterAccessToken, c.App.Config.TwitterAccessTokenSecret)
    defer api.Close()

    c.Search(api, "je mange un")
    c.Search(api, "je mange une")
    c.Search(api, "je mange du")
    c.Search(api, "je mange de la")
    c.Search(api, "je mange des")
    c.Search(api, "je cuisine un")
    c.Search(api, "je cuisine une")
    c.Search(api, "je cuisine de la")
    c.Search(api, "je cuisine des")
}

// Calls Twitter to execute the given query search.
// Stores the retrieved tweets into MongoDB, deduplicating
// them using their ID. Retweets aren't stored.
// Returns how many tweets were actually stored.
func (c *Crawler) Search(api *anaconda.TwitterApi, query string) int {
    // DAO
    tweetDao := db.NewTweetDAO(c.App.Mongo)

    // Maximum value for the search
    params := url.Values{}
    params.Set("count", "100")          // Amount of tweet possible in one query
    params.Set("result_type", "recent") // We want the more recent tweets
    searchResult, err := api.GetSearch("\""+query+"\"", params)

    // Error, end of job for this time.
    if err != nil {
        log.Printf("An error ocurred during the search on Twitter : %s\n", err.Error())
        return 0
    }

    i := 0;

    // Saves every tweet into MongoDB for further analysis
    for _ , tweet := range searchResult {
        // Don't insert retweet
        if tweet.RetweetedStatus != nil {
            continue
        }

        // Don't insert it if we already have it (retweets)
        existing, err := tweetDao.FindByTweetId(tweet.Id)
        // Look for existing
        if len(existing) == 0 {
            err = tweetDao.Upsert(db.NewTweetFromApiTweet(&tweet, query))
            if err == nil {
                i++;
            }
        }
    }

    log.Printf("[info] %d tweets saved for \"%s\".\n", i, query);
    return i
}
<|endoftext|>