Add a "clone" command for git clone'ing repos

libgit sucks hard, so let's just fork and call 'git clone', using sd_event to make things nice and async. In the case where the git repository already exists, let's transparently turn "git clone" into "git pull", in order to make updates simpler. There's probably bugs here. I need better (any) unit tests for the event loop portion of this...
falconindy · Oct 7, 2018 · c73bbee · c73bbee
1 parent 1fb0d47
commit c73bbee
Show file tree

Hide file tree

Showing 13 changed files with 339 additions and 36 deletions.
diff --git a/e2e/clone b/e2e/clone
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# shellcheck disable=SC1090
+source "${0%/*}/common"
+
+# should clone
+if ! auracle clone auracle-git; then
+  echo "FAIL: expected 0 exit status, got non-zero" >&2
+  exit 1
+fi
+
+# should do nothing (update)
+if ! auracle clone auracle-git; then
+  echo "FAIL: expected 0 exit status, got non-zero" >&2
+  exit 1
+fi
+
+assert_file_exists "$TEST_TMPDIR/auracle-git/PKGBUILD"
+assert_directory_exists "$TEST_TMPDIR/auracle-git/.git"
diff --git a/e2e/common b/e2e/common
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+find_build_directory() {
+  local build_dirs=(*/.ninja_log)
+
+  if [[ ! -e ${build_dirs[0]} ]]; then
+    echo "error: No build directory found. Have you run 'meson build' yet?" >&2
+    return 1
+  elif (( ${#build_dirs[*]} > 1 )); then
+    echo "error: Multiple build directories found. Unable to proceed." >&2
+    return 1
+  fi
+
+  printf '%s\n' "${build_dirs[0]%/*}"
+}
+
+assert_directory_exists() {
+  if [[ ! -d $1 ]]; then
+    printf 'FAIL: expected "%s", but not found\n' "$1" >&2
+    exit 1
+  fi
+}
+
+assert_file_exists() {
+  if [[ ! -f $1 ]]; then
+    printf 'FAIL: expected "%s", but not found\n' "$1" >&2
+    exit 1
+  fi
+}
+
+auracle() {
+  "$AURACLE_BIN" -C "$TEST_TMPDIR" "$@" 2>/dev/null
+}
+
+BUILD_DIR=$(find_build_directory) || exit 1
+AURACLE_BIN=$BUILD_DIR/auracle
+
+TEST_TMPDIR=$(mktemp -d)
+trap 'rm -rf "$TEST_TMPDIR"' EXIT
+
diff --git a/e2e/download b/e2e/download
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# shellcheck disable=SC1090
+source "${0%/*}/common"
+
+if ! auracle download pkgfile-git; then
+  echo "FAIL: expected 0 exit status, got non-zero" >&2
+  exit 1
+fi
+assert_file_exists "$TEST_TMPDIR/pkgfile-git/PKGBUILD"
+
+if ! auracle download -r auracle-git; then
+  echo "FAIL: expected 0 exit status, got non-zero" >&2
+  exit 1
+fi
+assert_file_exists "$TEST_TMPDIR/auracle-git/PKGBUILD"
+assert_file_exists "$TEST_TMPDIR/nlohmann-json/PKGBUILD"
diff --git a/e2e/rawquery b/e2e/rawquery
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# shellcheck disable=SC1090
+source "${0%/*}/common"
+
+if ! out=$(auracle rawsearch aura | jq .resultcount 2>&1) && (( out > 0 )); then
+  echo "FAIL: expected valid JSON, but got: $out" >&2
+  exit 1
+fi
+
+if ! out=$(auracle rawsearch auracle-git | jq .resultcount 2>&1) || (( out != 1 )); then
+  echo "FAIL: expected valid JSON with resultcount of 1, but got: $out" >&2
+  exit 1
+fi
+
+if ! out=$(auracle rawsearch --searchby maintainer falconindy | jq -r '.results[].Name') ||
+    [[ $out != *auracle-git* ]]; then
+  echo "FAIL: expected valid JSON with at least 'auracle-git', but got: $out" >&2
+  exit 1
+fi
diff --git a/extra/bash_completion b/extra/bash_completion
@@ -43,7 +43,7 @@ _auracle() {
   fi
 
   local -A VERBS=(
-      [AUR_PACKAGES]='buildorder download pkgbuild info rawinfo'
+      [AUR_PACKAGES]='buildorder clone download pkgbuild info rawinfo'
     [LOCAL_PACKAGES]='sync'
               [NONE]='search rawsearch'
   )

diff --git a/man/auracle.1.pod b/man/auracle.1.pod
@@ -102,6 +102,13 @@ input to this operation.
 Pass one to many arguments to download packages. Use the B<--recurse> flag
 to download dependencies of packages.
 
+=item B<clone> I<PACKAGES>...
+
+Pass one to many arguments to clone package git repositories. Use the
+B<--recurse> flag to clone dependencies of packages. If the git repository
+already exists, the repository will instead be updated via a fast-forward only
+git pull .
+
 =item B<buildorder> I<PACKAGES>...
 
 Pass one to many arguments to print a build order for the given packages.

diff --git a/src/aur/aur.cc b/src/aur/aur.cc
@@ -2,11 +2,14 @@
 
 #include <fcntl.h>
 #include <string.h>
-
 #include <unistd.h>
+
 #include <chrono>
+#include <filesystem>
 #include <string_view>
 
+namespace fs = std::filesystem;
+
 namespace aur {
 
 namespace {
@@ -58,7 +61,7 @@ class ResponseHandler {
 
   static size_t BodyCallback(char* ptr, size_t size, size_t nmemb,
                              void* userdata) {
-    auto* handler = static_cast<ResponseHandler*>(userdata);
+    auto handler = static_cast<ResponseHandler*>(userdata);
 
     handler->body.append(ptr, size * nmemb);
 
@@ -67,7 +70,7 @@ class ResponseHandler {
 
   static size_t HeaderCallback(char* buffer, size_t size, size_t nitems,
                                void* userdata) {
-    auto* handler = static_cast<ResponseHandler*>(userdata);
+    auto handler = static_cast<ResponseHandler*>(userdata);
 
     // Remove 2 bytes to ignore trailing \r\n
     ci_string_view header(buffer, size * nitems - 2);
@@ -122,8 +125,7 @@ class RpcResponseHandler : public ResponseHandler {
  public:
   using CallbackType = Aur::RpcResponseCallback;
 
-  RpcResponseHandler(Aur::RpcResponseCallback callback)
-      : callback_(std::move(callback)) {}
+  RpcResponseHandler(CallbackType callback) : callback_(std::move(callback)) {}
 
  private:
   int Run(const std::string& error) const override {
@@ -146,8 +148,7 @@ class RawResponseHandler : public ResponseHandler {
  public:
   using CallbackType = Aur::RawResponseCallback;
 
-  RawResponseHandler(Aur::RawResponseCallback callback)
-      : callback_(std::move(callback)) {}
+  RawResponseHandler(CallbackType callback) : callback_(std::move(callback)) {}
 
  private:
   int Run(const std::string& error) const override {
@@ -161,6 +162,33 @@ class RawResponseHandler : public ResponseHandler {
   const CallbackType callback_;
 };
 
+class CloneResponseHandler : public ResponseHandler {
+ public:
+  using CallbackType = Aur::CloneResponseCallback;
+
+  CloneResponseHandler(Aur* aur, CallbackType callback)
+      : aur_(aur), callback_(std::move(callback)) {}
+
+  Aur* aur() const { return aur_; }
+
+  void SetOperation(std::string operation) {
+    operation_ = std::move(operation);
+  }
+
+ private:
+  int Run(const std::string& error) const override {
+    if (!error.empty()) {
+      return callback_(error);
+    }
+
+    return callback_(CloneResponse{std::move(operation_)});
+  }
+
+  Aur* aur_;
+  const CallbackType callback_;
+  std::string operation_;
+};
+
 }  // namespace
 
 Aur::Aur(const std::string& baseurl) : baseurl_(baseurl) {
@@ -175,6 +203,10 @@ Aur::Aur(const std::string& baseurl) : baseurl_(baseurl) {
   curl_multi_setopt(curl_, CURLMOPT_TIMERFUNCTION, &Aur::TimerCallback);
   curl_multi_setopt(curl_, CURLMOPT_TIMERDATA, this);
 
+  sigset_t ss;
+  sigaddset(&ss, SIGCHLD);
+  sigprocmask(SIG_BLOCK, &ss, &saved_ss_);
+
   sd_event_default(&event_);
 }
 
@@ -183,12 +215,14 @@ Aur::~Aur() {
   curl_global_cleanup();
 
   sd_event_unref(event_);
+
+  sigprocmask(SIG_SETMASK, &saved_ss_, nullptr);
 }
 
 // static
 int Aur::SocketCallback(CURLM* curl, curl_socket_t s, int action,
                         void* userdata, void*) {
-  Aur* aur = static_cast<Aur*>(userdata);
+  auto aur = static_cast<Aur*>(userdata);
 
   auto iter = aur->active_io_.find(s);
   sd_event_source* io = iter != aur->active_io_.end() ? iter->second : nullptr;
@@ -255,7 +289,7 @@ int Aur::SocketCallback(CURLM* curl, curl_socket_t s, int action,
 
 // static
 int Aur::OnIO(sd_event_source* s, int fd, uint32_t revents, void* userdata) {
-  Aur* aur = static_cast<Aur*>(userdata);
+  auto aur = static_cast<Aur*>(userdata);
   int action, k = 0;
 
   // Throwing an exception here would indicate a bug in Aur::SocketCallback.
@@ -275,27 +309,25 @@ int Aur::OnIO(sd_event_source* s, int fd, uint32_t revents, void* userdata) {
     return -EINVAL;
   }
 
-  aur->ProcessDoneEvents();
-  return 0;
+  return aur->ProcessDoneEvents();
 }
 
 // static
 int Aur::OnTimer(sd_event_source* s, uint64_t usec, void* userdata) {
-  Aur* aur = static_cast<Aur*>(userdata);
+  auto aur = static_cast<Aur*>(userdata);
   int k = 0;
 
   if (curl_multi_socket_action(aur->curl_, CURL_SOCKET_TIMEOUT, 0, &k) !=
       CURLM_OK) {
     return -EINVAL;
   }
 
-  aur->ProcessDoneEvents();
-  return 0;
+  return aur->ProcessDoneEvents();
 }
 
 // static
 int Aur::TimerCallback(CURLM* curl, long timeout_ms, void* userdata) {
-  Aur* aur = static_cast<Aur*>(userdata);
+  auto aur = static_cast<Aur*>(userdata);
 
   if (timeout_ms < 0) {
     if (aur->timer_) {
@@ -332,7 +364,7 @@ int Aur::TimerCallback(CURLM* curl, long timeout_ms, void* userdata) {
 
 void Aur::StartRequest(CURL* curl) {
   curl_multi_add_handle(curl_, curl);
-  active_requests_.insert(curl);
+  active_requests_.Add(curl);
 }
 
 int Aur::FinishRequest(CURL* curl, CURLcode result, bool dispatch_callback) {
@@ -354,7 +386,7 @@ int Aur::FinishRequest(CURL* curl, CURLcode result, bool dispatch_callback) {
 
   auto r = dispatch_callback ? handler->RunCallback(error) : 0;
 
-  active_requests_.erase(curl);
+  active_requests_.Remove(curl);
   curl_multi_remove_handle(curl_, curl);
   curl_easy_cleanup(curl);
 
@@ -383,16 +415,11 @@ int Aur::ProcessDoneEvents() {
   return 0;
 }
 
-void Aur::Cancel() {
-  while (!active_requests_.empty()) {
-    FinishRequest(*active_requests_.begin(), CURLE_ABORTED_BY_CALLBACK,
-                  /* dispatch_callback = */ false);
-  }
-}
-
 int Aur::Wait() {
-  while (!active_requests_.empty()) {
-    sd_event_run(event_, 0);
+  while (!active_requests_.IsEmpty()) {
+    if (sd_event_run(event_, 0) < 0) {
+      break;
+    }
   }
 
   return 0;
@@ -473,6 +500,77 @@ void Aur::QueueRequest(
   }
 }
 
+// static
+int Aur::OnCloneExit(sd_event_source* s, const siginfo_t* si, void* userdata) {
+  auto handler = static_cast<CloneResponseHandler*>(userdata);
+
+  handler->aur()->active_requests_.Remove(s);
+  sd_event_source_unref(s);
+
+  std::string error;
+  if (si->si_status != 0) {
+    error.assign("TODO: useful error message for non-zero exit status: " +
+                 std::to_string(si->si_status));
+  }
+
+  return handler->RunCallback(error);
+}
+
+void Aur::QueueCloneRequest(const CloneRequest& request,
+                            const CloneResponseCallback& callback) {
+  auto response_handler = new CloneResponseHandler(this, callback);
+
+  const bool update = fs::exists(fs::path(request.reponame()) / ".git");
+  if (update) {
+    response_handler->SetOperation("update");
+  } else {
+    response_handler->SetOperation("clone");
+  }
+
+  int pid = fork();
+  if (pid < 0) {
+    response_handler->RunCallback(std::string(strerror(errno)));
+    return;
+  }
+
+  if (pid == 0) {
+    auto url = request.Build(baseurl_)[0];
+
+    const char* cmd[] = {
+        NULL,  // git
+        NULL,  // if pulling, -C
+        NULL,  // if pulling, arg to -C
+        NULL,  // 'clone' or 'pull'
+        NULL,  // --quiet
+        NULL,  // --ff-only (if pulling), URL if cloning
+        NULL,
+    };
+    int idx = 0;
+
+    cmd[idx++] = "git";
+    if (update) {
+      cmd[idx++] = "-C";
+      cmd[idx++] = request.reponame().c_str();
+      cmd[idx++] = "pull";
+      cmd[idx++] = "--quiet";
+      cmd[idx++] = "--ff-only";
+    } else {
+      cmd[idx++] = "clone";
+      cmd[idx++] = "--quiet";
+      cmd[idx++] = url.c_str();
+    }
+
+    execvp(cmd[0], const_cast<char* const*>(cmd));
+    _exit(127);
+  }
+
+  sd_event_source* child;
+  sd_event_add_child(event_, &child, pid, WEXITED, &Aur::OnCloneExit,
+                     response_handler);
+
+  active_requests_.Add(child);
+}
+
 void Aur::QueueRawRpcRequest(const RpcRequest& request,
                              const RawResponseCallback& callback) {
   QueueRequest<RawRpcRequestTraits>(request, callback);