Home /Modules/advantage/bulk_cli.cpp

bulk_cli.cpp

advantage/transport/bulk_cli.cpp · 66 line(s) · c

// bulk_cli.cpp - Command-line front end for the bulk parallel fetcher.
//
// Usage:
//   bulkfetch <list-file> [--store DIR] [--jobs N] [--rate MS]
//                         [--retries N] [--insecure] [--accept-new]
//
// <list-file> has one URL per line (optionally "URL SHA256"); blank lines and
// lines starting with '#' are ignored.
#include <cstdlib>
#include <iostream>
#include <string>

#include "bulk.h"

int main(int argc, char** argv) {
    if (argc < 2) {
        std::cerr <<
            "usage: bulkfetch <list-file> [--store DIR] [--jobs N] [--rate MS]\n"
            "                 [--retries N] [--insecure] [--accept-new]\n";
        return 2;
    }

    std::string listFile = argv[1];
    advantage::BulkOptions opts;

    for (int i = 2; i < argc; ++i) {
        std::string a = argv[i];
        if (a == "--store" && i + 1 < argc) opts.storeDir = argv[++i];
        else if (a == "--jobs" && i + 1 < argc) opts.concurrency = (size_t)std::strtoul(argv[++i], nullptr, 10);
        else if (a == "--rate" && i + 1 < argc) opts.minIntervalMs = std::strtol(argv[++i], nullptr, 10);
        else if (a == "--retries" && i + 1 < argc) opts.maxRetries = (int)std::strtol(argv[++i], nullptr, 10);
        else if (a == "--insecure") { opts.acquire.http.verifyPeer = false; opts.acquire.http.verifyHost = false; }
        else if (a == "--accept-new") opts.acquire.sftp.hostKey = advantage::HostKeyPolicy::AcceptNew;
        else { std::cerr << "unknown option: " << a << "\n"; return 2; }
    }

    std::string err;
    auto items = advantage::loadItemsFromFile(listFile, err);
    if (!err.empty()) { std::cerr << err << "\n"; return 1; }
    if (items.empty()) { std::cerr << "no URLs in list\n"; return 1; }

    std::cout << "bulk fetch: " << items.size() << " item(s), jobs="
              << opts.concurrency << ", rate=" << opts.minIntervalMs
              << "ms, retries=" << opts.maxRetries << "\n";

    advantage::BulkSummary s = advantage::bulkFetch(items, opts);

    for (const auto& r : s.results) {
        if (r.ok) {
            std::cout << (r.fromCache ? "  [cache] " : "  [ ok  ] ")
                      << r.sha256.substr(0, 12) << "  " << r.size << "B  "
                      << r.url << (r.attempts > 1 ? "  (" + std::to_string(r.attempts) + " attempts)" : "")
                      << "\n";
        } else {
            std::cout << "  [fail ] " << r.url << "  (" << r.attempts
                      << " attempts): " << r.error << "\n";
        }
    }

    std::cout << "----\n"
              << "total " << s.total << ", succeeded " << s.succeeded
              << " (cached " << s.fromCache << "), failed " << s.failed << "\n";

    return s.failed == 0 ? 0 : 1;
}