Home /Modules/advantage/bulk.cpp

bulk.cpp

advantage/transport/bulk.cpp · 142 line(s) · c

// bulk.cpp - Rate-limited parallel bulk acquisition (see bulk.h).
#include "bulk.h"

#include <atomic>
#include <chrono>
#include <condition_variable>
#include <fstream>
#include <mutex>
#include <sstream>
#include <thread>

namespace advantage {

namespace {

// A simple token-paced gate: ensures request *starts* are separated by at
// least minIntervalMs across all workers.
class RateGate {
public:
    explicit RateGate(long minIntervalMs) : interval_(minIntervalMs) {}

    void wait() {
        if (interval_ <= 0) return;
        std::unique_lock<std::mutex> lk(m_);
        auto now = std::chrono::steady_clock::now();
        if (next_ > now) {
            cv_.wait_until(lk, next_);
            now = std::chrono::steady_clock::now();
        }
        auto base = (now > next_) ? now : next_;
        next_ = base + std::chrono::milliseconds(interval_);
    }

private:
    long interval_;
    std::mutex m_;
    std::condition_variable cv_;
    std::chrono::steady_clock::time_point next_ = std::chrono::steady_clock::now();
};

bool transientError(const std::string& e) {
    // Heuristic: network/timeout-ish failures are worth retrying; a hash
    // mismatch or an unsupported scheme is not.
    if (e.find("hash mismatch") != std::string::npos) return false;
    if (e.find("unsupported") != std::string::npos) return false;
    if (e.find("invalid URL") != std::string::npos) return false;
    return true;
}

}  // namespace

BulkSummary bulkFetch(const std::vector<BulkItem>& items, const BulkOptions& opts) {
    BulkSummary summary;
    summary.total = items.size();
    summary.results.resize(items.size());

    if (items.empty()) return summary;

    AcquisitionStore store(opts.storeDir);  // thread-compatible for distinct writes
    RateGate gate(opts.minIntervalMs);

    std::atomic<size_t> nextIndex{0};
    size_t workers = opts.concurrency == 0 ? 1 : opts.concurrency;
    if (workers > items.size()) workers = items.size();

    auto worker = [&]() {
        for (;;) {
            size_t i = nextIndex.fetch_add(1);
            if (i >= items.size()) break;

            const BulkItem& item = items[i];
            BulkItemResult res;
            res.url = item.url;

            AcquireOptions ao = opts.acquire;
            ao.expectedSha256 = item.expectedSha256;

            int attempt = 0;
            for (;;) {
                gate.wait();                 // rate limit request starts
                attempt++;
                AcquireResult r = store.fetch(item.url, ao);
                res.attempts = attempt;
                if (r.ok) {
                    res.ok = true;
                    res.fromCache = r.fromCache;
                    res.sha256 = r.sha256;
                    res.objectPath = r.objectPath;
                    res.size = r.size;
                    res.error.clear();
                    break;
                }
                res.error = r.error;
                if (attempt > opts.maxRetries || !transientError(r.error)) break;

                // Exponential backoff: base * 2^(attempt-1).
                long delay = opts.backoffBaseMs * (1L << (attempt - 1));
                std::this_thread::sleep_for(std::chrono::milliseconds(delay));
            }

            summary.results[i] = res;
        }
    };

    std::vector<std::thread> pool;
    pool.reserve(workers);
    for (size_t w = 0; w < workers; ++w) pool.emplace_back(worker);
    for (auto& t : pool) t.join();

    for (const auto& r : summary.results) {
        if (r.ok) { summary.succeeded++; if (r.fromCache) summary.fromCache++; }
        else summary.failed++;
    }
    return summary;
}

std::vector<BulkItem> loadItemsFromFile(const std::string& path, std::string& error) {
    std::vector<BulkItem> items;
    std::ifstream in(path);
    if (!in) { error = "could not open list file: " + path; return items; }

    std::string line;
    while (std::getline(in, line)) {
        // Trim leading whitespace.
        size_t a = line.find_first_not_of(" \t\r\n");
        if (a == std::string::npos) continue;
        if (line[a] == '#') continue;  // comment
        std::string rest = line.substr(a);

        std::istringstream ss(rest);
        BulkItem it;
        ss >> it.url;
        if (it.url.empty()) continue;
        ss >> it.expectedSha256;  // optional; empty if absent
        items.push_back(it);
    }
    error.clear();
    return items;
}

}  // namespace advantage