Home /Modules/advantage/bulk.h

bulk.h

advantage/transport/bulk.h · 68 line(s) · c

// bulk.h - Rate-limited, parallel bulk acquisition over the secure facade.
//
// Fetches a list of URLs into a single content-addressed store, with:
//   * a bounded worker pool (parallelism)
//   * a global rate limiter (minimum interval between request starts)
//   * bounded retries with exponential backoff for transient failures
//   * the same security posture as AcquisitionStore (TLS verify-on, SSH
//     host-key checking, size caps, idempotent content addressing)
//
// This performs no network access on its own beyond what the supplied URLs
// request; callers remain responsible for pointing it only at sources they
// are authorized to fetch.
#ifndef ADVANTAGE_BULK_H
#define ADVANTAGE_BULK_H

#include <cstddef>
#include <string>
#include <vector>

#include "acquire.h"

namespace advantage {

struct BulkItem {
    std::string url;
    std::string expectedSha256;  // optional per-item content pin
};

struct BulkOptions {
    std::string storeDir = "./advantage-store";
    size_t      concurrency = 4;        // worker threads
    long        minIntervalMs = 0;      // min gap between request *starts* (rate limit)
    int         maxRetries = 2;         // retries after the first attempt
    long        backoffBaseMs = 250;    // exponential backoff base
    AcquireOptions acquire;             // per-fetch transport options
};

struct BulkItemResult {
    std::string url;
    bool        ok = false;
    bool        fromCache = false;
    std::string sha256;
    std::string objectPath;
    size_t      size = 0;
    int         attempts = 0;
    std::string error;
};

struct BulkSummary {
    size_t total = 0;
    size_t succeeded = 0;
    size_t failed = 0;
    size_t fromCache = 0;
    std::vector<BulkItemResult> results;  // in input order
};

// Fetch all items. Thread-safe internally; returns a per-item summary.
BulkSummary bulkFetch(const std::vector<BulkItem>& items, const BulkOptions& opts);

// Convenience: load items from a text file, one URL per line. A line may be
// "URL" or "URL<whitespace>SHA256". Lines beginning with '#' and blank lines
// are ignored.
std::vector<BulkItem> loadItemsFromFile(const std::string& path, std::string& error);

}  // namespace advantage

#endif  // ADVANTAGE_BULK_H