Home /Modules/artificial-intelligence/large_language_models_data.c

large_language_models_data.c

artificial-intelligence/large-language-models/large_language_models_data.c · 88 line(s) · c

#include "large_language_models_data.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
struct sllmd_file_source { FILE *file; };
sllmd_status sllmd_file_open(const char *path, sllmd_file_source **out) {
    sllmd_file_source *source;
    if (!path || !path[0] || !out) return SLLMD_INVALID_ARGUMENT;
    *out = NULL;
    source = (sllmd_file_source *)calloc(1, sizeof(*source));
    if (!source) return SLLMD_OUT_OF_MEMORY;
    source->file = fopen(path, "rb");
    if (!source->file) { free(source); return SLLMD_IO_ERROR; }
    *out = source;
    return SLLMD_OK;
}
sllmd_status sllmd_file_read(void *opaque, unsigned char *buffer,
                             size_t capacity, size_t *bytes_read) {
    sllmd_file_source *source = (sllmd_file_source *)opaque;
    size_t count;
    if (!source || !source->file || (!buffer && capacity) || !bytes_read)
        return SLLMD_INVALID_ARGUMENT;
    *bytes_read = 0;
    if (!capacity) return SLLMD_OK;
    count = fread(buffer, 1, capacity, source->file);
    *bytes_read = count;
    if (count < capacity && ferror(source->file)) return SLLMD_IO_ERROR;
    return SLLMD_OK;
}
void sllmd_file_close(sllmd_file_source *source) {
    if (!source) return;
    if (source->file) fclose(source->file);
    free(source);
}
sllmd_status sllmd_jsonl_read(sllmd_read_fn read_fn, void *connector_user_data,
                              size_t max_record_bytes, sllmd_record_fn record_fn,
                              void *record_user_data, size_t *records_read) {
    unsigned char input[8192];
    char *record;
    size_t used = 0, total = 0;
    int saw_any = 0;
    sllmd_status status;
    if (records_read) *records_read = 0;
    if (!read_fn || !record_fn || max_record_bytes == 0 ||
        max_record_bytes == (size_t)-1) return SLLMD_INVALID_ARGUMENT;
    record = (char *)malloc(max_record_bytes + 1);
    if (!record) return SLLMD_OUT_OF_MEMORY;
    for (;;) {
        size_t got = 0, i;
        status = read_fn(connector_user_data, input, sizeof(input), &got);
        if (status != SLLMD_OK) { free(record); return status; }
        if (got > sizeof(input)) { free(record); return SLLMD_IO_ERROR; }
        if (got == 0) break;
        for (i = 0; i < got; ++i) {
            unsigned char ch = input[i];
            saw_any = 1;
            if (ch == 0) { free(record); return SLLMD_INVALID_ARGUMENT; }
            if (ch == '\n') {
                if (used && record[used - 1] == '\r') --used;
                if (used) {
                    record[used] = '\0';
                    if (record_fn(record_user_data, record, used) != 0) {
                        free(record); return SLLMD_CALLBACK_STOP;
                    }
                    ++total;
                }
                used = 0;
            } else {
                if (used >= max_record_bytes) { free(record); return SLLMD_RECORD_TOO_LARGE; }
                record[used++] = (char)ch;
            }
        }
    }
    if (saw_any && used) {
        if (used && record[used - 1] == '\r') --used;
        if (used) {
            record[used] = '\0';
            if (record_fn(record_user_data, record, used) != 0) {
                free(record); return SLLMD_CALLBACK_STOP;
            }
            ++total;
        }
    }
    if (records_read) *records_read = total;
    free(record);
    return SLLMD_OK;
}