Skip to content

Commit 80bb84f

Browse files
ThomasHickmanThomasHickman
committed
Add in-memory I/O
1 parent 1cc6afd commit 80bb84f

File tree

3 files changed

+140
-4
lines changed

3 files changed

+140
-4
lines changed

‎hfile.c‎

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,7 @@ off_t hseek(hFILE *fp, off_t offset, int whence)
404404
{
405405
off_t curpos, pos;
406406

407-
if (writebuffer_is_nonempty(fp)) {
407+
if (writebuffer_is_nonempty(fp) && fp->mobile) {
408408
int ret = flush_buffer(fp);
409409
if (ret < 0) return ret;
410410
}
@@ -615,6 +615,47 @@ static hFILE *hopen_fd(const char *filename, const char *mode)
615615
return NULL;
616616
}
617617

618+
static hFILE *hpreload_fd(const char *filename, const char *mode)
619+
{
620+
if(!strchr(mode, 'r'))
621+
{
622+
return NULL;
623+
}
624+
625+
hFILE_fd *fp = NULL;
626+
FILE *file = fopen(filename, mode);
627+
if (!file) goto error;
628+
629+
fseek(file, 0, SEEK_END);
630+
int len = ftell(file);
631+
fseek(file, 0, SEEK_SET);
632+
633+
char* buffer = malloc(len);
634+
if(buffer == NULL)
635+
{
636+
errno = ENOMEM;
637+
goto error;
638+
}
639+
if(fread(buffer, 1, len, file) != len)
640+
{
641+
errno = EIO;
642+
goto error;
643+
}
644+
645+
fp = (hFILE_fd *) hfile_init_fixed(sizeof (hFILE_fd), mode, buffer, len, len);
646+
if (fp == NULL) goto error;
647+
648+
fp->fd = fileno(file);
649+
fp->is_socket = 0;
650+
fp->base.backend = &fd_backend;
651+
return &fp->base;
652+
653+
error:
654+
if (file) { int save = errno; (void) fclose(file); errno = save; }
655+
hfile_destroy((hFILE *) fp);
656+
return NULL;
657+
}
658+
618659
hFILE *hdopen(int fd, const char *mode)
619660
{
620661
hFILE_fd *fp = (hFILE_fd*) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));
@@ -821,6 +862,41 @@ static int init_add_plugin(void *obj, int (*init)(struct hFILE_plugin *),
821862
return 0;
822863
}
823864

865+
hFILE *hopenv_mem(const char *filename, const char *mode, va_list args)
866+
{
867+
char* buffer = va_arg(args, char*);
868+
size_t sz = va_arg(args, size_t);
869+
va_end(args);
870+
871+
hFILE_mem *fp = (hFILE_mem *) hfile_init_fixed(sizeof(hFILE_mem), mode, buffer, sz, sz);
872+
873+
fp->base.backend = &mem_backend;
874+
875+
return &fp->base;
876+
}
877+
878+
int hfile_mem_get_buffer(hFILE *file, char **buffer, size_t *length){
879+
if(file->backend != &mem_backend) {
880+
errno = EINVAL;
881+
return -1;
882+
}
883+
884+
*buffer = file->buffer;
885+
*length = file->buffer - file->limit;
886+
887+
return 0;
888+
}
889+
890+
int hfile_plugin_init_mem(struct hFILE_plugin *self)
891+
{
892+
// mem files are declared remote so they work with a tabix index
893+
static const struct hFILE_scheme_handler handler =
894+
{NULL, hfile_always_remote, "mem", 2000 + 50, hopenv_mem};
895+
self->name = "mem";
896+
hfile_add_scheme_handler("mem", &handler);
897+
return 0;
898+
}
899+
824900
static void load_hfile_plugins()
825901
{
826902
static const struct hFILE_scheme_handler
@@ -833,6 +909,7 @@ static void load_hfile_plugins()
833909
hfile_add_scheme_handler("data", &data);
834910
hfile_add_scheme_handler("file", &file);
835911
init_add_plugin(NULL, hfile_plugin_init_net, "knetfile");
912+
init_add_plugin(NULL, hfile_plugin_init_mem, "mem");
836913

837914
#ifdef ENABLE_PLUGINS
838915
struct hts_path_itr path;
@@ -879,11 +956,25 @@ static hFILE *hopen_unknown_scheme(const char *fname, const char *mode)
879956
return fp;
880957
}
881958

959+
static hFILE *hopenv_unknown_scheme(const char *fname, const char *mode, va_list args)
960+
{
961+
char* method_type = va_arg(args, char*);
962+
va_end(args);
963+
if(!strcmp(method_type, "preload")){
964+
errno = EPROTONOSUPPORT;
965+
return NULL;
966+
}
967+
968+
hFILE *fp = hpreload_fd(fname, mode);
969+
if (fp == NULL && errno == ENOENT) errno = EPROTONOSUPPORT;
970+
return fp;
971+
}
972+
882973
/* Returns the appropriate handler, or NULL if the string isn't an URL. */
883974
static const struct hFILE_scheme_handler *find_scheme_handler(const char *s)
884975
{
885976
static const struct hFILE_scheme_handler unknown_scheme =
886-
{ hopen_unknown_scheme, hfile_always_local, "built-in", 0 };
977+
{ hopen_unknown_scheme, hfile_always_local, "built-in", 2000 + 50, hopenv_unknown_scheme };
887978

888979
char scheme[12];
889980
int i;

‎htslib/hfile.h‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ hread(hFILE *fp, void *buffer, size_t nbytes)
202202
if (n > nbytes) n = nbytes;
203203
memcpy(buffer, fp->begin, n);
204204
fp->begin += n;
205-
return (n == nbytes)? (ssize_t) n : hread2(fp, buffer, nbytes, n);
205+
return (n == nbytes || !fp->mobile)? (ssize_t) n : hread2(fp, buffer, nbytes, n);
206206
}
207207

208208
/// Write a character to the stream
@@ -239,7 +239,15 @@ static inline ssize_t HTS_RESULT_USED
239239
hwrite(hFILE *fp, const void *buffer, size_t nbytes)
240240
{
241241
extern ssize_t hwrite2(hFILE *, const void *, size_t, size_t);
242-
242+
extern int hfile_set_blksize(hFILE *fp, size_t bufsiz);
243+
244+
if(!fp->mobile){
245+
if (fp->limit - fp->begin < nbytes){
246+
hfile_set_blksize(fp, fp->limit - fp->buffer + nbytes);
247+
fp->end = fp->limit;
248+
}
249+
}
250+
243251
size_t n = fp->limit - fp->begin;
244252
if (n > nbytes) n = nbytes;
245253
memcpy(fp->begin, buffer, n);
@@ -254,6 +262,11 @@ This includes low-level flushing such as via `fdatasync(2)`.
254262
*/
255263
int hflush(hFILE *fp) HTS_RESULT_USED;
256264

265+
/// For hfile_mem: get the internal buffer and it's size from a hfile
266+
/** @return 0 if successful, or -1 if an error occurred
267+
*/
268+
int hfile_mem_get_buffer(hFILE *file, char **buffer, size_t *length);
269+
257270
#ifdef __cplusplus
258271
}
259272
#endif

‎test/hfile.c‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,38 @@ int main(void)
202202
if ((c = hgetc(fin)) != EOF) fail("chars: hgetc (EOF) returned %d", c);
203203
if (hclose(fin) != 0) fail("hclose(test/hfile_chars.tmp) for reading");
204204

205+
fin = hopen("test/hfile_chars.tmp", "r:", "preload");
206+
if (fin == NULL) fail("preloading hopen(\"test/hfile_chars.tmp\") for reading");
207+
for (i = 0; i < 256; i++)
208+
if ((c = hgetc(fin)) != i)
209+
fail("preloading chars: hgetc (%d = 0x%x) returned %d = 0x%x", i, i, c, c);
210+
if ((c = hgetc(fin)) != EOF) fail("preloading chars: hgetc (EOF) returned %d", c);
211+
if (hclose(fin) != 0) fail("preloading hclose(test/hfile_chars.tmp) for reading");
212+
213+
char* test_string = strdup("Test string");
214+
fin = hopen("mem:", "r:", test_string, 12);
215+
if (fin == NULL) fail("hopen(\"mem:\", \"r:\", ...)");
216+
if (hread(fin, buffer, 12) != 12)
217+
fail("hopen('mem:', 'r') failed read");
218+
if(strcmp(buffer, test_string) != 0)
219+
fail("hopen('mem:', 'r') missread '%s' != '%s'", buffer, test_string);
220+
if (hclose(fin) != 0) fail("hclose mem for reading");
221+
222+
test_string = strdup("Test string");
223+
fin = hopen("mem:", "wr:", test_string, 12);
224+
if (fin == NULL) fail("hopen(\"mem:\", \"w:\", ...)");
225+
if (hseek(fin, -1, SEEK_END) < 0)
226+
fail("hopen('mem:', 'wr') failed seek");
227+
if (hwrite(fin, strdup(" extra"), 7) != 7)
228+
fail("hopen('mem:', 'wr') failed write");
229+
if (hseek(fin, 0, SEEK_SET) < 0)
230+
fail("hopen('mem:', 'wr') failed seek");
231+
if (hread(fin, buffer, 18) != 18)
232+
fail("hopen('mem:', 'wr') failed read");
233+
if (strcmp(buffer, "Test string extra") != 0)
234+
fail("hopen('mem:', 'wr') misswrote '%s' != '%s'", buffer, "Test string extra");
235+
if (hclose(fin) != 0) fail("hclose mem for writing");
236+
205237
fin = hopen("data:,hello, world!%0A", "r");
206238
if (fin == NULL) fail("hopen(\"data:...\")");
207239
n = hread(fin, buffer, 300);

0 commit comments

Comments
 (0)