From c0e1f4a8cd9b8f0b196faf39044712cb935a4f8b Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Mon, 1 Dec 2025 21:06:22 +0200 Subject: [PATCH 1/5] Revert "Change the internal structure about object type binding to use array fetching for object types." This reverts commit 1fe7ea3ff808cd87726f3cc7a392edd0e73a06cf. --- ext/oci8/object.c | 61 +++--- lib/oci8/cursor.rb | 5 + test/config.rb | 2 +- test/test_dynamic_fetch_roundtrips.c | 279 +++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 38 deletions(-) create mode 100644 test/test_dynamic_fetch_roundtrips.c diff --git a/ext/oci8/object.c b/ext/oci8/object.c index c752e62e..79c70cd8 100644 --- a/ext/oci8/object.c +++ b/ext/oci8/object.c @@ -680,53 +680,43 @@ static VALUE oci8_named_collection_alloc(VALUE klass) return oci8_allocate_typeddata(klass, &oci8_named_collection_data_type); } -typedef struct { - oci8_bind_t bind; - VALUE *obj; -} bind_named_type_t; - static void bind_named_type_mark(oci8_base_t *base) { - bind_named_type_t *bnt = (bind_named_type_t *)base; + oci8_bind_t *obind = (oci8_bind_t *)base; + oci8_hp_obj_t *oho = (oci8_hp_obj_t *)obind->valuep; - if (bnt->obj != NULL) { + if (oho != NULL) { ub4 idx = 0; do { - rb_gc_mark(bnt->obj[idx]); - } while (++idx < bnt->bind.maxar_sz); + rb_gc_mark(oho[idx].obj); + } while (++idx < obind->maxar_sz); } - rb_gc_mark(bnt->bind.tdo); + rb_gc_mark(obind->tdo); } static void bind_named_type_free(oci8_base_t *base) { - bind_named_type_t *bnt = (bind_named_type_t *)base; - void **hp = (void **)bnt->bind.valuep; + oci8_bind_t *obind = (oci8_bind_t *)base; + oci8_hp_obj_t *oho = (oci8_hp_obj_t *)obind->valuep; - if (hp != NULL) { + if (oho != NULL) { ub4 idx = 0; do { - if (hp[idx] != NULL) { - OCIObjectFree(oci8_envhp, oci8_errhp, hp[idx], OCI_DEFAULT); - hp[idx] = NULL; + if (oho[idx].hp != NULL) { + OCIObjectFree(oci8_envhp, oci8_errhp, oho[idx].hp, OCI_DEFAULT); + oho[idx].hp = NULL; } - } while (++idx < bnt->bind.maxar_sz); - } - if (bnt->obj != NULL) { - xfree(bnt->obj); - bnt->obj = NULL; + } while (++idx < obind->maxar_sz); } oci8_bind_free(base); } static VALUE bind_named_type_get(oci8_bind_t *obind, void *data, void *null_struct) { - bind_named_type_t *bnt = (bind_named_type_t *)obind; - ub4 idx = obind->curar_idx; - - return bnt->obj[idx]; + oci8_hp_obj_t *oho = (oci8_hp_obj_t *)data; + return oho->obj; } NORETURN(static void bind_named_type_set(oci8_bind_t *obind, void *data, void **null_structp, VALUE val)); @@ -738,12 +728,10 @@ static void bind_named_type_set(oci8_bind_t *obind, void *data, void **null_stru static void bind_named_type_init(oci8_bind_t *obind, VALUE svc, VALUE val, VALUE length) { - bind_named_type_t *bnt = (bind_named_type_t *)obind; VALUE tdo_obj = length; obind->value_sz = sizeof(void*); - obind->alloc_sz = sizeof(void*); - bnt->obj = xcalloc(sizeof(VALUE), obind->maxar_sz ? obind->maxar_sz : 1); + obind->alloc_sz = sizeof(oci8_hp_obj_t); CHECK_TDO(tdo_obj); RB_OBJ_WRITE(obind->base.self, &obind->tdo, tdo_obj); @@ -751,8 +739,7 @@ static void bind_named_type_init(oci8_bind_t *obind, VALUE svc, VALUE val, VALUE static void bind_named_type_init_elem(oci8_bind_t *obind, VALUE svc) { - bind_named_type_t *bnt = (bind_named_type_t *)obind; - void **hp = (void **)obind->valuep; + oci8_hp_obj_t *oho = (oci8_hp_obj_t *)obind->valuep; oci8_base_t *tdo = DATA_PTR(obind->tdo); OCITypeCode tc = OCITypeTypeCode(oci8_envhp, oci8_errhp, tdo->hp.tdo); VALUE klass = Qnil; @@ -770,14 +757,14 @@ static void bind_named_type_init_elem(oci8_bind_t *obind, VALUE svc) } svcctx = oci8_get_svcctx(svc); do { - bnt->obj[idx] = rb_class_new_instance(0, NULL, klass); - RB_OBJ_WRITTEN(obind->base.self, Qundef, bnt->obj[idx]); - obj = DATA_PTR(bnt->obj[idx]); - RB_OBJ_WRITE(bnt->obj[idx], &obj->tdo, obind->tdo); - obj->instancep = (char**)&hp[idx]; + oho[idx].obj = rb_class_new_instance(0, NULL, klass); + RB_OBJ_WRITTEN(obind->base.self, Qundef, oho[idx].obj); + obj = DATA_PTR(oho[idx].obj); + RB_OBJ_WRITE(oho[idx].obj, &obj->tdo, obind->tdo); + obj->instancep = (char**)&oho[idx].hp; obj->null_structp = (char**)&obind->u.null_structs[idx]; oci8_link_to_parent(&obj->base, &obind->base); - RB_OBJ_WRITTEN(bnt->obj[idx], Qundef, obind->base.self); + RB_OBJ_WRITTEN(oho[idx].obj, Qundef, obind->base.self); chker2(OCIObjectNew(oci8_envhp, oci8_errhp, svcctx->base.hp.svc, tc, tdo->hp.tdo, NULL, OCI_DURATION_SESSION, TRUE, (dvoid**)obj->instancep), &svcctx->base); @@ -819,7 +806,7 @@ static const oci8_bind_data_type_t bind_named_type_data_type = { #endif }, bind_named_type_free, - sizeof(bind_named_type_t) + sizeof(oci8_bind_t) }, bind_named_type_get, bind_named_type_set, diff --git a/lib/oci8/cursor.rb b/lib/oci8/cursor.rb index 966c65e1..70cbbf0f 100644 --- a/lib/oci8/cursor.rb +++ b/lib/oci8/cursor.rb @@ -536,6 +536,11 @@ def define_columns # Rows prefetching doesn't work for CLOB, BLOB and BFILE. # Use array fetching to get more than one row in a network round trip. use_array_fetch = true + when :named_type + # Disable array fetching even when rows prefetching doesn't work. + # It causes SEGV now. + use_array_fetch = false + break end end @fetch_array_size = @prefetch_rows if use_array_fetch diff --git a/test/config.rb b/test/config.rb index d5497fd2..eafdbc3a 100644 --- a/test/config.rb +++ b/test/config.rb @@ -3,7 +3,7 @@ # GRANT EXECUTE ON dbms_lock TO ruby; $dbuser = "ruby" $dbpass = "oci8" -$dbname = nil +$dbname = "127.0.0.1:1521/systempdb" # for test_bind_string_as_nchar in test_encoding.rb ENV['ORA_NCHAR_LITERAL_REPLACE'] = 'TRUE' if OCI8.client_charset_name.include? 'UTF8' diff --git a/test/test_dynamic_fetch_roundtrips.c b/test/test_dynamic_fetch_roundtrips.c new file mode 100644 index 00000000..b0d6ae00 --- /dev/null +++ b/test/test_dynamic_fetch_roundtrips.c @@ -0,0 +1,279 @@ +/* + * Test whether OCIDefineDynamic reduces network round-trips for LOB queries + * Compile: gcc -I$ORACLE_HOME/rdbms/public -L$ORACLE_HOME/lib -lclntsh test_dynamic_fetch_roundtrips.c -o test_dynamic + */ +#include +#include +#include +#include +#include + +static OCIEnv *g_envhp; +static OCIError *g_errhp; +static int callback_count = 0; + +void check_error(sword status, const char *msg) +{ + text errbuf[512]; + sb4 errcode = 0; + + if (status != OCI_SUCCESS && status != OCI_SUCCESS_WITH_INFO) { + OCIErrorGet((dvoid *)g_errhp, (ub4)1, (text *)NULL, &errcode, + errbuf, (ub4)sizeof(errbuf), OCI_HTYPE_ERROR); + printf("ERROR: %s - %.*s\n", msg, 512, errbuf); + exit(1); + } +} + +/* Callback for NUMBER column */ +static sb4 number_callback(void *octxp, OCIDefine *defnp, ub4 iter, + void **bufpp, ub4 **alenp, ub1 *piecep, + void **indpp, ub2 **rcodep) +{ + static int numbers[100]; + static ub4 lengths[100]; + static sb2 indicators[100]; + + callback_count++; + + *bufpp = &numbers[iter]; + *alenp = &lengths[iter]; + *indpp = &indicators[iter]; + *piecep = OCI_ONE_PIECE; + + lengths[iter] = sizeof(int); + + return OCI_CONTINUE; +} + +/* Callback for VARCHAR2 column */ +static sb4 varchar_callback(void *octxp, OCIDefine *defnp, ub4 iter, + void **bufpp, ub4 **alenp, ub1 *piecep, + void **indpp, ub2 **rcodep) +{ + static char buffer[100][200]; + static ub4 lengths[100]; + static sb2 indicators[100]; + + callback_count++; + + *bufpp = buffer[iter]; + *alenp = &lengths[iter]; + *indpp = &indicators[iter]; + *piecep = OCI_ONE_PIECE; + + lengths[iter] = sizeof(buffer[iter]); + + return OCI_CONTINUE; +} + +/* Callback for LOB locator - THIS IS THE KEY TEST */ +static sb4 lob_callback(void *octxp, OCIDefine *defnp, ub4 iter, + void **bufpp, ub4 **alenp, ub1 *piecep, + void **indpp, ub2 **rcodep) +{ + static OCILobLocator *lob_locs[100]; + static ub4 lengths[100]; + static sb2 indicators[100]; + sword status; + + callback_count++; + + printf(" [LOB callback iter=%u]\n", iter); + + /* Allocate LOB descriptor if needed */ + if (lob_locs[iter] == NULL) { + status = OCIDescriptorAlloc((dvoid *)g_envhp, (dvoid **)&lob_locs[iter], + OCI_DTYPE_LOB, 0, NULL); + if (status != OCI_SUCCESS) { + printf(" ERROR: OCIDescriptorAlloc failed\n"); + return OCI_ERROR; + } + printf(" Allocated LOB descriptor for iter %u\n", iter); + } + + /* Provide LOB locator pointer */ + *bufpp = lob_locs[iter]; + *alenp = &lengths[iter]; + *indpp = &indicators[iter]; + *piecep = OCI_ONE_PIECE; + + lengths[iter] = 0; /* Let Oracle fill this in */ + + return OCI_CONTINUE; +} + +int main(int argc, char *argv[]) +{ + OCIServer *srvhp; + OCISvcCtx *svchp; + OCISession *usrhp; + OCIStmt *stmthp; + OCIDefine *defnp_id, *defnp_desc, *defnp_lob; + sword status; + + char *username = "ruby"; + char *password = "ruby"; + char *dbname = "127.0.0.1:1521/systempdb"; + + if (argc >= 4) { + username = argv[1]; + password = argv[2]; + dbname = argv[3]; + } + + printf("=== Testing OCIDefineDynamic with LOB columns ===\n\n"); + + /* Initialize OCI */ + printf("Initializing OCI...\n"); + status = OCIInitialize(OCI_DEFAULT, NULL, NULL, NULL, NULL); + check_error(status, "OCIInitialize"); + + status = OCIEnvInit(&g_envhp, OCI_DEFAULT, 0, NULL); + check_error(status, "OCIEnvInit"); + + status = OCIHandleAlloc(g_envhp, (dvoid **)&g_errhp, OCI_HTYPE_ERROR, 0, NULL); + check_error(status, "OCIHandleAlloc ERROR"); + + status = OCIHandleAlloc(g_envhp, (dvoid **)&srvhp, OCI_HTYPE_SERVER, 0, NULL); + check_error(status, "OCIHandleAlloc SERVER"); + + status = OCIHandleAlloc(g_envhp, (dvoid **)&svchp, OCI_HTYPE_SVCCTX, 0, NULL); + check_error(status, "OCIHandleAlloc SVCCTX"); + + status = OCIHandleAlloc(g_envhp, (dvoid **)&usrhp, OCI_HTYPE_SESSION, 0, NULL); + check_error(status, "OCIHandleAlloc SESSION"); + + /* Connect */ + printf("Connecting to %s as %s...\n", dbname, username); + status = OCIServerAttach(srvhp, g_errhp, (text *)dbname, strlen(dbname), OCI_DEFAULT); + check_error(status, "OCIServerAttach"); + + status = OCIAttrSet(svchp, OCI_HTYPE_SVCCTX, srvhp, 0, OCI_ATTR_SERVER, g_errhp); + check_error(status, "OCIAttrSet SERVER"); + + status = OCIAttrSet(usrhp, OCI_HTYPE_SESSION, username, strlen(username), OCI_ATTR_USERNAME, g_errhp); + check_error(status, "OCIAttrSet USERNAME"); + + status = OCIAttrSet(usrhp, OCI_HTYPE_SESSION, password, strlen(password), OCI_ATTR_PASSWORD, g_errhp); + check_error(status, "OCIAttrSet PASSWORD"); + + status = OCISessionBegin(svchp, g_errhp, usrhp, OCI_CRED_RDBMS, OCI_DEFAULT); + check_error(status, "OCISessionBegin"); + + status = OCIAttrSet(svchp, OCI_HTYPE_SVCCTX, usrhp, 0, OCI_ATTR_SESSION, g_errhp); + check_error(status, "OCIAttrSet SESSION"); + + /* Prepare statement with LOB column */ + printf("\nPreparing statement...\n"); + status = OCIHandleAlloc(g_envhp, (dvoid **)&stmthp, OCI_HTYPE_STMT, 0, NULL); + check_error(status, "OCIHandleAlloc STMT"); + + char *sql = "SELECT id, description, content FROM test_lob_fetch WHERE ROWNUM <= 100 ORDER BY id"; + printf("SQL: %s\n", sql); + status = OCIStmtPrepare(stmthp, g_errhp, (text *)sql, strlen(sql), OCI_NTV_SYNTAX, OCI_DEFAULT); + check_error(status, "OCIStmtPrepare"); + + /* Define columns with DYNAMIC FETCH */ + printf("\nDefining columns with OCI_DYNAMIC_FETCH...\n"); + + // Column 1: id (NUMBER) - dynamic + printf(" Column 1: id (NUMBER)\n"); + status = OCIDefineByPos(stmthp, &defnp_id, g_errhp, 1, NULL, sizeof(int), + SQLT_INT, NULL, NULL, NULL, OCI_DYNAMIC_FETCH); + check_error(status, "OCIDefineByPos id"); + + status = OCIDefineDynamic(defnp_id, g_errhp, NULL, number_callback); + check_error(status, "OCIDefineDynamic id"); + + // Column 2: description (VARCHAR2) - dynamic + printf(" Column 2: description (VARCHAR2)\n"); + status = OCIDefineByPos(stmthp, &defnp_desc, g_errhp, 2, NULL, 200, + SQLT_CHR, NULL, NULL, NULL, OCI_DYNAMIC_FETCH); + check_error(status, "OCIDefineByPos description"); + + status = OCIDefineDynamic(defnp_desc, g_errhp, NULL, varchar_callback); + check_error(status, "OCIDefineDynamic description"); + + // Column 3: content (CLOB) - dynamic, LOB locator + printf(" Column 3: content (CLOB) - THE CRITICAL TEST\n"); + status = OCIDefineByPos(stmthp, &defnp_lob, g_errhp, 3, NULL, 0, + SQLT_CLOB, NULL, NULL, NULL, OCI_DYNAMIC_FETCH); + check_error(status, "OCIDefineByPos CLOB"); + + status = OCIDefineDynamic(defnp_lob, g_errhp, g_envhp, lob_callback); + check_error(status, "OCIDefineDynamic CLOB"); + + /* Execute */ + printf("\nExecuting statement...\n"); + status = OCIStmtExecute(svchp, stmthp, g_errhp, 0, 0, NULL, NULL, OCI_DEFAULT); + check_error(status, "OCIStmtExecute"); + + /* CRITICAL TEST: Fetch 100 rows at once */ + printf("\n=== CRITICAL TEST ===\n"); + printf("Calling OCIStmtFetch(nrows=100)...\n"); + + clock_t start = clock(); + callback_count = 0; + + status = OCIStmtFetch(stmthp, g_errhp, 100, OCI_FETCH_NEXT, OCI_DEFAULT); + + clock_t end = clock(); + double elapsed = ((double)(end - start)) / CLOCKS_PER_SEC; + + if (status != OCI_SUCCESS && status != OCI_NO_DATA) { + check_error(status, "OCIStmtFetch"); + } + + /* Get actual rows fetched */ + ub4 rows_fetched = 0; + OCIAttrGet(stmthp, OCI_HTYPE_STMT, &rows_fetched, NULL, OCI_ATTR_ROWS_FETCHED, g_errhp); + + printf("\n=== RESULTS ===\n"); + printf("OCIStmtFetch returned: %d (%s)\n", status, + status == OCI_SUCCESS ? "SUCCESS" : + status == OCI_NO_DATA ? "NO_DATA" : "OTHER"); + printf("Rows fetched: %u\n", rows_fetched); + printf("Total callback invocations: %d\n", callback_count); + printf("Expected callbacks: %u (3 columns × %u rows)\n", rows_fetched * 3, rows_fetched); + printf("Elapsed time: %.6f seconds\n", elapsed); + + /* Analysis */ + printf("\n=== ANALYSIS ===\n"); + int expected_callbacks = rows_fetched * 3; /* 3 columns */ + + if (callback_count == expected_callbacks) { + printf("✓ Callbacks invoked %d times (correct: %u rows × 3 columns)\n", + callback_count, rows_fetched); + if (elapsed < 0.05) { + printf("✓ Very fast execution (%.6fs) = ONE ROUND TRIP!\n", elapsed); + printf("\n** CONCLUSION: OCIDefineDynamic DOES work with LOBs! **\n"); + printf("** Piecewise fetch is VIABLE - proceed with implementation! **\n"); + } else if (elapsed < 0.2) { + printf("? Moderate execution time (%.6fs)\n", elapsed); + printf(" Suggests possible batching, run with tcpdump to verify\n"); + } else { + printf("✗ Slow execution (%.6fs) = MULTIPLE ROUND TRIPS\n", elapsed); + printf("\n** CONCLUSION: OCIDefineDynamic does NOT reduce round trips **\n"); + printf("** Piecewise fetch is NOT viable - use alternative solutions **\n"); + } + } else { + printf("✗ Unexpected callback count: %d (expected %d)\n", + callback_count, expected_callbacks); + printf(" Investigation needed\n"); + } + + /* Cleanup */ + printf("\nCleaning up...\n"); + OCISessionEnd(svchp, g_errhp, usrhp, OCI_DEFAULT); + OCIServerDetach(srvhp, g_errhp, OCI_DEFAULT); + OCIHandleFree(stmthp, OCI_HTYPE_STMT); + OCIHandleFree(usrhp, OCI_HTYPE_SESSION); + OCIHandleFree(svchp, OCI_HTYPE_SVCCTX); + OCIHandleFree(srvhp, OCI_HTYPE_SERVER); + OCIHandleFree(g_errhp, OCI_HTYPE_ERROR); + OCIHandleFree(g_envhp, OCI_HTYPE_ENV); + + printf("Done.\n"); + return 0; +} From c3cc5fcbd387e235393bc7a1e67d37d4119c0429 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Mon, 1 Dec 2025 21:06:32 +0200 Subject: [PATCH 2/5] Revert "Read lob data without checking readable size to reduce number of network round trips from two to one." This reverts commit 5969e34d1027cd7b1404263584ecbb3b3c2f682d. --- ext/oci8/lob.c | 51 ++++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/ext/oci8/lob.c b/ext/oci8/lob.c index 00be3f7d..4af174d0 100644 --- a/ext/oci8/lob.c +++ b/ext/oci8/lob.c @@ -639,9 +639,10 @@ static VALUE oci8_lob_read(int argc, VALUE *argv, VALUE self) { oci8_lob_t *lob = TO_LOB(self); oci8_svcctx_t *svcctx = check_svcctx(lob); + ub8 lob_length; + ub8 read_len; ub8 pos = lob->pos; - long strbufsiz = 512; - ub8 sz; + long strbufsiz; ub8 byte_amt; ub8 char_amt; sword rv; @@ -651,21 +652,36 @@ static VALUE oci8_lob_read(int argc, VALUE *argv, VALUE self) ub1 piece = OCI_FIRST_PIECE; rb_scan_args(argc, argv, "01", &size); + lob_length = oci8_lob_get_length(lob); + if (lob_length == 0 && NIL_P(size)) { + return rb_usascii_str_new("", 0); + } + if (lob_length <= pos) /* EOF */ + return Qnil; if (NIL_P(size)) { - sz = UB4MAXVAL; + read_len = lob_length - pos; } else { - sz = NUM2ULL(size); - } - if (lob->state == S_BFILE_CLOSE) { - open_bfile(svcctx, lob, errhp); + ub8 sz = NUM2ULL(size); + read_len = MIN(sz, lob_length - pos); } -read_more_data: if (lob->lobtype == OCI_TEMP_CLOB) { byte_amt = 0; - char_amt = sz; + char_amt = read_len; + if (oci8_nls_ratio == 1) { + strbufsiz = MIN(read_len, ULONG_MAX); + } else { + strbufsiz = MIN(read_len + read_len / 8, ULONG_MAX); + } + if (strbufsiz <= 10) { + strbufsiz = 10; + } } else { - byte_amt = sz; + byte_amt = read_len; char_amt = 0; + strbufsiz = MIN(read_len, ULONG_MAX); + } + if (lob->state == S_BFILE_CLOSE) { + open_bfile(svcctx, lob, errhp); } do { VALUE strbuf = rb_str_buf_new(strbufsiz); @@ -695,24 +711,15 @@ static VALUE oci8_lob_read(int argc, VALUE *argv, VALUE self) } rb_str_set_len(strbuf, byte_amt); rb_ary_push(v, strbuf); - if (strbufsiz < 128 * 1024 * 1024) { - strbufsiz *= 2; - } } while (rv == OCI_NEED_DATA); - if (NIL_P(size) && pos - lob->pos == sz) { - lob->pos = pos; - piece = OCI_FIRST_PIECE; - goto read_more_data; + if (pos >= lob_length) { + bfile_close(lob); } lob->pos = pos; switch (RARRAY_LEN(v)) { case 0: - if (NIL_P(size) && pos == 0) { - return rb_usascii_str_new("", 0); - } else { - return Qnil; - } + return Qnil; case 1: v = RARRAY_AREF(v, 0); break; From 5b4d2c80e3c2497ebb0bbebe9d2e59731588c39e Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Mon, 1 Dec 2025 21:06:46 +0200 Subject: [PATCH 3/5] Revert "Use array fetching when LOB colums are in a query and no object types are in the query." This reverts commit a35c64d43bfa1391abf4e3d3110cf0a11e0408c1. --- ext/oci8/bind.c | 15 +++-------- ext/oci8/stmt.c | 29 ++++++--------------- lib/oci8/cursor.rb | 64 ++++++++++------------------------------------ lib/oci8/oci8.rb | 2 +- test/test_oci8.rb | 22 ---------------- 5 files changed, 25 insertions(+), 107 deletions(-) diff --git a/ext/oci8/bind.c b/ext/oci8/bind.c index 624f0d7c..824eb46b 100644 --- a/ext/oci8/bind.c +++ b/ext/oci8/bind.c @@ -633,20 +633,11 @@ static VALUE oci8_bind_get(VALUE self) return data_type->get(obind, (void*)((size_t)obind->valuep + obind->alloc_sz * idx), null_structp); } -static VALUE oci8_bind_get_data(int argc, VALUE *argv, VALUE self) +static VALUE oci8_bind_get_data(VALUE self) { oci8_bind_t *obind = TO_BIND(self); - VALUE index; - rb_scan_args(argc, argv, "01", &index); - if (!NIL_P(index)) { - ub4 idx = NUM2UINT(index); - if (idx >= obind->maxar_sz) { - rb_raise(rb_eRuntimeError, "data index is too big. (%u for %u)", idx, obind->maxar_sz); - } - obind->curar_idx = idx; - return rb_funcall(self, oci8_id_get, 0); - } else if (obind->maxar_sz == 0) { + if (obind->maxar_sz == 0) { obind->curar_idx = 0; return rb_funcall(self, oci8_id_get, 0); } else { @@ -817,7 +808,7 @@ void Init_oci8_bind(VALUE klass) rb_define_method(cOCI8BindTypeBase, "initialize", oci8_bind_initialize, 4); rb_define_method(cOCI8BindTypeBase, "get", oci8_bind_get, 0); rb_define_method(cOCI8BindTypeBase, "set", oci8_bind_set, 1); - rb_define_private_method(cOCI8BindTypeBase, "get_data", oci8_bind_get_data, -1); + rb_define_private_method(cOCI8BindTypeBase, "get_data", oci8_bind_get_data, 0); rb_define_private_method(cOCI8BindTypeBase, "set_data", oci8_bind_set_data, 1); rb_define_singleton_method(klass, "initial_chunk_size", get_initial_chunk_size, 0); diff --git a/ext/oci8/stmt.c b/ext/oci8/stmt.c index 00f1b60a..dbe9e011 100644 --- a/ext/oci8/stmt.c +++ b/ext/oci8/stmt.c @@ -15,8 +15,7 @@ static VALUE cOCIStmt; typedef struct { oci8_base_t base; VALUE svc; - char use_stmt_release; - char end_of_fetch; + int use_stmt_release; } oci8_stmt_t; static void oci8_stmt_mark(oci8_base_t *base) @@ -261,7 +260,6 @@ static VALUE oci8_stmt_execute(VALUE self, VALUE iteration_count) oci8_stmt_t *stmt = TO_STMT(self); oci8_svcctx_t *svcctx = oci8_get_svcctx(stmt->svc); - stmt->end_of_fetch = 0; chker3(oci8_call_stmt_execute(svcctx, stmt, NUM2UINT(iteration_count), svcctx->is_autocommit ? OCI_COMMIT_ON_SUCCESS : OCI_DEFAULT), &stmt->base, stmt->base.hp.stmt); @@ -269,7 +267,7 @@ static VALUE oci8_stmt_execute(VALUE self, VALUE iteration_count) } /* - * @overload __fetch(connection, max_rows) + * @overload __fetch(connection) * * Fetches one row and set the result to @define_handles. * This is called by private methods of OCI8::Cursor. @@ -279,18 +277,13 @@ static VALUE oci8_stmt_execute(VALUE self, VALUE iteration_count) * * @private */ -static VALUE oci8_stmt_fetch(VALUE self, VALUE svc, VALUE max_rows) +static VALUE oci8_stmt_fetch(VALUE self, VALUE svc) { oci8_stmt_t *stmt = TO_STMT(self); oci8_svcctx_t *svcctx = oci8_get_svcctx(svc); sword rv; oci8_bind_t *obind; const oci8_bind_data_type_t *data_type; - ub4 nrows = NUM2UINT(max_rows); - - if (stmt->end_of_fetch) { - return Qnil; - } if (stmt->base.children != NULL) { obind = (oci8_bind_t *)stmt->base.children; @@ -300,22 +293,16 @@ static VALUE oci8_stmt_fetch(VALUE self, VALUE svc, VALUE max_rows) if (data_type->pre_fetch_hook != NULL) { data_type->pre_fetch_hook(obind, stmt->svc); } - if (nrows > 1 && nrows != obind->maxar_sz) { - rb_raise(rb_eRuntimeError, "fetch size (%u) != define-handle size %u", nrows, obind->maxar_sz); - } } obind = (oci8_bind_t *)obind->base.next; } while (obind != (oci8_bind_t*)stmt->base.children); } - rv = OCIStmtFetch_nb(svcctx, stmt->base.hp.stmt, oci8_errhp, nrows, OCI_FETCH_NEXT, OCI_DEFAULT); + rv = OCIStmtFetch_nb(svcctx, stmt->base.hp.stmt, oci8_errhp, 1, OCI_FETCH_NEXT, OCI_DEFAULT); if (rv == OCI_NO_DATA) { - stmt->end_of_fetch = 1; - } else { - chker3(rv, &svcctx->base, stmt->base.hp.stmt); + return Qfalse; } - chker2(OCIAttrGet(stmt->base.hp.stmt, OCI_HTYPE_STMT, &nrows, 0, OCI_ATTR_ROWS_FETCHED, oci8_errhp), - &svcctx->base); - return nrows ? UINT2NUM(nrows) : Qnil; + chker3(rv, &svcctx->base, stmt->base.hp.stmt); + return Qtrue; } /* @@ -440,7 +427,7 @@ void Init_oci8_stmt(VALUE cOCI8) rb_define_private_method(cOCIStmt, "__define", oci8_define_by_pos, 2); rb_define_private_method(cOCIStmt, "__bind", oci8_bind, 2); rb_define_private_method(cOCIStmt, "__execute", oci8_stmt_execute, 1); - rb_define_private_method(cOCIStmt, "__fetch", oci8_stmt_fetch, 2); + rb_define_private_method(cOCIStmt, "__fetch", oci8_stmt_fetch, 1); rb_define_private_method(cOCIStmt, "__paramGet", oci8_stmt_get_param, 1); rb_define_method(cOCIStmt, "rowid", oci8_stmt_get_rowid, 0); diff --git a/lib/oci8/cursor.rb b/lib/oci8/cursor.rb index 70cbbf0f..66092ec8 100644 --- a/lib/oci8/cursor.rb +++ b/lib/oci8/cursor.rb @@ -25,11 +25,7 @@ def initialize(conn, sql = nil) @names = nil @con = conn @max_array_size = nil - @fetch_array_size = nil - @rowbuf_size = 0 - @rowbuf_index = 0 __initialize(conn, sql) # Initialize the internal C structure. - self.prefetch_rows = conn.instance_variable_get(:@prefetch_rows) end # explicitly indicate the date type of fetched value. run this @@ -42,7 +38,7 @@ def initialize(conn, sql = nil) # cursor.define(2, Time) # fetch the second column as Time. # cursor.exec() def define(pos, type, length = nil) - bindobj = make_bind_object({:type => type, :length => length}, @fetch_array_size || 1) + bindobj = make_bind_object(:type => type, :length => length) __define(pos, bindobj) if old = @define_handles[pos - 1] old.send(:free) @@ -130,8 +126,6 @@ def exec(*bindvars) when :select_stmt __execute(0) define_columns() if @column_metadata.size == 0 - @rowbuf_size = 0 - @rowbuf_index = 0 @column_metadata.size else __execute(1) @@ -390,7 +384,6 @@ def keys # @param [Integer] rows The number of rows to be prefetched def prefetch_rows=(rows) attr_set_ub4(11, rows) # OCI_ATTR_PREFETCH_ROWS(11) - @prefetch_rows = rows end if OCI8::oracle_client_version >= ORAVER_12_1 @@ -475,7 +468,7 @@ def type private - def make_bind_object(param, fetch_array_size = nil) + def make_bind_object(param) case param when Hash key = param[:type] @@ -517,42 +510,22 @@ def make_bind_object(param, fetch_array_size = nil) OCI8::BindType::Mapping[key] = bindclass if bindclass end raise "unsupported datatype: #{key}" if bindclass.nil? - bindclass.create(@con, val, param, fetch_array_size || max_array_size) + bindclass.create(@con, val, param, max_array_size) end - @@use_array_fetch = false - def define_columns # http://docs.oracle.com/cd/E11882_01/appdev.112/e10646/ociaahan.htm#sthref5494 num_cols = attr_get_ub4(18) # OCI_ATTR_PARAM_COUNT(18) - @column_metadata = 1.upto(num_cols).collect do |i| - __paramGet(i) - end - if @define_handles.size == 0 - use_array_fetch = @@use_array_fetch - @column_metadata.each do |md| - case md.data_type - when :clob, :blob, :bfile - # Rows prefetching doesn't work for CLOB, BLOB and BFILE. - # Use array fetching to get more than one row in a network round trip. - use_array_fetch = true - when :named_type - # Disable array fetching even when rows prefetching doesn't work. - # It causes SEGV now. - use_array_fetch = false - break - end - end - @fetch_array_size = @prefetch_rows if use_array_fetch - end - @column_metadata.each_with_index do |md, i| - define_one_column(i + 1, md) unless @define_handles[i] + 1.upto(num_cols) do |i| + parm = __paramGet(i) + define_one_column(i, parm) unless @define_handles[i - 1] + @column_metadata[i - 1] = parm end num_cols end def define_one_column(pos, param) - bindobj = make_bind_object(param, @fetch_array_size || 1) + bindobj = make_bind_object(param) __define(pos, bindobj) @define_handles[pos - 1] = bindobj end @@ -567,33 +540,22 @@ def bind_params(*bindvars) end end - def fetch_row_internal - if @rowbuf_size && @rowbuf_size == @rowbuf_index - @rowbuf_size = __fetch(@con, @fetch_array_size || 1) - @rowbuf_index = 0 - end - @rowbuf_size - end - def fetch_one_row_as_array - if fetch_row_internal - ret = @define_handles.collect do |handle| - handle.send(:get_data, @rowbuf_index) + if __fetch(@con) + @define_handles.collect do |handle| + handle.send(:get_data) end - @rowbuf_index += 1 - ret else nil end end def fetch_one_row_as_hash - if fetch_row_internal + if __fetch(@con) ret = {} get_col_names.each_with_index do |name, idx| - ret[name] = @define_handles[idx].send(:get_data, @rowbuf_index) + ret[name] = @define_handles[idx].send(:get_data) end - @rowbuf_index += 1 ret else nil diff --git a/lib/oci8/oci8.rb b/lib/oci8/oci8.rb index c7b9bb86..76297af0 100644 --- a/lib/oci8/oci8.rb +++ b/lib/oci8/oci8.rb @@ -169,6 +169,7 @@ def parse(sql) # @private def parse_internal(sql) cursor = OCI8::Cursor.new(self, sql) + cursor.prefetch_rows = @prefetch_rows if @prefetch_rows cursor end @@ -304,7 +305,6 @@ def exec_internal(sql, *bindvars) # @return [Array] an array of first row. def select_one(sql, *bindvars) cursor = self.parse(sql) - cursor.prefetch_rows = 1 begin cursor.exec(*bindvars) row = cursor.fetch diff --git a/test/test_oci8.rb b/test/test_oci8.rb index 6c7d6f0b..18226c70 100644 --- a/test/test_oci8.rb +++ b/test/test_oci8.rb @@ -539,7 +539,6 @@ def test_last_error assert_nil(@conn.last_error) @conn.last_error = 'dummy' cursor = @conn.parse('select col1, max(col2) from (select 1 as col1, null as col2 from dual) group by col1') - cursor.prefetch_rows = 1 assert_nil(@conn.last_error) # When an OCI function returns OCI_SUCCESS_WITH_INFO, OCI8#last_error is set. @@ -600,25 +599,4 @@ def test_server_version end assert_equal(ver, @conn.oracle_server_version.to_s) end - - def test_array_fetch - drop_table('test_table') - @conn.exec("CREATE TABLE test_table (id number, val clob)") - cursor = @conn.parse("INSERT INTO test_table VALUES (:1, :2)") - 1.upto(10) do |i| - cursor.exec(i, ('a'.ord + i).chr * i) - end - cursor.close - cursor = @conn.parse("select * from test_table where id <= :1 order by id") - cursor.prefetch_rows = 4 - [1, 6, 2, 7, 3, 8, 4, 9, 5, 10].each do |i| - cursor.exec(i) - 1.upto(i) do |j| - row = cursor.fetch - assert_equal(j, row[0]) - assert_equal(('a'.ord + j).chr * j, row[1].read) - end - assert_nil(cursor.fetch) - end - end end # TestOCI8 From 696a2ac98e7694dd8d28ecc73aecc1b698d952be Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Thu, 4 Dec 2025 14:44:54 +0200 Subject: [PATCH 4/5] efficient piecewise LOB fetching --- lib/oci8/bindtype.rb | 11 +- lib/oci8/oci8.rb | 64 ++++++ test/config.rb | 2 +- test/test_clob.rb | 7 + test/test_dbi_clob.rb | 7 + test/test_dynamic_fetch_roundtrips.c | 279 --------------------------- test/test_encoding.rb | 7 + test/test_large_lob.rb | 99 ++++++++++ test/test_object.rb | 4 + test/test_oci8.rb | 60 +++--- 10 files changed, 227 insertions(+), 313 deletions(-) delete mode 100644 test/test_dynamic_fetch_roundtrips.c create mode 100644 test/test_large_lob.rb diff --git a/lib/oci8/bindtype.rb b/lib/oci8/bindtype.rb index 6151c9db..e86bb96e 100644 --- a/lib/oci8/bindtype.rb +++ b/lib/oci8/bindtype.rb @@ -245,13 +245,18 @@ def self.create(con, val, param, max_array_size) # datatype type size prec scale # ------------------------------------------------- # CLOB SQLT_CLOB 4000 0 0 -OCI8::BindType::Mapping[:clob] = OCI8::BindType::CLOB -OCI8::BindType::Mapping[:nclob] = OCI8::BindType::NCLOB +# NCLOB SQLT_CLOB 4000 0 0 +# Default: Fetch as String using SQLT_CHR (fast, max 2GB) +# See: OCI8::lob_fetch_mode +OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long +OCI8::BindType::Mapping[:nclob] = OCI8::BindType::Long # datatype type size prec scale # ------------------------------------------------- # BLOB SQLT_BLOB 4000 0 0 -OCI8::BindType::Mapping[:blob] = OCI8::BindType::BLOB +# Default: Fetch as binary String using SQLT_CHR (fast, max 2GB) +# See: OCI8::lob_fetch_mode +OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw # datatype type size prec scale # ------------------------------------------------- diff --git a/lib/oci8/oci8.rb b/lib/oci8/oci8.rb index 76297af0..076237c8 100644 --- a/lib/oci8/oci8.rb +++ b/lib/oci8/oci8.rb @@ -152,6 +152,7 @@ def initialize(*args) end @prefetch_rows = 100 + @lob_prefetch_size = 0 # 0 means use Oracle default (disabled) @username = nil end @@ -333,6 +334,27 @@ def prefetch_rows=(num) @prefetch_rows = num end + # Sets the LOB prefetch size in bytes. Only used when lob_fetch_mode is :locator. + # i.e. sets OCI_ATTR_DEFAULT_LOBPREFETCH_SIZE on the session handle. + # The default value is 0 (disabled). + # + # When set to a non-zero value (e.g., 65536 for 64KB), Oracle prefetches + # LOB data along with the row if the LOB size is <= this value. + # This reduces network round trips when fetching small to medium LOBs. + # + # This is a session-wide setting that applies to all LOB columns fetched + # from this connection. + # + # It has no effect when lob_fetch_mode is :long_as_string (the default). + # + # @param [Integer] size prefetch size in bytes (0 to disable) + # @see OCI8::lob_fetch_mode= + # @note Requires Oracle 11g or later + def lob_prefetch_size=(size) + @lob_prefetch_size = size + @session_handle.send(:attr_set_ub4, 438, size) + end + # @private def inspect "#" @@ -370,6 +392,48 @@ def self.client_charset_name @@client_charset_name end + # Returns the current LOB fetch mode. + # + # @return [Symbol] :long_as_string or :locator + # @see lob_fetch_mode= + def self.lob_fetch_mode + @@lob_fetch_mode ||= :long_as_string + end + + # Sets the LOB fetch mode. + # + # - +:long_as_string+ (default): Fetch LOBs as Strings using + # Runtime Data Allocation and Piecewise Operations in OCI. + # Fastest and most efficient fetch but limited to 2GB LOBs. + # + # - +:locator+: Fetch LOB locators (OCI8::CLOB/BLOB objects). + # Calls OCILobRead2() on the lob fields individually. + # Required for LOBs > 2GB, random access, or read/write operations. + # @conn.lob_prefetch_size may reduce network roundtrips but my + # unscientific testing only showed performance degradation. + # + # @param [Symbol] mode :long_as_string or :locator + # @return [Symbol] the mode that was set + # @see https://github.com/oracle/odpi/issues/163 + def self.lob_fetch_mode=(mode) + unless [:long_as_string, :locator].include?(mode) + raise ArgumentError, "lob_fetch_mode must be :long_as_string or :locator" + end + + case mode + when :long_as_string + OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long + OCI8::BindType::Mapping[:nclob] = OCI8::BindType::Long + OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw + when :locator + OCI8::BindType::Mapping[:clob] = OCI8::BindType::CLOB + OCI8::BindType::Mapping[:nclob] = OCI8::BindType::NCLOB + OCI8::BindType::Mapping[:blob] = OCI8::BindType::BLOB + end + + @@lob_fetch_mode = mode + end + if OCI8.oracle_client_version >= OCI8::ORAVER_11_1 # Returns send timeout in seconds. # Zero means no timeout. diff --git a/test/config.rb b/test/config.rb index eafdbc3a..d5497fd2 100644 --- a/test/config.rb +++ b/test/config.rb @@ -3,7 +3,7 @@ # GRANT EXECUTE ON dbms_lock TO ruby; $dbuser = "ruby" $dbpass = "oci8" -$dbname = "127.0.0.1:1521/systempdb" +$dbname = nil # for test_bind_string_as_nchar in test_encoding.rb ENV['ORA_NCHAR_LITERAL_REPLACE'] = 'TRUE' if OCI8.client_charset_name.include? 'UTF8' diff --git a/test/test_clob.rb b/test/test_clob.rb index 748d4b86..13385ffd 100644 --- a/test/test_clob.rb +++ b/test/test_clob.rb @@ -6,10 +6,17 @@ class TestCLob < Minitest::Test def setup @conn = get_oci8_connection + # This test needs LOB locators for read/write operations + OCI8.lob_fetch_mode = :locator drop_table('test_table') @conn.exec('CREATE TABLE test_table (filename VARCHAR2(40), content CLOB)') end + def teardown + # Restore default mode + OCI8.lob_fetch_mode = :long_as_string if @conn + end + def test_insert filename = File.basename($lobfile) @conn.exec("DELETE FROM test_table WHERE filename = :1", filename) diff --git a/test/test_dbi_clob.rb b/test/test_dbi_clob.rb index b2c7aa26..fe092087 100644 --- a/test/test_dbi_clob.rb +++ b/test/test_dbi_clob.rb @@ -6,10 +6,17 @@ class TestDbiCLob < Minitest::Test def setup @dbh = get_dbi_connection() + # This test needs LOB locators for read/write operations + OCI8.lob_fetch_mode = :locator drop_table('test_table') @dbh.execute('CREATE TABLE test_table (filename VARCHAR2(40), content CLOB)') end + def teardown + # Restore default mode + OCI8.lob_fetch_mode = :long_as_string if @dbh + end + def test_insert filename = File.basename($lobfile) diff --git a/test/test_dynamic_fetch_roundtrips.c b/test/test_dynamic_fetch_roundtrips.c deleted file mode 100644 index b0d6ae00..00000000 --- a/test/test_dynamic_fetch_roundtrips.c +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Test whether OCIDefineDynamic reduces network round-trips for LOB queries - * Compile: gcc -I$ORACLE_HOME/rdbms/public -L$ORACLE_HOME/lib -lclntsh test_dynamic_fetch_roundtrips.c -o test_dynamic - */ -#include -#include -#include -#include -#include - -static OCIEnv *g_envhp; -static OCIError *g_errhp; -static int callback_count = 0; - -void check_error(sword status, const char *msg) -{ - text errbuf[512]; - sb4 errcode = 0; - - if (status != OCI_SUCCESS && status != OCI_SUCCESS_WITH_INFO) { - OCIErrorGet((dvoid *)g_errhp, (ub4)1, (text *)NULL, &errcode, - errbuf, (ub4)sizeof(errbuf), OCI_HTYPE_ERROR); - printf("ERROR: %s - %.*s\n", msg, 512, errbuf); - exit(1); - } -} - -/* Callback for NUMBER column */ -static sb4 number_callback(void *octxp, OCIDefine *defnp, ub4 iter, - void **bufpp, ub4 **alenp, ub1 *piecep, - void **indpp, ub2 **rcodep) -{ - static int numbers[100]; - static ub4 lengths[100]; - static sb2 indicators[100]; - - callback_count++; - - *bufpp = &numbers[iter]; - *alenp = &lengths[iter]; - *indpp = &indicators[iter]; - *piecep = OCI_ONE_PIECE; - - lengths[iter] = sizeof(int); - - return OCI_CONTINUE; -} - -/* Callback for VARCHAR2 column */ -static sb4 varchar_callback(void *octxp, OCIDefine *defnp, ub4 iter, - void **bufpp, ub4 **alenp, ub1 *piecep, - void **indpp, ub2 **rcodep) -{ - static char buffer[100][200]; - static ub4 lengths[100]; - static sb2 indicators[100]; - - callback_count++; - - *bufpp = buffer[iter]; - *alenp = &lengths[iter]; - *indpp = &indicators[iter]; - *piecep = OCI_ONE_PIECE; - - lengths[iter] = sizeof(buffer[iter]); - - return OCI_CONTINUE; -} - -/* Callback for LOB locator - THIS IS THE KEY TEST */ -static sb4 lob_callback(void *octxp, OCIDefine *defnp, ub4 iter, - void **bufpp, ub4 **alenp, ub1 *piecep, - void **indpp, ub2 **rcodep) -{ - static OCILobLocator *lob_locs[100]; - static ub4 lengths[100]; - static sb2 indicators[100]; - sword status; - - callback_count++; - - printf(" [LOB callback iter=%u]\n", iter); - - /* Allocate LOB descriptor if needed */ - if (lob_locs[iter] == NULL) { - status = OCIDescriptorAlloc((dvoid *)g_envhp, (dvoid **)&lob_locs[iter], - OCI_DTYPE_LOB, 0, NULL); - if (status != OCI_SUCCESS) { - printf(" ERROR: OCIDescriptorAlloc failed\n"); - return OCI_ERROR; - } - printf(" Allocated LOB descriptor for iter %u\n", iter); - } - - /* Provide LOB locator pointer */ - *bufpp = lob_locs[iter]; - *alenp = &lengths[iter]; - *indpp = &indicators[iter]; - *piecep = OCI_ONE_PIECE; - - lengths[iter] = 0; /* Let Oracle fill this in */ - - return OCI_CONTINUE; -} - -int main(int argc, char *argv[]) -{ - OCIServer *srvhp; - OCISvcCtx *svchp; - OCISession *usrhp; - OCIStmt *stmthp; - OCIDefine *defnp_id, *defnp_desc, *defnp_lob; - sword status; - - char *username = "ruby"; - char *password = "ruby"; - char *dbname = "127.0.0.1:1521/systempdb"; - - if (argc >= 4) { - username = argv[1]; - password = argv[2]; - dbname = argv[3]; - } - - printf("=== Testing OCIDefineDynamic with LOB columns ===\n\n"); - - /* Initialize OCI */ - printf("Initializing OCI...\n"); - status = OCIInitialize(OCI_DEFAULT, NULL, NULL, NULL, NULL); - check_error(status, "OCIInitialize"); - - status = OCIEnvInit(&g_envhp, OCI_DEFAULT, 0, NULL); - check_error(status, "OCIEnvInit"); - - status = OCIHandleAlloc(g_envhp, (dvoid **)&g_errhp, OCI_HTYPE_ERROR, 0, NULL); - check_error(status, "OCIHandleAlloc ERROR"); - - status = OCIHandleAlloc(g_envhp, (dvoid **)&srvhp, OCI_HTYPE_SERVER, 0, NULL); - check_error(status, "OCIHandleAlloc SERVER"); - - status = OCIHandleAlloc(g_envhp, (dvoid **)&svchp, OCI_HTYPE_SVCCTX, 0, NULL); - check_error(status, "OCIHandleAlloc SVCCTX"); - - status = OCIHandleAlloc(g_envhp, (dvoid **)&usrhp, OCI_HTYPE_SESSION, 0, NULL); - check_error(status, "OCIHandleAlloc SESSION"); - - /* Connect */ - printf("Connecting to %s as %s...\n", dbname, username); - status = OCIServerAttach(srvhp, g_errhp, (text *)dbname, strlen(dbname), OCI_DEFAULT); - check_error(status, "OCIServerAttach"); - - status = OCIAttrSet(svchp, OCI_HTYPE_SVCCTX, srvhp, 0, OCI_ATTR_SERVER, g_errhp); - check_error(status, "OCIAttrSet SERVER"); - - status = OCIAttrSet(usrhp, OCI_HTYPE_SESSION, username, strlen(username), OCI_ATTR_USERNAME, g_errhp); - check_error(status, "OCIAttrSet USERNAME"); - - status = OCIAttrSet(usrhp, OCI_HTYPE_SESSION, password, strlen(password), OCI_ATTR_PASSWORD, g_errhp); - check_error(status, "OCIAttrSet PASSWORD"); - - status = OCISessionBegin(svchp, g_errhp, usrhp, OCI_CRED_RDBMS, OCI_DEFAULT); - check_error(status, "OCISessionBegin"); - - status = OCIAttrSet(svchp, OCI_HTYPE_SVCCTX, usrhp, 0, OCI_ATTR_SESSION, g_errhp); - check_error(status, "OCIAttrSet SESSION"); - - /* Prepare statement with LOB column */ - printf("\nPreparing statement...\n"); - status = OCIHandleAlloc(g_envhp, (dvoid **)&stmthp, OCI_HTYPE_STMT, 0, NULL); - check_error(status, "OCIHandleAlloc STMT"); - - char *sql = "SELECT id, description, content FROM test_lob_fetch WHERE ROWNUM <= 100 ORDER BY id"; - printf("SQL: %s\n", sql); - status = OCIStmtPrepare(stmthp, g_errhp, (text *)sql, strlen(sql), OCI_NTV_SYNTAX, OCI_DEFAULT); - check_error(status, "OCIStmtPrepare"); - - /* Define columns with DYNAMIC FETCH */ - printf("\nDefining columns with OCI_DYNAMIC_FETCH...\n"); - - // Column 1: id (NUMBER) - dynamic - printf(" Column 1: id (NUMBER)\n"); - status = OCIDefineByPos(stmthp, &defnp_id, g_errhp, 1, NULL, sizeof(int), - SQLT_INT, NULL, NULL, NULL, OCI_DYNAMIC_FETCH); - check_error(status, "OCIDefineByPos id"); - - status = OCIDefineDynamic(defnp_id, g_errhp, NULL, number_callback); - check_error(status, "OCIDefineDynamic id"); - - // Column 2: description (VARCHAR2) - dynamic - printf(" Column 2: description (VARCHAR2)\n"); - status = OCIDefineByPos(stmthp, &defnp_desc, g_errhp, 2, NULL, 200, - SQLT_CHR, NULL, NULL, NULL, OCI_DYNAMIC_FETCH); - check_error(status, "OCIDefineByPos description"); - - status = OCIDefineDynamic(defnp_desc, g_errhp, NULL, varchar_callback); - check_error(status, "OCIDefineDynamic description"); - - // Column 3: content (CLOB) - dynamic, LOB locator - printf(" Column 3: content (CLOB) - THE CRITICAL TEST\n"); - status = OCIDefineByPos(stmthp, &defnp_lob, g_errhp, 3, NULL, 0, - SQLT_CLOB, NULL, NULL, NULL, OCI_DYNAMIC_FETCH); - check_error(status, "OCIDefineByPos CLOB"); - - status = OCIDefineDynamic(defnp_lob, g_errhp, g_envhp, lob_callback); - check_error(status, "OCIDefineDynamic CLOB"); - - /* Execute */ - printf("\nExecuting statement...\n"); - status = OCIStmtExecute(svchp, stmthp, g_errhp, 0, 0, NULL, NULL, OCI_DEFAULT); - check_error(status, "OCIStmtExecute"); - - /* CRITICAL TEST: Fetch 100 rows at once */ - printf("\n=== CRITICAL TEST ===\n"); - printf("Calling OCIStmtFetch(nrows=100)...\n"); - - clock_t start = clock(); - callback_count = 0; - - status = OCIStmtFetch(stmthp, g_errhp, 100, OCI_FETCH_NEXT, OCI_DEFAULT); - - clock_t end = clock(); - double elapsed = ((double)(end - start)) / CLOCKS_PER_SEC; - - if (status != OCI_SUCCESS && status != OCI_NO_DATA) { - check_error(status, "OCIStmtFetch"); - } - - /* Get actual rows fetched */ - ub4 rows_fetched = 0; - OCIAttrGet(stmthp, OCI_HTYPE_STMT, &rows_fetched, NULL, OCI_ATTR_ROWS_FETCHED, g_errhp); - - printf("\n=== RESULTS ===\n"); - printf("OCIStmtFetch returned: %d (%s)\n", status, - status == OCI_SUCCESS ? "SUCCESS" : - status == OCI_NO_DATA ? "NO_DATA" : "OTHER"); - printf("Rows fetched: %u\n", rows_fetched); - printf("Total callback invocations: %d\n", callback_count); - printf("Expected callbacks: %u (3 columns × %u rows)\n", rows_fetched * 3, rows_fetched); - printf("Elapsed time: %.6f seconds\n", elapsed); - - /* Analysis */ - printf("\n=== ANALYSIS ===\n"); - int expected_callbacks = rows_fetched * 3; /* 3 columns */ - - if (callback_count == expected_callbacks) { - printf("✓ Callbacks invoked %d times (correct: %u rows × 3 columns)\n", - callback_count, rows_fetched); - if (elapsed < 0.05) { - printf("✓ Very fast execution (%.6fs) = ONE ROUND TRIP!\n", elapsed); - printf("\n** CONCLUSION: OCIDefineDynamic DOES work with LOBs! **\n"); - printf("** Piecewise fetch is VIABLE - proceed with implementation! **\n"); - } else if (elapsed < 0.2) { - printf("? Moderate execution time (%.6fs)\n", elapsed); - printf(" Suggests possible batching, run with tcpdump to verify\n"); - } else { - printf("✗ Slow execution (%.6fs) = MULTIPLE ROUND TRIPS\n", elapsed); - printf("\n** CONCLUSION: OCIDefineDynamic does NOT reduce round trips **\n"); - printf("** Piecewise fetch is NOT viable - use alternative solutions **\n"); - } - } else { - printf("✗ Unexpected callback count: %d (expected %d)\n", - callback_count, expected_callbacks); - printf(" Investigation needed\n"); - } - - /* Cleanup */ - printf("\nCleaning up...\n"); - OCISessionEnd(svchp, g_errhp, usrhp, OCI_DEFAULT); - OCIServerDetach(srvhp, g_errhp, OCI_DEFAULT); - OCIHandleFree(stmthp, OCI_HTYPE_STMT); - OCIHandleFree(usrhp, OCI_HTYPE_SESSION); - OCIHandleFree(svchp, OCI_HTYPE_SVCCTX); - OCIHandleFree(srvhp, OCI_HTYPE_SERVER); - OCIHandleFree(g_errhp, OCI_HTYPE_ERROR); - OCIHandleFree(g_envhp, OCI_HTYPE_ENV); - - printf("Done.\n"); - return 0; -} diff --git a/test/test_encoding.rb b/test/test_encoding.rb index 888d96ab..4195f155 100644 --- a/test/test_encoding.rb +++ b/test/test_encoding.rb @@ -4,6 +4,13 @@ class TestEncoding < Minitest::Test def setup @conn = get_oci8_connection + # This test needs LOB locators for read/write operations + OCI8.lob_fetch_mode = :locator + end + + def teardown + # Restore default mode + OCI8.lob_fetch_mode = :long_as_string if @conn end def test_select diff --git a/test/test_large_lob.rb b/test/test_large_lob.rb new file mode 100644 index 00000000..6dfd2ea4 --- /dev/null +++ b/test/test_large_lob.rb @@ -0,0 +1,99 @@ +#!/usr/bin/env ruby +# +# Test to verify that 512KB LOBs can be fetched using both :long_as_string and :locator modes +# + +require 'oci8' +require_relative 'config' + +class TestLargeLob < Minitest::Test + def setup + @conn = get_oci8_connection + @saved_lob_fetch_mode = OCI8.lob_fetch_mode + drop_table('test_large_lob') + @conn.exec(<<-SQL) + CREATE TABLE test_large_lob ( + id NUMBER, + clob_data CLOB, + blob_data BLOB + ) + SQL + end + + def teardown + return unless @conn + + OCI8.lob_fetch_mode = @saved_lob_fetch_mode + drop_table('test_large_lob') + @conn.logoff + end + + def test_512kb_lob_roundtrip_with_long_interface + size_kb = 512 + clob_data = generate_text_data(size_kb) + blob_data = generate_binary_data(size_kb) + insert_lob_row(1, clob_data, blob_data) + + # Fetch using :long_as_string mode (LONG interface) + OCI8.lob_fetch_mode = :long_as_string + cursor = @conn.exec("SELECT id, clob_data, blob_data FROM test_large_lob WHERE id = 1") + row = cursor.fetch + cursor.close + + # Verify we got String objects with correct data + assert_equal 1, row[0] + assert_instance_of String, row[1], "CLOB should be fetched as String" + assert_instance_of String, row[2], "BLOB should be fetched as String" + assert_equal size_kb * 1024, row[1].size, "CLOB size should match" + assert_equal size_kb * 1024, row[2].size, "BLOB size should match" + assert_equal clob_data, row[1], "CLOB data should match (verifies chunk order)" + assert_equal blob_data, row[2], "BLOB data should match (verifies chunk order)" + end + + def test_512kb_lob_roundtrip_with_locator_mode + size_kb = 512 + clob_data = generate_text_data(size_kb) + blob_data = generate_binary_data(size_kb) + insert_lob_row(2, clob_data, blob_data) + + # Fetch using :locator mode (LOB locators) + OCI8.lob_fetch_mode = :locator + cursor = @conn.exec("SELECT id, clob_data, blob_data FROM test_large_lob WHERE id = 2") + row = cursor.fetch + cursor.close + + # Verify we got LOB locator objects + assert_equal 2, row[0] + assert_instance_of OCI8::CLOB, row[1], "CLOB should be fetched as locator" + assert_instance_of OCI8::BLOB, row[2], "BLOB should be fetched as locator" + + # Read from locators and verify data + fetched_clob_data = row[1].read + fetched_blob_data = row[2].read + assert_equal size_kb * 1024, fetched_clob_data.size, "CLOB size should match" + assert_equal size_kb * 1024, fetched_blob_data.size, "BLOB size should match" + assert_equal clob_data, fetched_clob_data, "CLOB data should match (verifies chunk order)" + assert_equal blob_data, fetched_blob_data, "BLOB data should match (verifies chunk order)" + end + + private + + def generate_text_data(size_kb, seed = 42) + # hex encoding: 1 byte -> 2 hex characters + binary_size_kb = (size_kb / 2.0).ceil + binary_data = generate_binary_data(binary_size_kb, seed) + hex_data = binary_data.unpack1('H*') + hex_data[0, size_bytes] # truncate to exact requested size + end + + def generate_binary_data(size_kb, seed = 42) + Random.new(seed).bytes(size_kb * 1024) + end + + def insert_lob_row(id, clob_data, blob_data) + cursor = @conn.parse("INSERT INTO test_large_lob VALUES (:1, :2, :3)") + cursor.exec(id, OCI8::CLOB.new(@conn, clob_data), OCI8::BLOB.new(@conn, blob_data)) + cursor.close + @conn.commit + end +end diff --git a/test/test_object.rb b/test/test_object.rb index b29ba7b8..6e648cae 100644 --- a/test/test_object.rb +++ b/test/test_object.rb @@ -59,10 +59,14 @@ class TestObj1 < Minitest::Test def setup @conn = get_oci8_connection + # This test needs LOB locators for read operations on object attributes + OCI8.lob_fetch_mode = :locator RbTestObj.default_connection = @conn end def teardown + # Restore default mode + OCI8.lob_fetch_mode = :long_as_string @conn.logoff end diff --git a/test/test_oci8.rb b/test/test_oci8.rb index 18226c70..a2b04a7c 100644 --- a/test/test_oci8.rb +++ b/test/test_oci8.rb @@ -41,8 +41,6 @@ def test_rename ] def test_long_type - clob_bind_type = OCI8::BindType::Mapping[:clob] - blob_bind_type = OCI8::BindType::Mapping[:blob] initial_cunk_size = OCI8::BindType::Base.initial_chunk_size begin OCI8::BindType::Base.initial_chunk_size = 5 @@ -50,6 +48,8 @@ def test_long_type drop_table('test_table') ascii_enc = Encoding.find('US-ASCII') 0.upto(1) do |i| + # First part of test uses LOB locators + OCI8.lob_fetch_mode = :locator if i == 0 @conn.exec("CREATE TABLE test_table (id number(38), long_column long, clob_column clob)") cursor = @conn.parse('insert into test_table values (:1, :2, :3)') @@ -108,39 +108,35 @@ def test_long_type assert_nil(cursor.fetch) cursor.close - begin - OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long - OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw - cursor = @conn.parse('SELECT * from test_table order by id') - cursor.exec - LONG_TEST_DATA.each_with_index do |data, index| - row = cursor.fetch - assert_equal(index, row[0]) - if data.nil? - assert_nil(row[1]) - assert_nil(row[2]) - elsif data.empty? - # '' is inserted to the long or long raw column as null. - assert_nil(row[1]) - # '' is inserted to the clob or blob column as an empty clob. - # However it is fetched as nil. - assert_nil(row[2]) - else - assert_equal(data, row[1]) - assert_equal(data, row[2]) - assert_equal(enc, row[1].encoding) - assert_equal(enc, row[2].encoding) - end + # Second part of test uses Long bind type (fetch as strings) + OCI8.lob_fetch_mode = :long_as_string + cursor = @conn.parse('SELECT * from test_table order by id') + cursor.exec + LONG_TEST_DATA.each_with_index do |data, index| + row = cursor.fetch + assert_equal(index, row[0]) + if data.nil? + assert_nil(row[1]) + assert_nil(row[2]) + elsif data.empty? + # '' is inserted to the long or long raw column as null. + assert_nil(row[1]) + # '' is inserted to the clob or blob column as an empty clob. + # However it is fetched as nil. + assert_nil(row[2]) + else + assert_equal(data, row[1]) + assert_equal(data, row[2]) + assert_equal(enc, row[1].encoding) + assert_equal(enc, row[2].encoding) end - assert_nil(cursor.fetch) - cursor.close - ensure - OCI8::BindType::Mapping[:clob] = clob_bind_type - OCI8::BindType::Mapping[:blob] = blob_bind_type end + assert_nil(cursor.fetch) + cursor.close drop_table('test_table') end ensure + OCI8.lob_fetch_mode = :long_as_string OCI8::BindType::Base.initial_chunk_size = initial_cunk_size end drop_table('test_table') @@ -396,6 +392,8 @@ def test_binary_float def test_clob_nclob_and_blob return if OCI8::oracle_client_version < OCI8::ORAVER_8_1 + # This test needs LOB locators + OCI8.lob_fetch_mode = :locator drop_table('test_table') sql = <<-EOS CREATE TABLE test_table (id number(5), C CLOB, NC NCLOB, B BLOB) @@ -435,6 +433,8 @@ def test_clob_nclob_and_blob assert_nil(cursor.fetch) cursor.close drop_table('test_table') + ensure + OCI8.lob_fetch_mode = :long_as_string end def test_select_number From c0d264d96630a91a907429e8ba58536639229b02 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Thu, 4 Dec 2025 20:18:24 +0200 Subject: [PATCH 5/5] restore test_array_fetch from reverted commits --- test/test_oci8.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/test_oci8.rb b/test/test_oci8.rb index a2b04a7c..55f52a6f 100644 --- a/test/test_oci8.rb +++ b/test/test_oci8.rb @@ -599,4 +599,25 @@ def test_server_version end assert_equal(ver, @conn.oracle_server_version.to_s) end + + def test_array_fetch + drop_table('test_table') + @conn.exec("CREATE TABLE test_table (id number, val clob)") + cursor = @conn.parse("INSERT INTO test_table VALUES (:1, :2)") + 1.upto(10) do |i| + cursor.exec(i, ('a'.ord + i).chr * i) + end + cursor.close + cursor = @conn.parse("select * from test_table where id <= :1 order by id") + cursor.prefetch_rows = 4 + [1, 6, 2, 7, 3, 8, 4, 9, 5, 10].each do |i| + cursor.exec(i) + 1.upto(i) do |j| + row = cursor.fetch + assert_equal(j, row[0]) + assert_equal(('a'.ord + j).chr * j, row[1].read) + end + assert_nil(cursor.fetch) + end + end end # TestOCI8