Skip to content

the most efficient LOB fetching #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
akostadinov wants to merge 5 commits into
base: master
Choose a base branch
from
Open

the most efficient LOB fetching #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c0e1f4a
Revert "Change the internal structure about object type binding to us…
akostadinov Dec 1, 2025
c3cc5fc
Revert "Read lob data without checking readable size to reduce number…
akostadinov Dec 1, 2025
5b4d2c8
Revert "Use array fetching when LOB colums are in a query and no obje…
akostadinov Dec 1, 2025
696a2ac
efficient piecewise LOB fetching
akostadinov Dec 4, 2025
c0d264d
restore test_array_fetch from reverted commits
akostadinov Dec 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions ext/oci8/bind.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -633,20 +633,11 @@ static VALUE oci8_bind_get(VALUE self)
return data_type->get(obind, (void*)((size_t)obind->valuep + obind->alloc_sz * idx), null_structp);
}

static VALUE oci8_bind_get_data(int argc, VALUE *argv, VALUE self)
static VALUE oci8_bind_get_data(VALUE self)
{
oci8_bind_t *obind = TO_BIND(self);
VALUE index;

rb_scan_args(argc, argv, "01", &index);
if (!NIL_P(index)) {
ub4 idx = NUM2UINT(index);
if (idx >= obind->maxar_sz) {
rb_raise(rb_eRuntimeError, "data index is too big. (%u for %u)", idx, obind->maxar_sz);
}
obind->curar_idx = idx;
return rb_funcall(self, oci8_id_get, 0);
} else if (obind->maxar_sz == 0) {
if (obind->maxar_sz == 0) {
obind->curar_idx = 0;
return rb_funcall(self, oci8_id_get, 0);
} else {
Expand Down Expand Up @@ -817,7 +808,7 @@ void Init_oci8_bind(VALUE klass)
rb_define_method(cOCI8BindTypeBase, "initialize", oci8_bind_initialize, 4);
rb_define_method(cOCI8BindTypeBase, "get", oci8_bind_get, 0);
rb_define_method(cOCI8BindTypeBase, "set", oci8_bind_set, 1);
rb_define_private_method(cOCI8BindTypeBase, "get_data", oci8_bind_get_data, -1);
rb_define_private_method(cOCI8BindTypeBase, "get_data", oci8_bind_get_data, 0);
rb_define_private_method(cOCI8BindTypeBase, "set_data", oci8_bind_set_data, 1);

rb_define_singleton_method(klass, "initial_chunk_size", get_initial_chunk_size, 0);
Expand Down
51 changes: 29 additions & 22 deletions ext/oci8/lob.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -639,9 +639,10 @@ static VALUE oci8_lob_read(int argc, VALUE *argv, VALUE self)
{
oci8_lob_t *lob = TO_LOB(self);
oci8_svcctx_t *svcctx = check_svcctx(lob);
ub8 lob_length;
ub8 read_len;
ub8 pos = lob->pos;
long strbufsiz = 512;
ub8 sz;
long strbufsiz;
ub8 byte_amt;
ub8 char_amt;
sword rv;
Expand All @@ -651,21 +652,36 @@ static VALUE oci8_lob_read(int argc, VALUE *argv, VALUE self)
ub1 piece = OCI_FIRST_PIECE;

rb_scan_args(argc, argv, "01", &size);
lob_length = oci8_lob_get_length(lob);
if (lob_length == 0 && NIL_P(size)) {
return rb_usascii_str_new("", 0);
}
if (lob_length <= pos) /* EOF */
return Qnil;
if (NIL_P(size)) {
sz = UB4MAXVAL;
read_len = lob_length - pos;
} else {
sz = NUM2ULL(size);
}
if (lob->state == S_BFILE_CLOSE) {
open_bfile(svcctx, lob, errhp);
ub8 sz = NUM2ULL(size);
read_len = MIN(sz, lob_length - pos);
}
read_more_data:
if (lob->lobtype == OCI_TEMP_CLOB) {
byte_amt = 0;
char_amt = sz;
char_amt = read_len;
if (oci8_nls_ratio == 1) {
strbufsiz = MIN(read_len, ULONG_MAX);
} else {
strbufsiz = MIN(read_len + read_len / 8, ULONG_MAX);
}
if (strbufsiz <= 10) {
strbufsiz = 10;
}
} else {
byte_amt = sz;
byte_amt = read_len;
char_amt = 0;
strbufsiz = MIN(read_len, ULONG_MAX);
}
if (lob->state == S_BFILE_CLOSE) {
open_bfile(svcctx, lob, errhp);
}
do {
VALUE strbuf = rb_str_buf_new(strbufsiz);
Expand Down Expand Up @@ -695,24 +711,15 @@ static VALUE oci8_lob_read(int argc, VALUE *argv, VALUE self)
}
rb_str_set_len(strbuf, byte_amt);
rb_ary_push(v, strbuf);
if (strbufsiz < 128 * 1024 * 1024) {
strbufsiz *= 2;
}
} while (rv == OCI_NEED_DATA);

if (NIL_P(size) && pos - lob->pos == sz) {
lob->pos = pos;
piece = OCI_FIRST_PIECE;
goto read_more_data;
if (pos >= lob_length) {
bfile_close(lob);
}
lob->pos = pos;
switch (RARRAY_LEN(v)) {
case 0:
if (NIL_P(size) && pos == 0) {
return rb_usascii_str_new("", 0);
} else {
return Qnil;
}
return Qnil;
case 1:
v = RARRAY_AREF(v, 0);
break;
Expand Down
61 changes: 24 additions & 37 deletions ext/oci8/object.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -680,53 +680,43 @@ static VALUE oci8_named_collection_alloc(VALUE klass)
return oci8_allocate_typeddata(klass, &oci8_named_collection_data_type);
}

typedef struct {
oci8_bind_t bind;
VALUE *obj;
} bind_named_type_t;

static void bind_named_type_mark(oci8_base_t *base)
{
bind_named_type_t *bnt = (bind_named_type_t *)base;
oci8_bind_t *obind = (oci8_bind_t *)base;
oci8_hp_obj_t *oho = (oci8_hp_obj_t *)obind->valuep;

if (bnt->obj != NULL) {
if (oho != NULL) {
ub4 idx = 0;

do {
rb_gc_mark(bnt->obj[idx]);
} while (++idx < bnt->bind.maxar_sz);
rb_gc_mark(oho[idx].obj);
} while (++idx < obind->maxar_sz);
}
rb_gc_mark(bnt->bind.tdo);
rb_gc_mark(obind->tdo);
}

static void bind_named_type_free(oci8_base_t *base)
{
bind_named_type_t *bnt = (bind_named_type_t *)base;
void **hp = (void **)bnt->bind.valuep;
oci8_bind_t *obind = (oci8_bind_t *)base;
oci8_hp_obj_t *oho = (oci8_hp_obj_t *)obind->valuep;

if (hp != NULL) {
if (oho != NULL) {
ub4 idx = 0;

do {
if (hp[idx] != NULL) {
OCIObjectFree(oci8_envhp, oci8_errhp, hp[idx], OCI_DEFAULT);
hp[idx] = NULL;
if (oho[idx].hp != NULL) {
OCIObjectFree(oci8_envhp, oci8_errhp, oho[idx].hp, OCI_DEFAULT);
oho[idx].hp = NULL;
}
} while (++idx < bnt->bind.maxar_sz);
}
if (bnt->obj != NULL) {
xfree(bnt->obj);
bnt->obj = NULL;
} while (++idx < obind->maxar_sz);
}
oci8_bind_free(base);
}

static VALUE bind_named_type_get(oci8_bind_t *obind, void *data, void *null_struct)
{
bind_named_type_t *bnt = (bind_named_type_t *)obind;
ub4 idx = obind->curar_idx;

return bnt->obj[idx];
oci8_hp_obj_t *oho = (oci8_hp_obj_t *)data;
return oho->obj;
}

NORETURN(static void bind_named_type_set(oci8_bind_t *obind, void *data, void **null_structp, VALUE val));
Expand All @@ -738,21 +728,18 @@ static void bind_named_type_set(oci8_bind_t *obind, void *data, void **null_stru

static void bind_named_type_init(oci8_bind_t *obind, VALUE svc, VALUE val, VALUE length)
{
bind_named_type_t *bnt = (bind_named_type_t *)obind;
VALUE tdo_obj = length;

obind->value_sz = sizeof(void*);
obind->alloc_sz = sizeof(void*);
bnt->obj = xcalloc(sizeof(VALUE), obind->maxar_sz ? obind->maxar_sz : 1);
obind->alloc_sz = sizeof(oci8_hp_obj_t);

CHECK_TDO(tdo_obj);
RB_OBJ_WRITE(obind->base.self, &obind->tdo, tdo_obj);
}

static void bind_named_type_init_elem(oci8_bind_t *obind, VALUE svc)
{
bind_named_type_t *bnt = (bind_named_type_t *)obind;
void **hp = (void **)obind->valuep;
oci8_hp_obj_t *oho = (oci8_hp_obj_t *)obind->valuep;
oci8_base_t *tdo = DATA_PTR(obind->tdo);
OCITypeCode tc = OCITypeTypeCode(oci8_envhp, oci8_errhp, tdo->hp.tdo);
VALUE klass = Qnil;
Expand All @@ -770,14 +757,14 @@ static void bind_named_type_init_elem(oci8_bind_t *obind, VALUE svc)
}
svcctx = oci8_get_svcctx(svc);
do {
bnt->obj[idx] = rb_class_new_instance(0, NULL, klass);
RB_OBJ_WRITTEN(obind->base.self, Qundef, bnt->obj[idx]);
obj = DATA_PTR(bnt->obj[idx]);
RB_OBJ_WRITE(bnt->obj[idx], &obj->tdo, obind->tdo);
obj->instancep = (char**)&hp[idx];
oho[idx].obj = rb_class_new_instance(0, NULL, klass);
RB_OBJ_WRITTEN(obind->base.self, Qundef, oho[idx].obj);
obj = DATA_PTR(oho[idx].obj);
RB_OBJ_WRITE(oho[idx].obj, &obj->tdo, obind->tdo);
obj->instancep = (char**)&oho[idx].hp;
obj->null_structp = (char**)&obind->u.null_structs[idx];
oci8_link_to_parent(&obj->base, &obind->base);
RB_OBJ_WRITTEN(bnt->obj[idx], Qundef, obind->base.self);
RB_OBJ_WRITTEN(oho[idx].obj, Qundef, obind->base.self);

chker2(OCIObjectNew(oci8_envhp, oci8_errhp, svcctx->base.hp.svc, tc, tdo->hp.tdo, NULL, OCI_DURATION_SESSION, TRUE, (dvoid**)obj->instancep),
&svcctx->base);
Expand Down Expand Up @@ -819,7 +806,7 @@ static const oci8_bind_data_type_t bind_named_type_data_type = {
#endif
},
bind_named_type_free,
sizeof(bind_named_type_t)
sizeof(oci8_bind_t)
},
bind_named_type_get,
bind_named_type_set,
Expand Down
29 changes: 8 additions & 21 deletions ext/oci8/stmt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@ static VALUE cOCIStmt;
typedef struct {
oci8_base_t base;
VALUE svc;
char use_stmt_release;
char end_of_fetch;
int use_stmt_release;
} oci8_stmt_t;

static void oci8_stmt_mark(oci8_base_t *base)
Expand Down Expand Up @@ -261,15 +260,14 @@ static VALUE oci8_stmt_execute(VALUE self, VALUE iteration_count)
oci8_stmt_t *stmt = TO_STMT(self);
oci8_svcctx_t *svcctx = oci8_get_svcctx(stmt->svc);

stmt->end_of_fetch = 0;
chker3(oci8_call_stmt_execute(svcctx, stmt, NUM2UINT(iteration_count),
svcctx->is_autocommit ? OCI_COMMIT_ON_SUCCESS : OCI_DEFAULT),
&stmt->base, stmt->base.hp.stmt);
return self;
}

/*
* @overload __fetch(connection, max_rows)
* @overload __fetch(connection)
*
* Fetches one row and set the result to <code>@define_handles</code>.
* This is called by private methods of OCI8::Cursor.
Expand All @@ -279,18 +277,13 @@ static VALUE oci8_stmt_execute(VALUE self, VALUE iteration_count)
*
* @private
*/
static VALUE oci8_stmt_fetch(VALUE self, VALUE svc, VALUE max_rows)
static VALUE oci8_stmt_fetch(VALUE self, VALUE svc)
{
oci8_stmt_t *stmt = TO_STMT(self);
oci8_svcctx_t *svcctx = oci8_get_svcctx(svc);
sword rv;
oci8_bind_t *obind;
const oci8_bind_data_type_t *data_type;
ub4 nrows = NUM2UINT(max_rows);

if (stmt->end_of_fetch) {
return Qnil;
}

if (stmt->base.children != NULL) {
obind = (oci8_bind_t *)stmt->base.children;
Expand All @@ -300,22 +293,16 @@ static VALUE oci8_stmt_fetch(VALUE self, VALUE svc, VALUE max_rows)
if (data_type->pre_fetch_hook != NULL) {
data_type->pre_fetch_hook(obind, stmt->svc);
}
if (nrows > 1 && nrows != obind->maxar_sz) {
rb_raise(rb_eRuntimeError, "fetch size (%u) != define-handle size %u", nrows, obind->maxar_sz);
}
}
obind = (oci8_bind_t *)obind->base.next;
} while (obind != (oci8_bind_t*)stmt->base.children);
}
rv = OCIStmtFetch_nb(svcctx, stmt->base.hp.stmt, oci8_errhp, nrows, OCI_FETCH_NEXT, OCI_DEFAULT);
rv = OCIStmtFetch_nb(svcctx, stmt->base.hp.stmt, oci8_errhp, 1, OCI_FETCH_NEXT, OCI_DEFAULT);
if (rv == OCI_NO_DATA) {
stmt->end_of_fetch = 1;
} else {
chker3(rv, &svcctx->base, stmt->base.hp.stmt);
return Qfalse;
}
chker2(OCIAttrGet(stmt->base.hp.stmt, OCI_HTYPE_STMT, &nrows, 0, OCI_ATTR_ROWS_FETCHED, oci8_errhp),
&svcctx->base);
return nrows ? UINT2NUM(nrows) : Qnil;
chker3(rv, &svcctx->base, stmt->base.hp.stmt);
return Qtrue;
}

/*
Expand Down Expand Up @@ -440,7 +427,7 @@ void Init_oci8_stmt(VALUE cOCI8)
rb_define_private_method(cOCIStmt, "__define", oci8_define_by_pos, 2);
rb_define_private_method(cOCIStmt, "__bind", oci8_bind, 2);
rb_define_private_method(cOCIStmt, "__execute", oci8_stmt_execute, 1);
rb_define_private_method(cOCIStmt, "__fetch", oci8_stmt_fetch, 2);
rb_define_private_method(cOCIStmt, "__fetch", oci8_stmt_fetch, 1);
rb_define_private_method(cOCIStmt, "__paramGet", oci8_stmt_get_param, 1);
rb_define_method(cOCIStmt, "rowid", oci8_stmt_get_rowid, 0);

Expand Down
11 changes: 8 additions & 3 deletions lib/oci8/bindtype.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,13 +245,18 @@ def self.create(con, val, param, max_array_size)
# datatype type size prec scale
# -------------------------------------------------
# CLOB SQLT_CLOB 4000 0 0
OCI8::BindType::Mapping[:clob] = OCI8::BindType::CLOB
OCI8::BindType::Mapping[:nclob] = OCI8::BindType::NCLOB
# NCLOB SQLT_CLOB 4000 0 0
# Default: Fetch as String using SQLT_CHR (fast, max 2GB)
# See: OCI8::lob_fetch_mode
OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long
OCI8::BindType::Mapping[:nclob] = OCI8::BindType::Long

# datatype type size prec scale
# -------------------------------------------------
# BLOB SQLT_BLOB 4000 0 0
OCI8::BindType::Mapping[:blob] = OCI8::BindType::BLOB
# Default: Fetch as binary String using SQLT_CHR (fast, max 2GB)
# See: OCI8::lob_fetch_mode
OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw

# datatype type size prec scale
# -------------------------------------------------
Expand Down
Loading