Skip to content

Commit 32ad568

Browse files
ShenaMaoShenaMao
committed
release 3.0.2.3
1 parent 38c4675 commit 32ad568

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+1082
-29958
lines changed

‎LICENSE‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright 2016-2024 DolphinDB, Inc.
1+
Copyright 2016-2025 DolphinDB, Inc.
22

33

44
Apache License
@@ -189,7 +189,7 @@ Copyright 2016-2024 DolphinDB, Inc.
189189
same "printed page" as the copyright notice for easier
190190
identification within third-party archives.
191191

192-
Copyright 2016-2024 DolphinDB, Inc.
192+
Copyright 2016-2025 DolphinDB, Inc.
193193

194194
Licensed under the Apache License, Version 2.0 (the "License");
195195
you may not use this file except in compliance with the License.

‎core/include/ConstantFactory.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,16 @@ class ConstantFactory{
8383
}
8484

8585
DATA_TYPE getDataType(const string& type) const {
86+
auto size = type.size();
87+
if (type[size - 1] == ']' && type[size - 2] == '[') {
88+
string baseType = type.substr(0, size - 2);
89+
std::unordered_map<string, DATA_TYPE>::const_iterator it = typeMap_.find(baseType);
90+
if (it == typeMap_.end())
91+
return DT_VOID;
92+
else
93+
return (DATA_TYPE)(it->second + ARRAY_TYPE_BASE);
94+
}
95+
8696
std::unordered_map<string,DATA_TYPE>::const_iterator it=typeMap_.find(type);
8797
if(it==typeMap_.end())
8898
return DT_VOID;
@@ -943,6 +953,8 @@ class ConstantFactory{
943953
typeMap_.insert(std::pair<string, DATA_TYPE>("decimal128", DT_DECIMAL128));
944954
typeMap_.insert(std::pair<string, DATA_TYPE>("object", DT_OBJECT));
945955
typeMap_.insert(std::pair<string, DATA_TYPE>("unknown", DATA_TYPE::DT_UNK));
956+
typeMap_.insert(std::pair<string,DATA_TYPE>("blob",DT_BLOB));
957+
typeMap_.insert(std::pair<string,DATA_TYPE>("decimal",DT_DECIMAL));
946958

947959
formMap_.insert(std::pair<string,DATA_FORM>("scalar",DF_SCALAR));
948960
formMap_.insert(std::pair<string,DATA_FORM>("pair",DF_PAIR));

‎core/include/DolphinDB.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,11 +140,11 @@ class EXPORT_DECL DBConnection {
140140
py::object runPy(
141141
const string& script, int priority=4, int parallelism=64,
142142
int fetchSize=0, bool clearMemory=false,
143-
bool pickleTableToList=false, bool disableDecimal=false);
143+
bool pickleTableToList=false, bool disableDecimal=false, bool withTableSchema = false);
144144
py::object runPy(
145145
const string& funcName, vector<ConstantSP>& args, int priority=4, int parallelism=64,
146146
int fetchSize=0, bool clearMemory=false,
147-
bool pickleTableToList=false, bool disableDecimal=false);
147+
bool pickleTableToList=false, bool disableDecimal=false, bool withTableSchema = false);
148148
void setKeepAliveTime(int keepAliveTime);
149149
void setTimeout(int readTimeout, int writeTimeout);
150150
const string getSessionId() const;

‎core/include/DolphinDBImp.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@ class DBConnectionImpl {
5050
py::object runPy(
5151
const string& script, int priority = 4, int parallelism = 64,
5252
int fetchSize = 0, bool clearMemory = false,
53-
bool pickleTableToList = false, bool disableDecimal = false);
53+
bool pickleTableToList = false, bool disableDecimal = false, bool withTableSchema = false);
5454
py::object runPy(
5555
const string& funcName, vector<ConstantSP>& args, int priority = 4, int parallelism = 64,
5656
int fetchSize = 0, bool clearMemory = false,
57-
bool pickleTableToList = false, bool disableDecimal = false);
57+
bool pickleTableToList = false, bool disableDecimal = false, bool withTableSchema = false);
5858
void setkeepAliveTime(int keepAliveTime){
5959
if (keepAliveTime > 0)
6060
keepAliveTime_ = keepAliveTime;
@@ -76,7 +76,7 @@ class DBConnectionImpl {
7676
py::object runPy(
7777
const string& script, const string& scriptType, vector<ConstantSP>& args,
7878
int priority = 4, int parallelism = 64, int fetchSize = 0, bool clearMemory = false,
79-
bool pickleTableToList = false, bool disableDecimal = false);
79+
bool pickleTableToList = false, bool disableDecimal = false, bool withTableSchema = false);
8080
bool connect();
8181
void login();
8282

‎core/src/DolphinDB.cpp‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <fstream>
1010
#include <istream>
1111
#include <stack>
12+
#include <regex>
1213
#include "Concurrent.h"
1314
#ifndef WINDOWS
1415
#include <uuid/uuid.h>
@@ -411,15 +412,28 @@ ConstantSP DBConnection::run(const string& script, int priority, int parallelism
411412
return NULL;
412413
}
413414

415+
std::string extractRefId(const std::string& errorMsg) {
416+
std::regex refIdRegex(R"(RefId:\s*(\w+))");
417+
std::smatch match;
418+
if (std::regex_search(errorMsg, match, refIdRegex)) {
419+
return match[1];
420+
}
421+
return "";
422+
}
423+
414424
py::object DBConnection::runPy(
415425
const string &script, int priority, int parallelism,
416-
int fetchSize, bool clearMemory, bool pickleTableToList, bool disableDecimal
426+
int fetchSize, bool clearMemory, bool pickleTableToList, bool disableDecimal, bool withTableSchema
417427
) {
418428
if (nodes_.empty() == false) {
419429
while (closed_ == false) {
420430
try {
421-
return conn_->runPy(script, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal);
431+
return conn_->runPy(script, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal, withTableSchema);
422432
} catch (IOException& e) {
433+
if (extractRefId(e.what()) == "S04009") {
434+
throw;
435+
}
436+
423437
string host;
424438
int port = 0;
425439
if (connected()) {
@@ -442,7 +456,7 @@ py::object DBConnection::runPy(
442456
}
443457
return py::none();
444458
} else {
445-
return conn_->runPy(script, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal);
459+
return conn_->runPy(script, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal, withTableSchema);
446460
}
447461
}
448462

@@ -476,12 +490,12 @@ ConstantSP DBConnection::run(const string& funcName, vector<dolphindb::ConstantS
476490

477491
py::object DBConnection::runPy(
478492
const string &funcName, vector<ConstantSP> &args, int priority, int parallelism,
479-
int fetchSize, bool clearMemory, bool pickleTableToList, bool disableDecimal
493+
int fetchSize, bool clearMemory, bool pickleTableToList, bool disableDecimal, bool withTableSchema
480494
) {
481495
if (nodes_.empty() == false) {
482496
while (closed_ == false) {
483497
try {
484-
return conn_->runPy(funcName, args, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal);
498+
return conn_->runPy(funcName, args, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal, withTableSchema);
485499
} catch (IOException& e) {
486500
string host;
487501
int port = 0;
@@ -499,7 +513,7 @@ py::object DBConnection::runPy(
499513
}
500514
return py::none();
501515
} else {
502-
return conn_->runPy(funcName, args, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal);
516+
return conn_->runPy(funcName, args, priority, parallelism, fetchSize, clearMemory, pickleTableToList, disableDecimal, withTableSchema);
503517
}
504518
}
505519

‎core/src/DolphinDBImp.cpp‎

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "DdbPythonUtil.h"
66
#include "Pickle.h"
77
#include "Wrappers.h"
8+
#include "ConstantFactory.h"
89

910
#define APIMinVersionRequirement 300
1011

@@ -144,9 +145,11 @@ bool DBConnectionImpl::connect() {
144145
if(asynTask_) {
145146
SmartPointer<DBConnection> newConn = new DBConnection(false, false);
146147
newConn->connect(hostName_, port_, userId_, pwd_);
147-
requiredVersion =newConn->run("getRequiredAPIVersion()");
148+
vector<ConstantSP> args;
149+
requiredVersion =newConn->run("getRequiredAPIVersion", args);
148150
}else{
149-
requiredVersion = run("getRequiredAPIVersion()");
151+
vector<ConstantSP> args;
152+
requiredVersion = run("getRequiredAPIVersion", args);
150153
}
151154
}
152155
catch(...){
@@ -198,23 +201,23 @@ ConstantSP DBConnectionImpl::run(const string& funcName, vector<ConstantSP>& arg
198201
py::object DBConnectionImpl::runPy(
199202
const string &script, int priority,
200203
int parallelism, int fetchSize, bool clearMemory,
201-
bool pickleTableToList, bool disableDecimal
204+
bool pickleTableToList, bool disableDecimal, bool withTableSchema
202205
) {
203206
vector<ConstantSP> args;
204207
return runPy(
205208
script, "script", args, priority, parallelism,
206-
fetchSize, clearMemory, pickleTableToList, disableDecimal
209+
fetchSize, clearMemory, pickleTableToList, disableDecimal, withTableSchema
207210
);
208211
}
209212

210213
py::object DBConnectionImpl::runPy(
211214
const string &funcName, vector<ConstantSP> &args, int priority,
212215
int parallelism, int fetchSize, bool clearMemory,
213-
bool pickleTableToList, bool disableDecimal
216+
bool pickleTableToList, bool disableDecimal, bool withTableSchema
214217
) {
215218
return runPy(
216219
funcName, "function", args, priority, parallelism,
217-
fetchSize, clearMemory, pickleTableToList, disableDecimal
220+
fetchSize, clearMemory, pickleTableToList, disableDecimal, withTableSchema
218221
);
219222
}
220223

@@ -423,10 +426,32 @@ ConstantSP DBConnectionImpl::run(const string& script, const string& scriptType,
423426
return result;
424427
}
425428

429+
py::dict getTableSchema(ConstantSP result)
430+
{
431+
TableSP ddbTbl = result;
432+
py::dict column_types;
433+
size_t columnSize = ddbTbl->columns();
434+
435+
for (size_t i = 0; i < columnSize; ++i)
436+
{
437+
py::str column_name = ddbTbl->getColumnName(i);
438+
DATA_TYPE type = ddbTbl->getColumnType(i);
439+
py::str typeString = Util::getDataTypeString(type);
440+
441+
py::object scale = py::none();
442+
if ((type >= DT_DECIMAL32 && type <= DT_DECIMAL128) || (type >= DT_DECIMAL32_ARRAY && type <= DT_DECIMAL128_ARRAY)) {
443+
scale = py::int_(ddbTbl->getColumn(i)->getExtraParamForType());
444+
}
445+
446+
column_types[column_name] = py::make_tuple(typeString, scale);
447+
}
448+
return column_types;
449+
}
450+
426451
py::object DBConnectionImpl::runPy(
427452
const string &script, const string &scriptType, vector<ConstantSP> &args,
428453
int priority, int parallelism, int fetchSize, bool clearMemory,
429-
bool pickleTableToList, bool disableDecimal
454+
bool pickleTableToList, bool disableDecimal, bool withTableSchema
430455
) {
431456
//RecordTime record("Db.runPy"+script);
432457
DLOG("runPy",script,"start argsize",args.size());
@@ -543,6 +568,10 @@ py::object DBConnectionImpl::runPy(
543568
}
544569

545570
if (numObject == 0) {
571+
if (withTableSchema) {
572+
py::gil_scoped_acquire pgil;
573+
return py::make_tuple(py::none(), py::none());
574+
}
546575
return py::none();
547576
}
548577

@@ -620,6 +649,15 @@ py::object DBConnectionImpl::runPy(
620649
py::gil_scoped_acquire pgil;
621650

622651
converter::ToPythonOption option(pickleTableToList);
652+
653+
if (withTableSchema) {
654+
py::object dataframe = converter::Converter::toPython_Old(result, option);
655+
if (result->getForm() == DATA_FORM::DF_TABLE) {
656+
return py::make_tuple(dataframe, getTableSchema(result));
657+
}
658+
return py::make_tuple(dataframe, py::none());
659+
}
660+
623661
return converter::Converter::toPython_Old(result, option);
624662
}
625663

‎core/src/Streaming.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -746,7 +746,8 @@ void StreamingClientImpl::checkServerVersion(std::string host, int port, const s
746746
}
747747
}
748748

749-
auto versionStr = conn.run("version()")->getString();
749+
vector<ConstantSP> args;
750+
auto versionStr = conn.run("version", args)->getString();
750751
auto _ = Util::split(Util::split(versionStr, ' ')[0], '.');
751752
auto v0 = std::stoi(_[0]);
752753
auto v1 = std::stoi(_[1]);

‎core/src/TypeHelper.cpp‎

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,17 +61,63 @@ Type createType(const py::handle &data) {
6161
// change int to Type
6262
type = std::make_pair(static_cast<HELPER_TYPE>(py::cast<int>(data)), EXPARAM_DEFAULT);
6363
}
64+
else if (CHECK_INS(data, py_str_)) {
65+
// change str to Type
66+
std::string type_name = py::cast<std::string>(data);
67+
HELPER_TYPE datatype = static_cast<HELPER_TYPE>(Util::getDataType(type_name));
68+
type = std::make_pair(datatype, EXPARAM_DEFAULT);
69+
}
6470
else if (CHECK_INS(data, py_list_)) {
65-
// change [int, int] to Type
6671
py::list tmplist = py::reinterpret_borrow<py::list>(data);
67-
if (tmplist.size() <=0 || tmplist.size() > 2)
68-
throw ConversionException("Conversion failed. Specify a valid type.");
69-
HELPER_TYPE datatype = static_cast<HELPER_TYPE>(py::cast<int>(tmplist[0]));
70-
int exparam = py::cast<int>(tmplist[1]);
71-
type = std::make_pair(datatype, exparam);
72+
size_t list_size = tmplist.size();
73+
74+
if (list_size == 1) {
75+
// change [int] to Type
76+
if (CHECK_INS(tmplist[0], py_int_)) {
77+
HELPER_TYPE datatype = static_cast<HELPER_TYPE>(py::cast<int>(tmplist[0]));
78+
type = std::make_pair(datatype, EXPARAM_DEFAULT);
79+
}
80+
// change [str] to Type
81+
else if (CHECK_INS(tmplist[0], py_str_)) {
82+
std::string type_name = py::cast<std::string>(tmplist[0]);
83+
HELPER_TYPE datatype = static_cast<HELPER_TYPE>(Util::getDataType(type_name));
84+
type = std::make_pair(datatype, EXPARAM_DEFAULT);
85+
}
86+
else {
87+
throw ConversionException("Conversion failed. The first element of the list must be of type int or str.");
88+
}
89+
}
90+
else if (list_size == 2) {
91+
// change [int, int], [str, int], [int, None], [str, None] to Type
92+
HELPER_TYPE datatype;
93+
if (CHECK_INS(tmplist[0], py_int_)) {
94+
datatype = static_cast<HELPER_TYPE>(py::cast<int>(tmplist[0]));
95+
}
96+
else if (CHECK_INS(tmplist[0], py_str_)) {
97+
std::string type_name = py::cast<std::string>(tmplist[0]);
98+
datatype = static_cast<HELPER_TYPE>(Util::getDataType(type_name));
99+
}
100+
else {
101+
throw ConversionException("Conversion failed. The first element of the list must be of type int or str.");
102+
}
103+
104+
int exparam = EXPARAM_DEFAULT;
105+
if (!tmplist[1].is_none()) {
106+
if (CHECK_INS(tmplist[1], py_int_)) {
107+
exparam = py::cast<int>(tmplist[1]);
108+
}
109+
else {
110+
throw ConversionException("Conversion failed. The second element of the list must be of type int or None.");
111+
}
112+
}
113+
type = std::make_pair(datatype, exparam);
114+
}
115+
else {
116+
throw ConversionException("Conversion failed. List must contain exactly 1 or 2 elements.");
117+
}
72118
}
73119
else {
74-
throw ConversionException("Conversion failed. Specify a valid type.");
120+
throw ConversionException("Conversion failed. Specify a int, str, or list type.");
75121
}
76122
return type;
77123
}

‎core/src/binding.cpp‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,9 @@ class PartitionedTableAppender{
237237
checker[colNames[i]] = colTypes[i];
238238
}
239239
TRY
240-
insertRows = partitionedTableAppender_.append(Converter::toDolphinDB_Table_fromDataFrame(table, checker));
240+
auto data = Converter::toDolphinDB_Table_fromDataFrame(table, checker);
241+
py::gil_scoped_release gil_release;
242+
insertRows = partitionedTableAppender_.append(data);
241243
CATCH_EXCEPTION("<Exception> in append: ")
242244
return insertRows;
243245
}
@@ -397,7 +399,7 @@ class SessionImpl {
397399

398400
py::object run(const std::string &script, const py::args &args, const py::handle &clearMemory = py::none(),
399401
const py::handle &pickleTableToList = py::none(), const py::handle &priority = py::none(),
400-
const py::handle &parallelism = py::none(), const py::handle &disableDecimal = py::none()) {
402+
const py::handle &parallelism = py::none(), const py::handle &disableDecimal = py::none(), const py::handle &withTableSchema = py::none()) {
401403
if (enableJobCancellation_) {
402404
ddb::LockGuard<ddb::Mutex> LockGuard(&SessionImpl::mapMutex_);
403405
if (SessionImpl::runningMap_.count(this) == 0) {
@@ -470,11 +472,16 @@ class SessionImpl {
470472
disableDecimal_ = disableDecimal.cast<bool>();
471473
}
472474

475+
bool withTableSchema_ = false;
476+
if (!withTableSchema.is_none()) {
477+
withTableSchema_ = withTableSchema.cast<bool>();
478+
}
479+
473480
py::object result;
474481
if (args.empty()) {
475482
// script mode
476483
TRY result = dbConnection_.runPy(script, priority_, parallelism_, 0, clearMemory_, pickleTableToList_,
477-
disableDecimal_);
484+
disableDecimal_, withTableSchema_);
478485
CATCH_EXCEPTION("<Exception> in run: ")
479486
} else {
480487
// function mode
@@ -483,7 +490,7 @@ class SessionImpl {
483490
ddbArgs.push_back(Converter::toDolphinDB(it));
484491
}
485492
result = dbConnection_.runPy(script, ddbArgs, priority_, parallelism_, 0, clearMemory_, pickleTableToList_,
486-
disableDecimal_);
493+
disableDecimal_, withTableSchema_);
487494
CATCH_EXCEPTION("<Exception> in run: ")
488495
}
489496
return result;
@@ -1429,7 +1436,8 @@ PYBIND11_MODULE(_dolphindbcpp, m) {
14291436
py::arg("pickleTableToList") = py::none(),
14301437
py::arg("priority") = py::none(),
14311438
py::arg("parallelism") = py::none(),
1432-
py::arg("disableDecimal") = py::none()
1439+
py::arg("disableDecimal") = py::none(),
1440+
py::arg("withTableSchema") = py::none()
14331441
)
14341442
.def("runBlock",&SessionImpl::runBlock,
14351443
py::arg("script"),

0 commit comments

Comments
 (0)