Skip to content

Commit 96973b8

Browse files
ShenaMaoShenaMao
committed
release 3.0.2.4
1 parent 32ad568 commit 96973b8

File tree

12 files changed

+495
-414
lines changed

12 files changed

+495
-414
lines changed

‎core/src/TypeConverter.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3395,13 +3395,13 @@ Converter::toNumpy_Vector(const ConstantSP &data, const ToPythonOption &option,
33953395
VectorEnumNumeric<char>(data, &Vector::getBoolConst, [&](const char *pbuf, INDEX startIndex, INDEX size) -> bool {
33963396
for (INDEX i = 0, index = startIndex; i < size; ++i, ++index) {
33973397
if (UNLIKELY(pbuf[i] == CHAR_MIN)) {
3398-
ptr[i] = pyNone;
3398+
ptr[index] = pyNone;
33993399
}
34003400
else {
34013401
if (pbuf[i] != 0)
3402-
ptr[i] = pyTrue;
3402+
ptr[index] = pyTrue;
34033403
else
3404-
ptr[i] = pyFalse;
3404+
ptr[index] = pyFalse;
34053405
}
34063406
}
34073407
return true;

‎core/src/binding.cpp‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -581,7 +581,7 @@ class SessionImpl {
581581
}
582582

583583
void subscribe(const string &host, const int &port, py::object handler, const string &tableName, const string &actionName,
584-
const long long &offset, const bool &resub, py::array filter,const string &userName,const string &password,
584+
const long long &offset, const bool &resub, py::object filter,const string &userName,const string &password,
585585
StreamDeserializer &streamDeserializer,
586586
const std::vector<std::string> backupSites = std::vector<std::string>(),
587587
int resubTimeout = 100, bool subOnce = false) {
@@ -599,7 +599,17 @@ class SessionImpl {
599599
}
600600
handler(row);
601601
};
602-
ddb::VectorSP ddbFilter = filter.size() ? Converter::toDolphinDB(filter) : nullptr;
602+
603+
ddb::VectorSP ddbFilter;
604+
if (py::isinstance<py::array>(filter)) {
605+
py::array arr_filter = py::cast<py::array>(filter);
606+
ddbFilter = arr_filter.size() ? Converter::toDolphinDB(arr_filter) : nullptr;
607+
} else if (py::isinstance<py::str>(filter)) {
608+
std::string strFilter = py::str(filter);
609+
ddb::ConstantSP ddbStrFilter = new ddb::String(strFilter);
610+
ddbFilter = (ddb::VectorSP)ddbStrFilter;
611+
}
612+
603613
TRY
604614
vector<ddb::ThreadSP> threads;
605615
if(subscriber_.isNull() == false){
@@ -615,7 +625,7 @@ class SessionImpl {
615625
}
616626

617627
// FIXME: not thread safe
618-
void subscribeBatch(string &host,int port, py::object handler,string &tableName,string &actionName,long long offset, bool resub, py::array filter,
628+
void subscribeBatch(string &host,int port, py::object handler,string &tableName,string &actionName,long long offset, bool resub, py::object filter,
619629
const bool & msgAsTable, int batchSize, double throttle,const string &userName,const string &password,StreamDeserializer &streamDeserializer,
620630
const std::vector<std::string> backupSites = std::vector<std::string>(),
621631
int resubTimeout = 100, bool subOnce = false) {
@@ -653,7 +663,17 @@ class SessionImpl {
653663
handler(pyMsg);
654664
}
655665
};
656-
ddb::VectorSP ddbFilter = filter.size() ? Converter::toDolphinDB(filter) : nullptr;
666+
667+
ddb::VectorSP ddbFilter;
668+
if (py::isinstance<py::array>(filter)) {
669+
py::array arr_filter = py::cast<py::array>(filter);
670+
ddbFilter = arr_filter.size() ? Converter::toDolphinDB(arr_filter) : nullptr;
671+
} else if (py::isinstance<py::str>(filter)) {
672+
std::string strFilter = py::str(filter);
673+
ddb::ConstantSP ddbStrFilter = new ddb::String(strFilter);
674+
ddbFilter = (ddb::VectorSP)ddbStrFilter;
675+
}
676+
657677
TRY
658678
vector<ddb::ThreadSP> threads;
659679
ddb::ThreadSP thread = subscriber_->subscribe(host, port, ddbHandler, tableName, actionName, offset, resub, ddbFilter, false,

‎dolphindb/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from .utils import month
2121
from . import cep, io, logger
2222

23-
__version__ = "3.0.2.3"
23+
__version__ = "3.0.2.4"
2424

2525
name = "dolphindb"
2626

‎pyproject.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,9 @@ before-build = [
6565
before-all = [
6666
"(if exist build rmdir /s /q build)",
6767
]
68+
before-build = [
69+
"python --version",
70+
"python -m pip config set global.index-url --site http://mirrors.aliyun.com/pypi/simple/",
71+
"python -m pip config set global.trusted-host mirrors.aliyun.com",
72+
"python -m pip install --upgrade pip --trusted-host mirrors.aliyun.com"
73+
]

‎test/basic_testing/test_download_new.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
from decimal import Decimal
2+
from random import randint
3+
14
import dolphindb as ddb
25
import numpy as np
36
import pandas as pd
@@ -186,3 +189,75 @@ def test_download_dfs_table(self):
186189
"price": pd.Series([], dtype=np.float64),
187190
})
188191
assert equalPlus(x, expect)
192+
193+
@pytest.mark.parametrize('script,expect,dtype', [
194+
["true false 00b", [True, False, None], "object"],
195+
["0c 127c -127c 00c", [0., 127., -127., np.nan], np.float64],
196+
["0h 32767h -32767h 00h", [0, 32767, -32767, np.nan], np.float64],
197+
["0i 2147483647i -2147483647i 00i", [0, 2147483647, -2147483647, np.nan], np.float64],
198+
["0l 9223372036854775807l -9223372036854775807l 00l", [0, 9223372036854775807, -9223372036854775807, np.nan],
199+
np.float64],
200+
["1970.01.01d 1970.01.02d 00d",
201+
[np.datetime64('1970-01-01', 'D'), np.datetime64('1970-01-02', 'D'), np.datetime64('nat', 'D')],
202+
'datetime64[ns]'],
203+
["1970.01M 1970.02M 00M",
204+
[np.datetime64('1970-01', 'M'), np.datetime64('1970-02', 'M'), np.datetime64('nat', 'M')], 'datetime64[ns]'],
205+
["00:00:00.000t 00:00:00.001t 00t",
206+
[np.datetime64('1970-01-01T00:00:00.000', 'ms'), np.datetime64('1970-01-01T00:00:00.001', 'ms'),
207+
np.datetime64('nat', 'ms')], 'datetime64[ns]'],
208+
["00:00m 00:01m 00m",
209+
[np.datetime64('1970-01-01T00:00', 'm'), np.datetime64('1970-01-01T00:01', 'm'), np.datetime64('nat', 'm')],
210+
'datetime64[ns]'],
211+
["00:00:00s 00:00:01s 00s",
212+
[np.datetime64('1970-01-01T00:00:00', 's'), np.datetime64('1970-01-01T00:00:01', 's'),
213+
np.datetime64('nat', 's')], 'datetime64[ns]'],
214+
["1970.01.01T00:00:00D 1970.01.01T00:00:01D 00D",
215+
[np.datetime64('1970-01-01T00:00:00', 's'), np.datetime64('1970-01-01T00:00:01', 's'),
216+
np.datetime64('nat', 's')], 'datetime64[ns]'],
217+
["1970.01.01 00:00:00.000T 1970.01.01 00:00:00.001T 00T",
218+
[np.datetime64('1970-01-01T00:00:00.000', 'ms'), np.datetime64('1970-01-01T00:00:00.001', 'ms'),
219+
np.datetime64('nat', 'ms')], 'datetime64[ns]'],
220+
["00:00:00.000000000n 00:00:00.000000001n 00n",
221+
[np.datetime64('1970-01-01T00:00:00.000000000', 'ns'), np.datetime64('1970-01-01T00:00:00.000000001', 'ns'),
222+
np.datetime64('nat', 'ns')], "datetime64[ns]"],
223+
["1970.01.01 00:00:00.000000000N 1970.01.01 00:00:00.000000001N 00N",
224+
[np.datetime64('1970-01-01T00:00:00.000000000', 'ns'), np.datetime64('1970-01-01T00:00:00.000000001', 'ns'),
225+
np.datetime64('nat', 'ns')], "datetime64[ns]"],
226+
["0.0f 3.14f 00f", [0., 3.14, np.nan], np.float32],
227+
["0.0F 3.14F 00F", [0., 3.14, np.nan], np.float64],
228+
["'abc!@#中文 123' \"\"", ['abc!@#中文 123', ''], "object"],
229+
["[uuid('5d212a78-cc48-e3b1-4235-b4d91473ee87'),uuid('00000000-0000-0000-0000-000000000000')]",
230+
['5d212a78-cc48-e3b1-4235-b4d91473ee87', '00000000-0000-0000-0000-000000000000'], "object"],
231+
["[datehour('1970.01.01T00'),datehour(NULL)]", [np.datetime64('1970-01-01T00', 'h'), np.datetime64('nat', 'h')],
232+
"datetime64[ns]"],
233+
["[ipaddr('127.0.0.1'),ipaddr('0.0.0.0')]", ['127.0.0.1', '0.0.0.0'], "object"],
234+
["[int128('e1671797c52e15f763380b45e841ec32'),int128('00000000000000000000000000000000')]",
235+
['e1671797c52e15f763380b45e841ec32', '00000000000000000000000000000000'], "object"],
236+
["[blob('abc!@#中文 123'),blob('')]", ['abc!@#中文 123'.encode(), b''], "object"],
237+
["[decimal32(\"0\",3),decimal32('3.141',3)]", [Decimal('0.000'), Decimal('3.141')], "object"],
238+
["[decimal64(\"0\",3),decimal64('3.141',3)]", [Decimal('0.000'), Decimal('3.141')], "object"],
239+
["[decimal128(\"0\",3),decimal128('3.141',3)]", [Decimal('0.000'), Decimal('3.141')], "object"],
240+
["array(BOOL[]).append!([[true,false,00b],[true],[false],[00b]])",
241+
[np.array([True, False, None]), np.array([True]), np.array([False]), np.array([None])], "object"],
242+
], ids=["BOOL", "CHAR", "SHORT", "INT", "LONG", "DATE", "MONTH", "TIME", "MINUTE", "SECOND", "DATETIME",
243+
"TIMESTAMP", "NANOTIME", "NANOTIMESTAMP", "FLOAT", "DOUBLE", "STRING", "UUID", "DATEHOUR", "IPADDR",
244+
"INT128", "BLOB", "DECIMAL32", "DECIMAL64", "DECIMAL128", "BOOL_ARRAY"])
245+
def test_download_table_ge_8192(self, script, expect, dtype):
246+
n = 20000
247+
index = [randint(0, len(expect) - 1) for i in range(n)]
248+
conn = ddb.Session(HOST, PORT, USER, PASSWD)
249+
conn.upload({"index": index})
250+
result = conn.run(f"""
251+
x = {script}
252+
col = array(x,capacity={n})
253+
for (i in index){{
254+
if (type(x)<64){{
255+
col.append!(x[i])
256+
}} else {{
257+
col.append!(x[i,])
258+
}}
259+
}}
260+
table(col as c_type)
261+
""")
262+
expect_pd = pd.DataFrame({'c_type': pd.Series([expect[i] for i in index], dtype=dtype)})
263+
assert equalPlus(result, expect_pd)

‎test/basic_testing/test_upload_new.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1+
from decimal import Decimal
12
from importlib.util import find_spec
3+
from random import randint
24

35
import dolphindb as ddb
46
import dolphindb.settings as keys
@@ -1179,3 +1181,82 @@ def test_upload_table_arrow_array_vector_contain_empty(self):
11791181
assertPlus(self.__class__.conn.run(f"typestr({k}[`a])=={v['expect_typestr']}"))
11801182
if 'expect_value' in v:
11811183
assertPlus(self.__class__.conn.run(f"{k}=={v['expect_value']}"))
1184+
1185+
@pytest.mark.parametrize('script,expect,dtype,type', [
1186+
["true false 00b", [True, False, None], "object", keys.DT_BOOL],
1187+
["0c 127c -127c 00c", [0, 127, -127, np.nan], "object", keys.DT_CHAR],
1188+
["0h 32767h -32767h 00h", [0, 32767, -32767, np.nan], "object", keys.DT_SHORT],
1189+
["0i 2147483647i -2147483647i 00i", [0, 2147483647, -2147483647, np.nan], "object", keys.DT_INT],
1190+
["0l 9223372036854775807l -9223372036854775807l 00l", [0, 9223372036854775807, -9223372036854775807, np.nan],
1191+
"object", keys.DT_LONG],
1192+
["1970.01.01d 1970.01.02d 00d",
1193+
[np.datetime64('1970-01-01', 'D'), np.datetime64('1970-01-02', 'D'), np.datetime64('nat', 'D')],
1194+
'datetime64[ns]', keys.DT_DATE],
1195+
["1970.01M 1970.02M 00M",
1196+
[np.datetime64('1970-01', 'M'), np.datetime64('1970-02', 'M'), np.datetime64('nat', 'M')], 'datetime64[ns]',
1197+
keys.DT_MONTH],
1198+
["00:00:00.000t 00:00:00.001t 00t",
1199+
[np.datetime64('1970-01-01T00:00:00.000', 'ms'), np.datetime64('1970-01-01T00:00:00.001', 'ms'),
1200+
np.datetime64('nat', 'ms')], 'datetime64[ns]', keys.DT_TIME],
1201+
["00:00m 00:01m 00m",
1202+
[np.datetime64('1970-01-01T00:00', 'm'), np.datetime64('1970-01-01T00:01', 'm'), np.datetime64('nat', 'm')],
1203+
'datetime64[ns]', keys.DT_MINUTE],
1204+
["00:00:00s 00:00:01s 00s",
1205+
[np.datetime64('1970-01-01T00:00:00', 's'), np.datetime64('1970-01-01T00:00:01', 's'),
1206+
np.datetime64('nat', 's')], 'datetime64[ns]', keys.DT_SECOND],
1207+
["1970.01.01T00:00:00D 1970.01.01T00:00:01D 00D",
1208+
[np.datetime64('1970-01-01T00:00:00', 's'), np.datetime64('1970-01-01T00:00:01', 's'),
1209+
np.datetime64('nat', 's')], 'datetime64[ns]', keys.DT_DATETIME],
1210+
["1970.01.01 00:00:00.000T 1970.01.01 00:00:00.001T 00T",
1211+
[np.datetime64('1970-01-01T00:00:00.000', 'ms'), np.datetime64('1970-01-01T00:00:00.001', 'ms'),
1212+
np.datetime64('nat', 'ms')], 'datetime64[ns]', keys.DT_TIMESTAMP],
1213+
["00:00:00.000000000n 00:00:00.000000001n 00n",
1214+
[np.datetime64('1970-01-01T00:00:00.000000000', 'ns'), np.datetime64('1970-01-01T00:00:00.000000001', 'ns'),
1215+
np.datetime64('nat', 'ns')], "datetime64[ns]", keys.DT_NANOTIME],
1216+
["1970.01.01 00:00:00.000000000N 1970.01.01 00:00:00.000000001N 00N",
1217+
[np.datetime64('1970-01-01T00:00:00.000000000', 'ns'), np.datetime64('1970-01-01T00:00:00.000000001', 'ns'),
1218+
np.datetime64('nat', 'ns')], "datetime64[ns]", keys.DT_NANOTIMESTAMP],
1219+
["0.0f 3.14f 00f", [0., 3.14, np.nan], np.float32, keys.DT_FLOAT],
1220+
["0.0F 3.14F 00F", [0., 3.14, np.nan], np.float64, keys.DT_DOUBLE],
1221+
["'abc!@#中文 123' \"\"", ['abc!@#中文 123', ''], "object", keys.DT_STRING],
1222+
["[uuid('5d212a78-cc48-e3b1-4235-b4d91473ee87'),uuid('00000000-0000-0000-0000-000000000000')]",
1223+
['5d212a78-cc48-e3b1-4235-b4d91473ee87', '00000000-0000-0000-0000-000000000000'], "object", keys.DT_UUID],
1224+
["[datehour('1970.01.01T00'),datehour(NULL)]", [np.datetime64('1970-01-01T00', 'h'), np.datetime64('nat', 'h')],
1225+
"datetime64[ns]", keys.DT_DATEHOUR],
1226+
["[ipaddr('127.0.0.1'),ipaddr('0.0.0.0')]", ['127.0.0.1', '0.0.0.0'], "object", keys.DT_IPADDR],
1227+
["[int128('e1671797c52e15f763380b45e841ec32'),int128('00000000000000000000000000000000')]",
1228+
['e1671797c52e15f763380b45e841ec32', '00000000000000000000000000000000'], "object", keys.DT_INT128],
1229+
["[blob('abc!@#中文 123'),blob('')]", ['abc!@#中文 123'.encode(), b''], "object", keys.DT_BLOB],
1230+
["[decimal32(\"0\",3),decimal32('3.141',3)]", [Decimal('0.000'), Decimal('3.141')], "object",
1231+
[keys.DT_DECIMAL32, 3]],
1232+
["[decimal64(\"0\",3),decimal64('3.141',3)]", [Decimal('0.000'), Decimal('3.141')], "object",
1233+
[keys.DT_DECIMAL64, 3]],
1234+
["[decimal128(\"0\",3),decimal128('3.141',3)]", [Decimal('0.000'), Decimal('3.141')], "object",
1235+
[keys.DT_DECIMAL64, 3]],
1236+
["array(BOOL[]).append!([[true,false,00b],[true],[false],[00b]])",
1237+
[np.array([True, False, None]), np.array([True]), np.array([False]), np.array([None])], "object",
1238+
keys.DT_BOOL_ARRAY],
1239+
], ids=["BOOL", "CHAR", "SHORT", "INT", "LONG", "DATE", "MONTH", "TIME", "MINUTE", "SECOND", "DATETIME",
1240+
"TIMESTAMP", "NANOTIME", "NANOTIMESTAMP", "FLOAT", "DOUBLE", "STRING", "UUID", "DATEHOUR", "IPADDR",
1241+
"INT128", "BLOB", "DECIMAL32", "DECIMAL64", "DECIMAL128", "BOOL_ARRAY"])
1242+
def test_upload_table_ge_65535(self, script, expect, dtype, type):
1243+
n = 200000
1244+
index = [randint(0, len(expect) - 1) for i in range(n)]
1245+
conn = ddb.Session(HOST, PORT, USER, PASSWD)
1246+
data = pd.DataFrame({'c_type': pd.Series([expect[i] for i in index], dtype=dtype)})
1247+
data.__DolphinDB_Type__ = {
1248+
"c_type": type
1249+
}
1250+
conn.upload({"index": index, "data": data})
1251+
assert conn.run(f"""
1252+
x = {script}
1253+
col = array(x,capacity={n})
1254+
for (i in index){{
1255+
if (type(x)<64){{
1256+
col.append!(x[i])
1257+
}} else {{
1258+
col.append!(x[i,])
1259+
}}
1260+
}}
1261+
eqObj(col,data[`c_type])
1262+
""")

‎test/basic_testing/utils.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ def assertPlus(data):
153153
# else:
154154
# sleep(0.5)
155155

156-
def operateNodes(conn,nodes,operate):
156+
def operateNodes(conn, nodes, operate):
157157
conn.run("""
158158
def check_nodes(state_,nodes){
159159
return all(exec state==state_ from rpc(getControllerAlias(),getClusterPerf) where name in nodes)
@@ -192,4 +192,4 @@ def check_nodes_start(nodes){
192192
else:
193193
sleep(0.5)
194194
else:
195-
conn.run("check_nodes_start", nodes)
195+
conn.run("check_nodes_start", nodes)

‎test/test_CEP.py‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,10 @@ def __eq__(self, other):
4545

4646
func_name = inspect.currentframe().f_code.co_name
4747
scripts = f"""
48-
all_pubTables = getStreamingStat().pubTables
49-
for(pubTables in all_pubTables){{
50-
if (pubTables.tableName==`{func_name}_input){{
51-
stopPublishTable(pubTables.subscriber.split(":")[0],int(pubTables.subscriber.split(":")[1]),pubTables.tableName,pubTables.actions)
52-
break
53-
}}
48+
subscribers = select * from getStreamingStat().pubTables where tableName=`{func_name};
49+
for(subscriber in subscribers){{
50+
ip_port = subscriber.subscriber.split(":");
51+
stopPublishTable(ip_port[0],int(ip_port[1]),subscriber.tableName,subscriber.actions);
5452
}}
5553
try{{dropStreamEngine(`{func_name}_serOutput)}} catch(ex) {{}}
5654
try{{dropStreamEngine(`{func_name}_cep1)}} catch(ex) {{}}
@@ -148,12 +146,10 @@ def __eq__(self, other):
148146

149147
func_name = inspect.currentframe().f_code.co_name
150148
scripts = f"""
151-
all_pubTables = getStreamingStat().pubTables
152-
for(pubTables in all_pubTables){{
153-
if (pubTables.tableName==`{func_name}_input){{
154-
stopPublishTable(pubTables.subscriber.split(":")[0],int(pubTables.subscriber.split(":")[1]),pubTables.tableName,pubTables.actions)
155-
break
156-
}}
149+
subscribers = select * from getStreamingStat().pubTables where tableName=`{func_name};
150+
for(subscriber in subscribers){{
151+
ip_port = subscriber.subscriber.split(":");
152+
stopPublishTable(ip_port[0],int(ip_port[1]),subscriber.tableName,subscriber.actions);
157153
}}
158154
try{{dropStreamEngine(`{func_name}_serOutput)}} catch(ex) {{}}
159155
try{{dropStreamEngine(`{func_name}_cep1)}} catch(ex) {{}}

‎test/test_readme_cn.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,7 @@ def test_readme_BasicOperation_AsynchronousWrite_SessionAsynchronousSubmission_1
389389
s = ddb.Session(enableASYNC=False)
390390
s.connect(HOST, PORT, USER, PASSWD)
391391
res = s.run(f"select * from loadTable('{db_name}', `{tbName}) order by price")
392-
assert_frame_equal(tb.sort_values(by='price').reset_index(drop=True), res)
392+
assert_frame_equal(tb.sort_values(by='price').reset_index(drop=True), res, False)
393393
s.close()
394394

395395
def test_readme_BasicOperation_AsynchronousWrite_SessionAsynchronousSubmission_2(self):

0 commit comments

Comments
 (0)