From 348ad68959e978d771b9756675ddd5fd591e301c Mon Sep 17 00:00:00 2001 From: Chuxiao Feng Date: Sun, 11 Sep 2022 21:13:12 -0500 Subject: [PATCH 1/3] support training from Python package --- Cargo.lock | 2 + crates/www/content/Cargo.toml | 1 + languages/python/Cargo.toml | 4 +- languages/python/Makefile | 12 + .../python/examples/basic/heart_disease.csv | 304 ++++++++++++++++++ .../examples/basic/heart_disease.modelfox | Bin 0 -> 29071 bytes languages/python/examples/basic/main.py | 9 +- .../python/examples/basic/pyrightconfig.json | 5 + languages/python/examples/basic/train.py | 33 ++ languages/python/lib.rs | 69 +++- languages/python/modelfox/tangram_python.pyi | 9 +- 11 files changed, 440 insertions(+), 8 deletions(-) create mode 100644 languages/python/Makefile create mode 100644 languages/python/examples/basic/heart_disease.csv create mode 100644 languages/python/examples/basic/heart_disease.modelfox create mode 100644 languages/python/examples/basic/pyrightconfig.json create mode 100644 languages/python/examples/basic/train.py diff --git a/Cargo.lock b/Cargo.lock index 71464cb0..e516fa7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3273,6 +3273,8 @@ dependencies = [ "chrono", "memmap", "modelfox_core", + "modelfox_id", + "modelfox_kill_chip", "modelfox_model", "pyo3", "reqwest", diff --git a/crates/www/content/Cargo.toml b/crates/www/content/Cargo.toml index 13270231..ddef8a74 100644 --- a/crates/www/content/Cargo.toml +++ b/crates/www/content/Cargo.toml @@ -15,6 +15,7 @@ version = { workspace = true } [lib] path = "lib.rs" +doctest = false [dependencies] anyhow = { workspace = true } diff --git a/languages/python/Cargo.toml b/languages/python/Cargo.toml index 4d13de66..61914724 100644 --- a/languages/python/Cargo.toml +++ b/languages/python/Cargo.toml @@ -26,4 +26,6 @@ serde_json = { workspace = true } url = { workspace = true } modelfox_core = { workspace = true } -modelfox_model = { workspace = true } \ No newline at end of file +modelfox_model = { workspace = true } +modelfox_kill_chip = { workspace = true } +modelfox_id = { workspace = true } diff --git a/languages/python/Makefile b/languages/python/Makefile new file mode 100644 index 00000000..82efffdc --- /dev/null +++ b/languages/python/Makefile @@ -0,0 +1,12 @@ +.PHONY: test dev + +test: dev + .venv/bin/python examples/basic/main.py + +dev: .venv + cargo build -p modelfox_python + cp ../../target/debug/libmodelfox_python.so modelfox/modelfox_python.so + .venv/bin/pip install -e . + +.venv: + virtualenv .venv diff --git a/languages/python/examples/basic/heart_disease.csv b/languages/python/examples/basic/heart_disease.csv new file mode 100644 index 00000000..f3b713d2 --- /dev/null +++ b/languages/python/examples/basic/heart_disease.csv @@ -0,0 +1,304 @@ +age,gender,chest_pain,resting_blood_pressure,cholesterol,fasting_blood_sugar_greater_than_120,resting_ecg_result,exercise_max_heart_rate,exercise_induced_angina,exercise_st_depression,exercise_st_slope,fluoroscopy_vessels_colored,thallium_stress_test,diagnosis +63,male,typical angina,145,233,true,probable or definite left ventricular hypertrophy,150,no,2.3,downsloping,0,fixed defect,Negative +67,male,asymptomatic,160,286,false,probable or definite left ventricular hypertrophy,108,yes,1.5,flat,3,normal,Positive +67,male,asymptomatic,120,229,false,probable or definite left ventricular hypertrophy,129,yes,2.6,flat,2,reversible defect,Positive +37,male,non-angina pain,130,250,false,normal,187,no,3.5,downsloping,0,normal,Negative +41,female,atypical angina,130,204,false,probable or definite left ventricular hypertrophy,172,no,1.4,upsloping,0,normal,Negative +56,male,atypical angina,120,236,false,normal,178,no,0.8,upsloping,0,normal,Negative +62,female,asymptomatic,140,268,false,probable or definite left ventricular hypertrophy,160,no,3.6,downsloping,2,normal,Positive +57,female,asymptomatic,120,354,false,normal,163,yes,0.6,upsloping,0,normal,Negative +63,male,asymptomatic,130,254,false,probable or definite left ventricular hypertrophy,147,no,1.4,flat,1,reversible defect,Positive +53,male,asymptomatic,140,203,true,probable or definite left ventricular hypertrophy,155,yes,3.1,downsloping,0,reversible defect,Positive +57,male,asymptomatic,140,192,false,normal,148,no,0.4,flat,0,fixed defect,Negative +56,female,atypical angina,140,294,false,probable or definite left ventricular hypertrophy,153,no,1.3,flat,0,normal,Negative +56,male,non-angina pain,130,256,true,probable or definite left ventricular hypertrophy,142,yes,0.6,flat,1,fixed defect,Positive +44,male,atypical angina,120,263,false,normal,173,no,0,upsloping,0,reversible defect,Negative +52,male,non-angina pain,172,199,true,normal,162,no,0.5,upsloping,0,reversible defect,Negative +57,male,non-angina pain,150,168,false,normal,174,no,1.6,upsloping,0,normal,Negative +48,male,atypical angina,110,229,false,normal,168,no,1,downsloping,0,reversible defect,Positive +54,male,asymptomatic,140,239,false,normal,160,no,1.2,upsloping,0,normal,Negative +48,female,non-angina pain,130,275,false,normal,139,no,0.2,upsloping,0,normal,Negative +49,male,atypical angina,130,266,false,normal,171,no,0.6,upsloping,0,normal,Negative +64,male,typical angina,110,211,false,probable or definite left ventricular hypertrophy,144,yes,1.8,flat,0,normal,Negative +58,female,typical angina,150,283,true,probable or definite left ventricular hypertrophy,162,no,1,upsloping,0,normal,Negative +58,male,atypical angina,120,284,false,probable or definite left ventricular hypertrophy,160,no,1.8,flat,0,normal,Positive +58,male,non-angina pain,132,224,false,probable or definite left ventricular hypertrophy,173,no,3.2,upsloping,2,reversible defect,Positive +60,male,asymptomatic,130,206,false,probable or definite left ventricular hypertrophy,132,yes,2.4,flat,2,reversible defect,Positive +50,female,non-angina pain,120,219,false,normal,158,no,1.6,flat,0,normal,Negative +58,female,non-angina pain,120,340,false,normal,172,no,0,upsloping,0,normal,Negative +66,female,typical angina,150,226,false,normal,114,no,2.6,downsloping,0,normal,Negative +43,male,asymptomatic,150,247,false,normal,171,no,1.5,upsloping,0,normal,Negative +40,male,asymptomatic,110,167,false,probable or definite left ventricular hypertrophy,114,yes,2,flat,0,reversible defect,Positive +69,female,typical angina,140,239,false,normal,151,no,1.8,upsloping,2,normal,Negative +60,male,asymptomatic,117,230,true,normal,160,yes,1.4,upsloping,2,reversible defect,Positive +64,male,non-angina pain,140,335,false,normal,158,no,0,upsloping,0,normal,Positive +59,male,asymptomatic,135,234,false,normal,161,no,0.5,flat,0,reversible defect,Negative +44,male,non-angina pain,130,233,false,normal,179,yes,0.4,upsloping,0,normal,Negative +42,male,asymptomatic,140,226,false,normal,178,no,0,upsloping,0,normal,Negative +43,male,asymptomatic,120,177,false,probable or definite left ventricular hypertrophy,120,yes,2.5,flat,0,reversible defect,Positive +57,male,asymptomatic,150,276,false,probable or definite left ventricular hypertrophy,112,yes,0.6,flat,1,fixed defect,Positive +55,male,asymptomatic,132,353,false,normal,132,yes,1.2,flat,1,reversible defect,Positive +61,male,non-angina pain,150,243,true,normal,137,yes,1,flat,0,normal,Negative +65,female,asymptomatic,150,225,false,probable or definite left ventricular hypertrophy,114,no,1,flat,3,reversible defect,Positive +40,male,typical angina,140,199,false,normal,178,yes,1.4,upsloping,0,reversible defect,Negative +71,female,atypical angina,160,302,false,normal,162,no,0.4,upsloping,2,normal,Negative +59,male,non-angina pain,150,212,true,normal,157,no,1.6,upsloping,0,normal,Negative +61,female,asymptomatic,130,330,false,probable or definite left ventricular hypertrophy,169,no,0,upsloping,0,normal,Positive +58,male,non-angina pain,112,230,false,probable or definite left ventricular hypertrophy,165,no,2.5,flat,1,reversible defect,Positive +51,male,non-angina pain,110,175,false,normal,123,no,0.6,upsloping,0,normal,Negative +50,male,asymptomatic,150,243,false,probable or definite left ventricular hypertrophy,128,no,2.6,flat,0,reversible defect,Positive +65,female,non-angina pain,140,417,true,probable or definite left ventricular hypertrophy,157,no,0.8,upsloping,1,normal,Negative +53,male,non-angina pain,130,197,true,probable or definite left ventricular hypertrophy,152,no,1.2,downsloping,0,normal,Negative +41,female,atypical angina,105,198,false,normal,168,no,0,upsloping,1,normal,Negative +65,male,asymptomatic,120,177,false,normal,140,no,0.4,upsloping,0,reversible defect,Negative +44,male,asymptomatic,112,290,false,probable or definite left ventricular hypertrophy,153,no,0,upsloping,1,normal,Positive +44,male,atypical angina,130,219,false,probable or definite left ventricular hypertrophy,188,no,0,upsloping,0,normal,Negative +60,male,asymptomatic,130,253,false,normal,144,yes,1.4,upsloping,1,reversible defect,Positive +54,male,asymptomatic,124,266,false,probable or definite left ventricular hypertrophy,109,yes,2.2,flat,1,reversible defect,Positive +50,male,non-angina pain,140,233,false,normal,163,no,0.6,flat,1,reversible defect,Positive +41,male,asymptomatic,110,172,false,probable or definite left ventricular hypertrophy,158,no,0,upsloping,0,reversible defect,Positive +54,male,non-angina pain,125,273,false,probable or definite left ventricular hypertrophy,152,no,0.5,downsloping,1,normal,Negative +51,male,typical angina,125,213,false,probable or definite left ventricular hypertrophy,125,yes,1.4,upsloping,1,normal,Negative +51,female,asymptomatic,130,305,false,normal,142,yes,1.2,flat,0,reversible defect,Positive +46,female,non-angina pain,142,177,false,probable or definite left ventricular hypertrophy,160,yes,1.4,downsloping,0,normal,Negative +58,male,asymptomatic,128,216,false,probable or definite left ventricular hypertrophy,131,yes,2.2,flat,3,reversible defect,Positive +54,female,non-angina pain,135,304,true,normal,170,no,0,upsloping,0,normal,Negative +54,male,asymptomatic,120,188,false,normal,113,no,1.4,flat,1,reversible defect,Positive +60,male,asymptomatic,145,282,false,probable or definite left ventricular hypertrophy,142,yes,2.8,flat,2,reversible defect,Positive +60,male,non-angina pain,140,185,false,probable or definite left ventricular hypertrophy,155,no,3,flat,0,normal,Positive +54,male,non-angina pain,150,232,false,probable or definite left ventricular hypertrophy,165,no,1.6,upsloping,0,reversible defect,Negative +59,male,asymptomatic,170,326,false,probable or definite left ventricular hypertrophy,140,yes,3.4,downsloping,0,reversible defect,Positive +46,male,non-angina pain,150,231,false,normal,147,no,3.6,flat,0,normal,Positive +65,female,non-angina pain,155,269,false,normal,148,no,0.8,upsloping,0,normal,Negative +67,male,asymptomatic,125,254,true,normal,163,no,0.2,flat,2,reversible defect,Positive +62,male,asymptomatic,120,267,false,normal,99,yes,1.8,flat,2,reversible defect,Positive +65,male,asymptomatic,110,248,false,probable or definite left ventricular hypertrophy,158,no,0.6,upsloping,2,fixed defect,Positive +44,male,asymptomatic,110,197,false,probable or definite left ventricular hypertrophy,177,no,0,upsloping,1,normal,Positive +65,female,non-angina pain,160,360,false,probable or definite left ventricular hypertrophy,151,no,0.8,upsloping,0,normal,Negative +60,male,asymptomatic,125,258,false,probable or definite left ventricular hypertrophy,141,yes,2.8,flat,1,reversible defect,Positive +51,female,non-angina pain,140,308,false,probable or definite left ventricular hypertrophy,142,no,1.5,upsloping,1,normal,Negative +48,male,atypical angina,130,245,false,probable or definite left ventricular hypertrophy,180,no,0.2,flat,0,normal,Negative +58,male,asymptomatic,150,270,false,probable or definite left ventricular hypertrophy,111,yes,0.8,upsloping,0,reversible defect,Positive +45,male,asymptomatic,104,208,false,probable or definite left ventricular hypertrophy,148,yes,3,flat,0,normal,Negative +53,female,asymptomatic,130,264,false,probable or definite left ventricular hypertrophy,143,no,0.4,flat,0,normal,Negative +39,male,non-angina pain,140,321,false,probable or definite left ventricular hypertrophy,182,no,0,upsloping,0,normal,Negative +68,male,non-angina pain,180,274,true,probable or definite left ventricular hypertrophy,150,yes,1.6,flat,0,reversible defect,Positive +52,male,atypical angina,120,325,false,normal,172,no,0.2,upsloping,0,normal,Negative +44,male,non-angina pain,140,235,false,probable or definite left ventricular hypertrophy,180,no,0,upsloping,0,normal,Negative +47,male,non-angina pain,138,257,false,probable or definite left ventricular hypertrophy,156,no,0,upsloping,0,normal,Negative +53,female,non-angina pain,128,216,false,probable or definite left ventricular hypertrophy,115,no,0,upsloping,0,,Negative +53,female,asymptomatic,138,234,false,probable or definite left ventricular hypertrophy,160,no,0,upsloping,0,normal,Negative +51,female,non-angina pain,130,256,false,probable or definite left ventricular hypertrophy,149,no,0.5,upsloping,0,normal,Negative +66,male,asymptomatic,120,302,false,probable or definite left ventricular hypertrophy,151,no,0.4,flat,0,normal,Negative +62,female,asymptomatic,160,164,false,probable or definite left ventricular hypertrophy,145,no,6.2,downsloping,3,reversible defect,Positive +62,male,non-angina pain,130,231,false,normal,146,no,1.8,flat,3,reversible defect,Negative +44,female,non-angina pain,108,141,false,normal,175,no,0.6,flat,0,normal,Negative +63,female,non-angina pain,135,252,false,probable or definite left ventricular hypertrophy,172,no,0,upsloping,0,normal,Negative +52,male,asymptomatic,128,255,false,normal,161,yes,0,upsloping,1,reversible defect,Positive +59,male,asymptomatic,110,239,false,probable or definite left ventricular hypertrophy,142,yes,1.2,flat,1,reversible defect,Positive +60,female,asymptomatic,150,258,false,probable or definite left ventricular hypertrophy,157,no,2.6,flat,2,reversible defect,Positive +52,male,atypical angina,134,201,false,normal,158,no,0.8,upsloping,1,normal,Negative +48,male,asymptomatic,122,222,false,probable or definite left ventricular hypertrophy,186,no,0,upsloping,0,normal,Negative +45,male,asymptomatic,115,260,false,probable or definite left ventricular hypertrophy,185,no,0,upsloping,0,normal,Negative +34,male,typical angina,118,182,false,probable or definite left ventricular hypertrophy,174,no,0,upsloping,0,normal,Negative +57,female,asymptomatic,128,303,false,probable or definite left ventricular hypertrophy,159,no,0,upsloping,1,normal,Negative +71,female,non-angina pain,110,265,true,probable or definite left ventricular hypertrophy,130,no,0,upsloping,1,normal,Negative +49,male,non-angina pain,120,188,false,normal,139,no,2,flat,3,reversible defect,Positive +54,male,atypical angina,108,309,false,normal,156,no,0,upsloping,0,reversible defect,Negative +59,male,asymptomatic,140,177,false,normal,162,yes,0,upsloping,1,reversible defect,Positive +57,male,non-angina pain,128,229,false,probable or definite left ventricular hypertrophy,150,no,0.4,flat,1,reversible defect,Positive +61,male,asymptomatic,120,260,false,normal,140,yes,3.6,flat,1,reversible defect,Positive +39,male,asymptomatic,118,219,false,normal,140,no,1.2,flat,0,reversible defect,Positive +61,female,asymptomatic,145,307,false,probable or definite left ventricular hypertrophy,146,yes,1,flat,0,reversible defect,Positive +56,male,asymptomatic,125,249,true,probable or definite left ventricular hypertrophy,144,yes,1.2,flat,1,normal,Positive +52,male,typical angina,118,186,false,probable or definite left ventricular hypertrophy,190,no,0,flat,0,fixed defect,Negative +43,female,asymptomatic,132,341,true,probable or definite left ventricular hypertrophy,136,yes,3,flat,0,reversible defect,Positive +62,female,non-angina pain,130,263,false,normal,97,no,1.2,flat,1,reversible defect,Positive +41,male,atypical angina,135,203,false,normal,132,no,0,flat,0,fixed defect,Negative +58,male,non-angina pain,140,211,true,probable or definite left ventricular hypertrophy,165,no,0,upsloping,0,normal,Negative +35,female,asymptomatic,138,183,false,normal,182,no,1.4,upsloping,0,normal,Negative +63,male,asymptomatic,130,330,true,probable or definite left ventricular hypertrophy,132,yes,1.8,upsloping,3,reversible defect,Positive +65,male,asymptomatic,135,254,false,probable or definite left ventricular hypertrophy,127,no,2.8,flat,1,reversible defect,Positive +48,male,asymptomatic,130,256,true,probable or definite left ventricular hypertrophy,150,yes,0,upsloping,2,reversible defect,Positive +63,female,asymptomatic,150,407,false,probable or definite left ventricular hypertrophy,154,no,4,flat,3,reversible defect,Positive +51,male,non-angina pain,100,222,false,normal,143,yes,1.2,flat,0,normal,Negative +55,male,asymptomatic,140,217,false,normal,111,yes,5.6,downsloping,0,reversible defect,Positive +65,male,typical angina,138,282,true,probable or definite left ventricular hypertrophy,174,no,1.4,flat,1,normal,Positive +45,female,atypical angina,130,234,false,probable or definite left ventricular hypertrophy,175,no,0.6,flat,0,normal,Negative +56,female,asymptomatic,200,288,true,probable or definite left ventricular hypertrophy,133,yes,4,downsloping,2,reversible defect,Positive +54,male,asymptomatic,110,239,false,normal,126,yes,2.8,flat,1,reversible defect,Positive +44,male,atypical angina,120,220,false,normal,170,no,0,upsloping,0,normal,Negative +62,female,asymptomatic,124,209,false,normal,163,no,0,upsloping,0,normal,Negative +54,male,non-angina pain,120,258,false,probable or definite left ventricular hypertrophy,147,no,0.4,flat,0,reversible defect,Negative +51,male,non-angina pain,94,227,false,normal,154,yes,0,upsloping,1,reversible defect,Negative +29,male,atypical angina,130,204,false,probable or definite left ventricular hypertrophy,202,no,0,upsloping,0,normal,Negative +51,male,asymptomatic,140,261,false,probable or definite left ventricular hypertrophy,186,yes,0,upsloping,0,normal,Negative +43,female,non-angina pain,122,213,false,normal,165,no,0.2,flat,0,normal,Negative +55,female,atypical angina,135,250,false,probable or definite left ventricular hypertrophy,161,no,1.4,flat,0,normal,Negative +70,male,asymptomatic,145,174,false,normal,125,yes,2.6,downsloping,0,reversible defect,Positive +62,male,atypical angina,120,281,false,probable or definite left ventricular hypertrophy,103,no,1.4,flat,1,reversible defect,Positive +35,male,asymptomatic,120,198,false,normal,130,yes,1.6,flat,0,reversible defect,Positive +51,male,non-angina pain,125,245,true,probable or definite left ventricular hypertrophy,166,no,2.4,flat,0,normal,Negative +59,male,atypical angina,140,221,false,normal,164,yes,0,upsloping,0,normal,Negative +59,male,typical angina,170,288,false,probable or definite left ventricular hypertrophy,159,no,0.2,flat,0,reversible defect,Positive +52,male,atypical angina,128,205,true,normal,184,no,0,upsloping,0,normal,Negative +64,male,non-angina pain,125,309,false,normal,131,yes,1.8,flat,0,reversible defect,Positive +58,male,non-angina pain,105,240,false,probable or definite left ventricular hypertrophy,154,yes,0.6,flat,0,reversible defect,Negative +47,male,non-angina pain,108,243,false,normal,152,no,0,upsloping,0,normal,Positive +57,male,asymptomatic,165,289,true,probable or definite left ventricular hypertrophy,124,no,1,flat,3,reversible defect,Positive +41,male,non-angina pain,112,250,false,normal,179,no,0,upsloping,0,normal,Negative +45,male,atypical angina,128,308,false,probable or definite left ventricular hypertrophy,170,no,0,upsloping,0,normal,Negative +60,female,non-angina pain,102,318,false,normal,160,no,0,upsloping,1,normal,Negative +52,male,typical angina,152,298,true,normal,178,no,1.2,flat,0,reversible defect,Negative +42,female,asymptomatic,102,265,false,probable or definite left ventricular hypertrophy,122,no,0.6,flat,0,normal,Negative +67,female,non-angina pain,115,564,false,probable or definite left ventricular hypertrophy,160,no,1.6,flat,0,reversible defect,Negative +55,male,asymptomatic,160,289,false,probable or definite left ventricular hypertrophy,145,yes,0.8,flat,1,reversible defect,Positive +64,male,asymptomatic,120,246,false,probable or definite left ventricular hypertrophy,96,yes,2.2,downsloping,1,normal,Positive +70,male,asymptomatic,130,322,false,probable or definite left ventricular hypertrophy,109,no,2.4,flat,3,normal,Positive +51,male,asymptomatic,140,299,false,normal,173,yes,1.6,upsloping,0,reversible defect,Positive +58,male,asymptomatic,125,300,false,probable or definite left ventricular hypertrophy,171,no,0,upsloping,2,reversible defect,Positive +60,male,asymptomatic,140,293,false,probable or definite left ventricular hypertrophy,170,no,1.2,flat,2,reversible defect,Positive +68,male,non-angina pain,118,277,false,normal,151,no,1,upsloping,1,reversible defect,Negative +46,male,atypical angina,101,197,true,normal,156,no,0,upsloping,0,reversible defect,Negative +77,male,asymptomatic,125,304,false,probable or definite left ventricular hypertrophy,162,yes,0,upsloping,3,normal,Positive +54,female,non-angina pain,110,214,false,normal,158,no,1.6,flat,0,normal,Negative +58,female,asymptomatic,100,248,false,probable or definite left ventricular hypertrophy,122,no,1,flat,0,normal,Negative +48,male,non-angina pain,124,255,true,normal,175,no,0,upsloping,2,normal,Negative +57,male,asymptomatic,132,207,false,normal,168,yes,0,upsloping,0,reversible defect,Negative +52,male,non-angina pain,138,223,false,normal,169,no,0,upsloping,,normal,Negative +54,female,atypical angina,132,288,true,probable or definite left ventricular hypertrophy,159,yes,0,upsloping,1,normal,Negative +35,male,asymptomatic,126,282,false,probable or definite left ventricular hypertrophy,156,yes,0,upsloping,0,reversible defect,Positive +45,female,atypical angina,112,160,false,normal,138,no,0,flat,0,normal,Negative +70,male,non-angina pain,160,269,false,normal,112,yes,2.9,flat,1,reversible defect,Positive +53,male,asymptomatic,142,226,false,probable or definite left ventricular hypertrophy,111,yes,0,upsloping,0,reversible defect,Negative +59,female,asymptomatic,174,249,false,normal,143,yes,0,flat,0,normal,Positive +62,female,asymptomatic,140,394,false,probable or definite left ventricular hypertrophy,157,no,1.2,flat,0,normal,Negative +64,male,asymptomatic,145,212,false,probable or definite left ventricular hypertrophy,132,no,2,flat,2,fixed defect,Positive +57,male,asymptomatic,152,274,false,normal,88,yes,1.2,flat,1,reversible defect,Positive +52,male,asymptomatic,108,233,true,normal,147,no,0.1,upsloping,3,reversible defect,Negative +56,male,asymptomatic,132,184,false,probable or definite left ventricular hypertrophy,105,yes,2.1,flat,1,fixed defect,Positive +43,male,non-angina pain,130,315,false,normal,162,no,1.9,upsloping,1,normal,Negative +53,male,non-angina pain,130,246,true,probable or definite left ventricular hypertrophy,173,no,0,upsloping,3,normal,Negative +48,male,asymptomatic,124,274,false,probable or definite left ventricular hypertrophy,166,no,0.5,flat,0,reversible defect,Positive +56,female,asymptomatic,134,409,false,probable or definite left ventricular hypertrophy,150,yes,1.9,flat,2,reversible defect,Positive +42,male,typical angina,148,244,false,probable or definite left ventricular hypertrophy,178,no,0.8,upsloping,2,normal,Negative +59,male,typical angina,178,270,false,probable or definite left ventricular hypertrophy,145,no,4.2,downsloping,0,reversible defect,Negative +60,female,asymptomatic,158,305,false,probable or definite left ventricular hypertrophy,161,no,0,upsloping,0,normal,Positive +63,female,atypical angina,140,195,false,normal,179,no,0,upsloping,2,normal,Negative +42,male,non-angina pain,120,240,true,normal,194,no,0.8,downsloping,0,reversible defect,Negative +66,male,atypical angina,160,246,false,normal,120,yes,0,flat,3,fixed defect,Positive +54,male,atypical angina,192,283,false,probable or definite left ventricular hypertrophy,195,no,0,upsloping,1,reversible defect,Positive +69,male,non-angina pain,140,254,false,probable or definite left ventricular hypertrophy,146,no,2,flat,3,reversible defect,Positive +50,male,non-angina pain,129,196,false,normal,163,no,0,upsloping,0,normal,Negative +51,male,asymptomatic,140,298,false,normal,122,yes,4.2,flat,3,reversible defect,Positive +43,male,asymptomatic,132,247,true,probable or definite left ventricular hypertrophy,143,yes,0.1,flat,,reversible defect,Positive +62,female,asymptomatic,138,294,true,normal,106,no,1.9,flat,3,normal,Positive +68,female,non-angina pain,120,211,false,probable or definite left ventricular hypertrophy,115,no,1.5,flat,0,normal,Negative +67,male,asymptomatic,100,299,false,probable or definite left ventricular hypertrophy,125,yes,0.9,flat,2,normal,Positive +69,male,typical angina,160,234,true,probable or definite left ventricular hypertrophy,131,no,0.1,flat,1,normal,Negative +45,female,asymptomatic,138,236,false,probable or definite left ventricular hypertrophy,152,yes,0.2,flat,0,normal,Negative +50,female,atypical angina,120,244,false,normal,162,no,1.1,upsloping,0,normal,Negative +59,male,typical angina,160,273,false,probable or definite left ventricular hypertrophy,125,no,0,upsloping,0,normal,Positive +50,female,asymptomatic,110,254,false,probable or definite left ventricular hypertrophy,159,no,0,upsloping,0,normal,Negative +64,female,asymptomatic,180,325,false,normal,154,yes,0,upsloping,0,normal,Negative +57,male,non-angina pain,150,126,true,normal,173,no,0.2,upsloping,1,reversible defect,Negative +64,female,non-angina pain,140,313,false,normal,133,no,0.2,upsloping,0,reversible defect,Negative +43,male,asymptomatic,110,211,false,normal,161,no,0,upsloping,0,reversible defect,Negative +45,male,asymptomatic,142,309,false,probable or definite left ventricular hypertrophy,147,yes,0,flat,3,reversible defect,Positive +58,male,asymptomatic,128,259,false,probable or definite left ventricular hypertrophy,130,yes,3,flat,2,reversible defect,Positive +50,male,asymptomatic,144,200,false,probable or definite left ventricular hypertrophy,126,yes,0.9,flat,0,reversible defect,Positive +55,male,atypical angina,130,262,false,normal,155,no,0,upsloping,0,normal,Negative +62,female,asymptomatic,150,244,false,normal,154,yes,1.4,flat,0,normal,Positive +37,female,non-angina pain,120,215,false,normal,170,no,0,upsloping,0,normal,Negative +38,male,typical angina,120,231,false,normal,182,yes,3.8,flat,0,reversible defect,Positive +41,male,non-angina pain,130,214,false,probable or definite left ventricular hypertrophy,168,no,2,flat,0,normal,Negative +66,female,asymptomatic,178,228,true,normal,165,yes,1,flat,2,reversible defect,Positive +52,male,asymptomatic,112,230,false,normal,160,no,0,upsloping,1,normal,Positive +56,male,typical angina,120,193,false,probable or definite left ventricular hypertrophy,162,no,1.9,flat,0,reversible defect,Negative +46,female,atypical angina,105,204,false,normal,172,no,0,upsloping,0,normal,Negative +46,female,asymptomatic,138,243,false,probable or definite left ventricular hypertrophy,152,yes,0,flat,0,normal,Negative +64,female,asymptomatic,130,303,false,normal,122,no,2,flat,2,normal,Negative +59,male,asymptomatic,138,271,false,probable or definite left ventricular hypertrophy,182,no,0,upsloping,0,normal,Negative +41,female,non-angina pain,112,268,false,probable or definite left ventricular hypertrophy,172,yes,0,upsloping,0,normal,Negative +54,female,non-angina pain,108,267,false,probable or definite left ventricular hypertrophy,167,no,0,upsloping,0,normal,Negative +39,female,non-angina pain,94,199,false,normal,179,no,0,upsloping,0,normal,Negative +53,male,asymptomatic,123,282,false,normal,95,yes,2,flat,2,reversible defect,Positive +63,female,asymptomatic,108,269,false,normal,169,yes,1.8,flat,2,normal,Positive +34,female,atypical angina,118,210,false,normal,192,no,0.7,upsloping,0,normal,Negative +47,male,asymptomatic,112,204,false,normal,143,no,0.1,upsloping,0,normal,Negative +67,female,non-angina pain,152,277,false,normal,172,no,0,upsloping,1,normal,Negative +54,male,asymptomatic,110,206,false,probable or definite left ventricular hypertrophy,108,yes,0,flat,1,normal,Positive +66,male,asymptomatic,112,212,false,probable or definite left ventricular hypertrophy,132,yes,0.1,upsloping,1,normal,Positive +52,female,non-angina pain,136,196,false,probable or definite left ventricular hypertrophy,169,no,0.1,flat,0,normal,Negative +55,female,asymptomatic,180,327,false,ST-T wave abnormality,117,yes,3.4,flat,0,normal,Positive +49,male,non-angina pain,118,149,false,probable or definite left ventricular hypertrophy,126,no,0.8,upsloping,3,normal,Positive +74,female,atypical angina,120,269,false,probable or definite left ventricular hypertrophy,121,yes,0.2,upsloping,1,normal,Negative +54,female,non-angina pain,160,201,false,normal,163,no,0,upsloping,1,normal,Negative +54,male,asymptomatic,122,286,false,probable or definite left ventricular hypertrophy,116,yes,3.2,flat,2,normal,Positive +56,male,asymptomatic,130,283,true,probable or definite left ventricular hypertrophy,103,yes,1.6,downsloping,0,reversible defect,Positive +46,male,asymptomatic,120,249,false,probable or definite left ventricular hypertrophy,144,no,0.8,upsloping,0,reversible defect,Positive +49,female,atypical angina,134,271,false,normal,162,no,0,flat,0,normal,Negative +42,male,atypical angina,120,295,false,normal,162,no,0,upsloping,0,normal,Negative +41,male,atypical angina,110,235,false,normal,153,no,0,upsloping,0,normal,Negative +41,female,atypical angina,126,306,false,normal,163,no,0,upsloping,0,normal,Negative +49,female,asymptomatic,130,269,false,normal,163,no,0,upsloping,0,normal,Negative +61,male,typical angina,134,234,false,normal,145,no,2.6,flat,2,normal,Positive +60,female,non-angina pain,120,178,true,normal,96,no,0,upsloping,0,normal,Negative +67,male,asymptomatic,120,237,false,normal,71,no,1,flat,0,normal,Positive +58,male,asymptomatic,100,234,false,normal,156,no,0.1,upsloping,1,reversible defect,Positive +47,male,asymptomatic,110,275,false,probable or definite left ventricular hypertrophy,118,yes,1,flat,1,normal,Positive +52,male,asymptomatic,125,212,false,normal,168,no,1,upsloping,2,reversible defect,Positive +62,male,atypical angina,128,208,true,probable or definite left ventricular hypertrophy,140,no,0,upsloping,0,normal,Negative +57,male,asymptomatic,110,201,false,normal,126,yes,1.5,flat,0,fixed defect,Negative +58,male,asymptomatic,146,218,false,normal,105,no,2,flat,1,reversible defect,Positive +64,male,asymptomatic,128,263,false,normal,105,yes,0.2,flat,1,reversible defect,Negative +51,female,non-angina pain,120,295,false,probable or definite left ventricular hypertrophy,157,no,0.6,upsloping,0,normal,Negative +43,male,asymptomatic,115,303,false,normal,181,no,1.2,flat,0,normal,Negative +42,female,non-angina pain,120,209,false,normal,173,no,0,flat,0,normal,Negative +67,female,asymptomatic,106,223,false,normal,142,no,0.3,upsloping,2,normal,Negative +76,female,non-angina pain,140,197,false,ST-T wave abnormality,116,no,1.1,flat,0,normal,Negative +70,male,atypical angina,156,245,false,probable or definite left ventricular hypertrophy,143,no,0,upsloping,0,normal,Negative +57,male,atypical angina,124,261,false,normal,141,no,0.3,upsloping,0,reversible defect,Positive +44,female,non-angina pain,118,242,false,normal,149,no,0.3,flat,1,normal,Negative +58,female,atypical angina,136,319,true,probable or definite left ventricular hypertrophy,152,no,0,upsloping,2,normal,Positive +60,female,typical angina,150,240,false,normal,171,no,0.9,upsloping,0,normal,Negative +44,male,non-angina pain,120,226,false,normal,169,no,0,upsloping,0,normal,Negative +61,male,asymptomatic,138,166,false,probable or definite left ventricular hypertrophy,125,yes,3.6,flat,1,normal,Positive +42,male,asymptomatic,136,315,false,normal,125,yes,1.8,flat,0,fixed defect,Positive +52,male,asymptomatic,128,204,true,normal,156,yes,1,flat,0,,Positive +59,male,non-angina pain,126,218,true,normal,134,no,2.2,flat,1,fixed defect,Positive +40,male,asymptomatic,152,223,false,normal,181,no,0,upsloping,0,reversible defect,Positive +42,male,non-angina pain,130,180,false,normal,150,no,0,upsloping,0,normal,Negative +61,male,asymptomatic,140,207,false,probable or definite left ventricular hypertrophy,138,yes,1.9,upsloping,1,reversible defect,Positive +66,male,asymptomatic,160,228,false,probable or definite left ventricular hypertrophy,138,no,2.3,upsloping,0,fixed defect,Negative +46,male,asymptomatic,140,311,false,normal,120,yes,1.8,flat,2,reversible defect,Positive +71,female,asymptomatic,112,149,false,normal,125,no,1.6,flat,0,normal,Negative +59,male,typical angina,134,204,false,normal,162,no,0.8,upsloping,2,normal,Positive +64,male,typical angina,170,227,false,probable or definite left ventricular hypertrophy,155,no,0.6,flat,0,reversible defect,Negative +66,female,non-angina pain,146,278,false,probable or definite left ventricular hypertrophy,152,no,0,flat,1,normal,Negative +39,female,non-angina pain,138,220,false,normal,152,no,0,flat,0,normal,Negative +57,male,atypical angina,154,232,false,probable or definite left ventricular hypertrophy,164,no,0,upsloping,1,normal,Positive +58,female,asymptomatic,130,197,false,normal,131,no,0.6,flat,0,normal,Negative +57,male,asymptomatic,110,335,false,normal,143,yes,3,flat,1,reversible defect,Positive +47,male,non-angina pain,130,253,false,normal,179,no,0,upsloping,0,normal,Negative +55,female,asymptomatic,128,205,false,ST-T wave abnormality,130,yes,2,flat,1,reversible defect,Positive +35,male,atypical angina,122,192,false,normal,174,no,0,upsloping,0,normal,Negative +61,male,asymptomatic,148,203,false,normal,161,no,0,upsloping,1,reversible defect,Positive +58,male,asymptomatic,114,318,false,ST-T wave abnormality,140,no,4.4,downsloping,3,fixed defect,Positive +58,female,asymptomatic,170,225,true,probable or definite left ventricular hypertrophy,146,yes,2.8,flat,2,fixed defect,Positive +58,male,atypical angina,125,220,false,normal,144,no,0.4,flat,,reversible defect,Negative +56,male,atypical angina,130,221,false,probable or definite left ventricular hypertrophy,163,no,0,upsloping,0,reversible defect,Negative +56,male,atypical angina,120,240,false,normal,169,no,0,downsloping,0,normal,Negative +67,male,non-angina pain,152,212,false,probable or definite left ventricular hypertrophy,150,no,0.8,flat,0,reversible defect,Positive +55,female,atypical angina,132,342,false,normal,166,no,1.2,upsloping,0,normal,Negative +44,male,asymptomatic,120,169,false,normal,144,yes,2.8,downsloping,0,fixed defect,Positive +63,male,asymptomatic,140,187,false,probable or definite left ventricular hypertrophy,144,yes,4,upsloping,2,reversible defect,Positive +63,female,asymptomatic,124,197,false,normal,136,yes,0,flat,0,normal,Positive +41,male,atypical angina,120,157,false,normal,182,no,0,upsloping,0,normal,Negative +59,male,asymptomatic,164,176,true,probable or definite left ventricular hypertrophy,90,no,1,flat,2,fixed defect,Positive +57,female,asymptomatic,140,241,false,normal,123,yes,0.2,flat,0,reversible defect,Positive +45,male,typical angina,110,264,false,normal,132,no,1.2,flat,0,reversible defect,Positive +68,male,asymptomatic,144,193,true,normal,141,no,3.4,flat,2,reversible defect,Positive +57,male,asymptomatic,130,131,false,normal,115,yes,1.2,flat,1,reversible defect,Positive +57,female,atypical angina,130,236,false,probable or definite left ventricular hypertrophy,174,no,0,flat,1,normal,Positive +38,male,non-angina pain,138,175,false,normal,173,no,0,upsloping,,normal,Negative diff --git a/languages/python/examples/basic/heart_disease.modelfox b/languages/python/examples/basic/heart_disease.modelfox new file mode 100644 index 0000000000000000000000000000000000000000..0ad475fd5f9560a661b8174b9c7062aec82b2ec0 GIT binary patch literal 29071 zcmdUY34D!5_y2Q4tx0XMZy}c06A6Ok`$VqoRZXp>gorGJNFrh@)s&*KmRd?%ic+++ zl)fnHuGu4rkXWK(Clp2P^`GatXP!IB`)hxFd;7ls|39D4bJ<74$$7T- zZSUz$iH@EfJGS%e+Rn>sh^J4tjy~Nyz1w%~;@Q==Qzu{VHlCi{JUuIrXi&ts(1fUI z5j;I0c1o0zu1L?rqe92Uu)HZcse)9NXA&(#G;~}yF zG7SV7w?GalNzDSe4wB>%SY9jJR^UAa8J|Gj21)V_Hu|{8d7~tk-yF*7a4TI&4g~lG;gLQYWdajT?mj|l7%u_16p*sDDxDS+jTVV?v|{c#a7;Suq?lOiJ~g+@p4 zA*Apet|?VKjA%2WhlrNdROTHrmY#6Zj~I@~<-*2COi2id3yq3#CW+7~GbhC*#7<)K zB#d`sXu{06sIbszH#TLXVnTU#Ol(X$TaKH8zz0c5VcwoGgg>8XFwr(dhZ5~iG>B+T zqL+#CHd9g>LkzE{7Nz4^!=hrwg^Z1kjSUZpW63E~<0CroV$xw@f~E)qBtyv0)&_+n ziSiXe=JPmr^4iWF5pXg z`-Il%?as+bZ^tjIw>y|}db{%|FYp!gcIQ+{z;xC((AxN_AX8P($Ce~j6L6{v`K|(A zOK*2Jbp-i(g3k2?zJb6u5_mT~AIr>%3i-_izO|l@4oPY&`^w4P$ZlPmiV&j?fi%OWu9iuJ7i$LPyW5VJfN{Dcaj)+Wfn-&q15FZsbH99ojZT!r*i1>u~*tqdC`8bX)rBw)}7ZD95%01MZ zWS^sy&&dE{IBso9N6d(b4~v=-5i%)sM#%Vx(D;OqcxLancsXgAFqOX*_w-*4mu7c4cLn@k1o;Po zyDjP7>}8({uJO4b|C@j#X;V{@O1e54m(22IgsCjLy;p>?q(^cTR34C{f_Y-&se4xPVHU52C(?;3{ zvg$>+FVPLv9xP+-U$6gubWMFrkC*z74(#i}4kc_ksKY+Z_5b3VqGH0QhDC&j*!FGQ zFT})hXErlp3eRjvj_Z4(?nFltokmKAQ<{?iVmJ=pk!k*s?Ks0D)E!w=Y)o@rj8Wda z%!~|wzC+2(=nuZ5S-)-_xS~u3T!s~u`C_(yU0s>)Wm#E`kBp3zd09zHl=Gh>O& z-44T-tC2pDk=wL`R*e!@~X_Ip11dXO@_e*@F^1+SuTLrm6_oj z!1eL=t`Wek#q`(m*}+)LkBn>)z}x8U?H$0~%ldU)HB44j03R5v?OOi&b#DonW3+r` z6HVURdy1CN>?72>K;xMmG?~aqtHyhiF>mt5kqvp@>aGY6l!Bdx%#**qzD*v{dZnz$ zDjlU5lN( zqW!2PNtEWK^qSB2X_3)WW8-6|gvG|q44KA`7!lD^Lc(IBW8)*jxqb@7XB>wOJU@+> z0-9=kKtK)R6X_fkMrqy@j;G_Z#P&QOp#S&|^7fQ4`R3b`Rb$%n6J=KdVB5Xu=%}fa zLZ&1rr=XAow&&zCBr>KBg(ej;$0CRF)bo~N|dtUvWanh^aRh3JiP8T zl;4AB6w$5JuC4%5mBC=L+LHlm{3Y{5c;nrnA zrWRc04}wgjKa(sOWa4m=J&PoH3L7sUAqeCvYy|xTK0r8=mJBlS0V_%U zg|qMgA&@gfC_7w`2^KbdC4)@7J0xj*ARoh$6fN*^!nwF)kcs!LB+VAul_=;_GRVZc zLXs8;G9L(SUnJOeiI6XD^zDJAx5x=yvUi%=!@Qq$_p}GL@KNo9BLd&AyfM&!_umsf z3ZMbijOc2jGATBk(wqbk!v_;z>i+vdqsgQR4;nS1^iZPQtp`z>JFmLrR2>T%wIfui zRM3dqMUs9$Xry(iRPsb-pK#}d!Fxd%m{*0tRq}+z9kmiu;0uuwG_u!K@>FG?h);!q zTry~6pIA+5QE`~5(XUANy46xnsFBkbh=TyV&a z|NFrq`$Ye9!6Cby?ZKhWw9D;h9~5fq`Pk9*(jM4?Lp40h=s#?;RR4bTf*$`%aA+lA z+xyyP*hI~~8ePX~ue284LWNzzrp z<=ho`F~IX!$bTxV_s<3QCI)p%26F6Evy5;MFDnEF91G;wZRjkxG#8<+l7Sq%J|zP= zcAHceTvQDK^Y;Te_NAa?Ajh6xGLU0;X@4$|W0(K^K#pCesSr?c7lKJ8-KX6SVj!nv z@WzfQ#^2ir_Gv5Fr=4K)_JT|YA-`ns#*XhT;E2JS&O-j53DW%UftiN%@+04F0M#Mz zd92IP9@qjiLHe5d(1p+S`%=H_AxV|=l2n&{-2axq%o6fN?nM7Vvvdh z^YxPlTKKy99knEhMiwWfqXB%+^Ctr>GdK<#NbNOV3gEF!B|^pn7B?E(1j0y;4?;dz z{w2_2ncyqCM#ac)J5N`QX)E_%3$$?4^13IH-aUwpAo{ACjp2b7J|s$;mgDKx(KK_a z$QfUHrpu>Us^bx9r~HaGbSEcbbgne|o%rj(|4N{Rx88$VJ)WpD&4o}(a}q!d?`1v( zS)j!}I9!FzN&{ho;wI#`74kiWL%)w8<15Jc34DMc(^n|lUyvCfY?6iu`NM_$U?D$T z2-u7lFr$UK;skz%PElQpg z_&jBe5_W?nPX~PaAxYvZ-16FMH04d0Hmum@%Z}+RU{(`kY6-9S>Im}UtrR!m^wC-< z>!Zo*DWo^TUb)&g+?Kkb+Jh7}sD1R(u~+m-pYPM}b@`YLc79W&*8g-<#EW{lRC9VJe4lp&L`5a1= z2Ve7A*)kjqFxfZ3#{@ThLEtY7ycl!-b2of6M<$6ke8l+kQvv_Euq24Lcbv4LsNC4G zW0n>8$^!2!@Gb&hrG#782CYpy!)i8|&A*v*rG)_+TYqczP5+M-E3RD73|45#Ou1Q5ej z{Nn&e9bSx4zK2o*9Azk932<;;32<=JDFF_i&qTFbB1(Y6F4JENYA68?p05NrIA8ka zi5>HpR#pjW@O&kx!OJQ^4Le^$3p6OVM0o3!zyy~OZ;7yW3Hfhn^2%)xE-&8hs74oD zgZM+T54WH0AHoBrU|4Fboa3G;zo@iXJ*)b7e+O@G(|_)E2M@pSm4f#>4>~jUF}ldd&L^Qc{aI_OZ1EamhbooAfBr5E}-9wyX%OdEWsVp{W^Y7wL3 zmpYuB$#VadpaXwZPdg8Z5487arxaSE^VkreG@p`{*%vjOC0D6|)J$qCd9v@8bdzLB z&%SNK{}RD5d0qSor-y&|0Xp>`)tT%VRgv{gqWTHpGR!O4SsTaUIDFHlO%)Gv?4>#a zRgM?g#u+KV5g0yn3JGolQLd$nTCJfeK-sJM6TEWWn1>{5>P zM@7j+swUNynn--^GQAR&>uCPhsR@Y$qw7md8CI^ERFnNnf8B}R-2INc@5e-}rgZ5F zCsV#!MWuXqzkjKs8%cx_mAr|WnmF z*QiHIL${=S75Wdgu1k_7wB8>1wY=?jpsT1oP)Vx6mf-qQW66Wf3STyFdPx6}UY7<+ zZoE}GWotoy%4m1%p=hb=NLc|c)K@gsxLsfJ=!qg1TYVn0fm@Fsd3#1v>AJtnlKtv$ zM}I0^+5mh;=yW=FDx!!urUkzdpyOKPjhZEYn3IIu-_fEiSKF?IZRWf6P&(zW*&>}P z)3o@a-aGQmO&`m{$)KfevG-rK;I{bdS`4JhG%XZcEJB)grFH@d)Fx^swI>rjL0a+1 zNkdEYzJk9fcQ>FBe7;G?@-s^EHy*6OoP`{w$~ z*AE;~%DnmG2*0q0u1D{TF7WT!`}3np-qCB}M^iHYfnJM$RpmE!neRWlSr|MfeSWlO zKo5CNgG&D2jvnjZ`(?);U&hsx-IPM??~Bn5RLpl8u0h~EAJqXa2(K)AZ1~mxi>I>? z+V^|^Q=#*I*cbi%VMV^YWw3mu*i)@z!t$@WFaJuCzj*o#!T;Q^KF>beOlD=U`beSb ziHg%kDLd0@3e>2PJ`Nu@si)@Izh!+5)6OG&kvDWvA^(T z3*Kw_R0khM+@O4zXmynMGQ}jCczl@f*T2U zZY$czw;@RWwv{j7|KV%wU8q_wqQgm@cPY(V!r#8=a!9~wHfPRK<8`|^vxFEYH_q^H z*J$?vo%oj~bVq0oIrSn&SA*v)B_;T-rb4`zT-o1KUi)eXdH2#4 zbIwJ6J;X)+ZDk<*kZx+q)HHm!t~v zHunnhjUUU)d85k9!(Nn=CvPYx*XU7B_Pp|noVMr{InV1Ax!0Mpa-H|f%7Goq%16$Y zkrR{3$cuc-$fvG5$^F(l$@ls@$!5t(KD1var%u$#_nYeE)@LPo$|^}d5+uo$DoOI@ zB`?ve{7cN5_Z#-=euMeF7g$~91zIe8j)~6CG2)|VxL^MnI&OQ42VPGR^y3q>4|;+{ z=O1H4++%Di^BDa;euS~@A0fl?D^`yF6&oHuM443&vBC2pW@SCV+DQ*^&gB7yd~+Xn z2j0hlhri&X^}k?Iz%N*L?H=|lzlQ?fdnkYDF4`}0bJ)+ws`xVo9J!4>lW(K2#cf&byVAO9epCMqprtw%rCx%wclKWPr@~f z>v|2HUtYx;%T>%@aup8;Uxlv0RV=%F1v_?KLE!W&i0W|#6`Zf2b^c{!ZN7}+n9F## z`(-Sva2X9xT|(n8*bXHA5@z{fNcr|6CcbwOvj$zn1NV!_eprOsDMhfX zFT$lsMHtYd2yQitF!II)JUVg#H&$Li_wg5S%b$H}T)>cRT!RHau`8+Y)29$~ zs}`c#?Q^)CdJfOFokP(3=kOr(9NT-cyoTq{@A+9=J98G6gJ*H!!c=(qk1=FU2U;@~q_fHN5Db_VfIXK>*1X>>?FjlvzLacs$H zoQ^$>x`R(+edp8YQ|~ld{#Jl47Yi`WT7b(t3ovha0bWZeK)_oCn2rKux))$@#R9zi zIUi^8@=-6QPT-BZ zxyUTc#l-YnG&q=x+uL$cuqqchb8~S$E*EYgxd?kb7l!V+KgIe22sLG8UckT>OE>4!Neem4gpu{pRjItL4aa^Qm;JZqPOf`&QBa>+sNi)>uG zosEX)vk{S%jr~Wn;r>lFHf+p>&xhG4Ow7je$=L`Sn~lK1+3@a}jn192F}OuG=G4x{ z_Z8S_;^}cj6(7f$bH@>!bsP_WJdV%y9LJz7$Kk&EIG!vxjvF(MU0E3UIs1H+g>ehA zFmq-Wc0^~PFeD4D2WMeU?<`#EnuS5_vXJSPh0(5Acu_tJo1bN3#GOnuy_AXjr!sLe zEfdzGnK-hS(QTP1T%UkKs8l!0MuGO&Dc22Ler zpvANd%$t~j8)Gss^34pKc`XBDdt~5o=L~$*HUr(7X5dck4D4{pz?3o>ke{T(?M^zJ zucYJQnRGnLPRCcE~eq@(`ne5 zordknY1n%-4d(r6IP-NH%72jtk5AGNvN8=DlG0F=n1(hp(l9SJ4d)}%(06ni4h&^7 zucu*iPxj1@;d!T_Uz;>sX_|(Q>ZPGe^)%e8n1&s?G)#Vy3g3IFaJi9+%NJ5%E=a}K z*{N8Yl8QCQQnBJdDpr4+ip|?ona@r|&c~^E_+csQ&9Igo<(ds0wmX9}urNkQ39Q}A+C3Y?dvpyt99 zcqFDko|%H-@hO--F$EjLQ;;z#1!ab%pxb~Hyz`F~91Bc=i(d+cbxOhB_9-auk%Dne zQjk|K1-`B+*yEDI*5DMZlTzUJG#Oj(C!^!-WEid{W5k7I+&`U+wI`C%Ju?|MtjSn? zEE)X{CF7NS$uRFqM$(tb7`!PNZl5IM&Z=ajElb9p50bHHZZcwKCu8)qWCX<}BXB}8 z{K6T|Xhw%8qyL~}4C|kasNTtVHy|17yCvhWPckmIPe%Dx$?$bgMr^}mY^t4%Gu4vO z*qLFKOU5oK8IPY@(fgqlYwuX`=!O+TE?aTvycG>kTe0+n6;Cp)n3!TkvC)dLN36K^ zPb((vwc^3oRxJF|ifWs!*!HOvSZl?F6;`~r#0rmvR%E|t#jM#@cu%+D`eZ9MMO!g? zoE1&ST5&triao=um@~+V0k2!(-q#A9-iiW$R=2Me8@;WV-`WfmMrVrAxA zuprTbyR$49J>7zgcnbnzEZ7%ifp>%jd&XGMHki?27IYbGfjP*6e*G*s*UN&5JuHy? zE!g18p7~gCs)GeH+gRY%(gJH!3t}2s(6F8bxiu|VSj~d&E*3njz_7|%Fio<+=eZe= zADLmiZ^rUFW{fB{qxCg2US2fg)Oj;@pD|@wr<4l@pJHDmuKGmd`7KI_=C)n??bFr#p(8F!M*&@C{d=3Fz{&oQIV z+h#;fV|B!vu_K1bPB7zcq#5s9biWP{$|YYYlfkx8D)By z(c9mQd0m+fofu|EGuvO9v8%Niyo-;B|9%sAv~M%`*=%yeP8R5T-? z9IMmG4A++?%zb9U-A5*bJ}{x+t_eZ6O)%du!SAXG2QHe>?I%XhGFo7QC67JNF(Eh8 zgpp|`+_IW5$7DjaVkYQKi7zE$Jz4?Ba%{$IAAs6jL8URgAtvN8WHt_kw8V&0Nk%kX$jZKNgz-Hi z>b`5lgxN+MnaTL+M#Lu=VV!J5i&&;-v=K#7Mqr!~8^T$=V;N?M5oy7!JtK_RILwHO zLyed`$cPJX81csIMi}}T;n~-St-XwJ)f=%mz=*P7SlwAWeHngdBW`&cG0}_Js6C@? zjkwc_>C@7PN6n0Qr>PMy8XK{+AuC^>wWp2|?Q0ovw1yFbsxf_DHDZ#p5xPo7tSxUu zhgX=sPDX@CMm&FEz=~%Ebbew$)~^Okd|*J;UkuoH$AH1N4S00ZfQ{D;tRD=xeaV22 ziVTo{GT_=d13o#!@(T=ja?*gWPZ%&P#{ie(23Rr-n3-mPXNm#WEe32d88FgdK+PWw z$UAJn%0mqMfB`kWH{j%V2CUy>!02xbXtB$H8#@iyyPcKaWfHNBn*#4OT zlRq(__c~T?tpRRyUN0K(k)OiO7(CnG??yCxVIWEOZYTS9G&}o!k~PPF&61~nmk(|B z8@SA%%B|e(>eqGHAeFyz@^QBh;&=ERZ!*pAr-7Z-vcGvb`5#<<#V@Ga5;Z-0QEk5= zUON8|K}rx9sTcs`Rv`qJ<=75zVP;OY24v zqsFCK-YyF6DN%hMcH)60TM_%efACWBfd;chgK17BDZZiM^kjZd7j*Nea@T@gl*nYE z0fiZ@pDA~5Xp>rQe5k=RqkO(jQ$!-BPz%v-5QQ^M`Y1w1q3sqOC1&Fe!YHj zRVuUGI8}q;p)wxY^LL{hM!oFv>G=l)vu$g06~^P6Ch`@6DIR!0 zgW;y8A;wW8dJC;yqG28W)y~!`%#}{Aa@)o>sSNg_jTh(i-+-AyFuVH{X)rYM*(=*taEjbGI1a<^@31ML^t!4|^sY`S%#j7ZpfAC+3a;V^%yNQRx2~=uFsBHH z%_&D<8aA=De3ol_@E$5{8)gdi@6ONdW!;8?in6Zus03NpQtv4%jAEN`f+-&8tHJP& z=CB+_HL=pYHW($Tz|3^Hg0BcBczK{BFuxECa}I!zTfs6ueQ>r+1Nc z4f?ZbuG;3PUuU&Zomb@}kzjWB33UV}nP3{tpQyp`i$MG(2-oKD58Y+m(4jR+n|h&L zRc-FGlkltVw$5bUPlMrphWi~3!|u|^I@Rf@jrwMjp9=HMro$LYFl?4M0`n=s9B4G( z5tvH^6X(AOd`9zk2K@fN6OCfEQE4se%eoGS#>={;?Yqee6YcR0TGQIbT)GCsT{(B{ z9HwW_o{EofBbW}G{8bo_Z#H2v!4$o{3iMkT5?@&OLQ{scQDIbv+`INDS+}=FcUfVs zbXtW21hbA^gLDAqCBY0ju)z_S-n4yS>y!plks3kkvcwyueBn`d_WO3SPHhyM&1$3A zRsc&0#`ti%2E*5GTEaA#YSjkHy4%ft2BXE*bPri!6m8}Z4BJsS0%IYV`NcVoz&O(@UH?k?j=&5in1>$c z9D!L+Fl?IyzOZnUHX+k;ldC57a@NbbCCp>#p3a4=Fl>K=B7$Lel^uccp!dm*<`+8x z6H74pTkbjn^F6_^y`Kid2S2TZ61R;y8C0jE+NLA3lESdX*Z&2<^eK9(!SEUFUdqJk z3}@bjOze`?M}=W7-M=Tl70-?pyr=#G%wmEGUhd=wOfJE&4Xq+bT7z>Fc7 zIR8qHzIs#)N82?JOWxnSttqouP)KPq~C{)&MS z0yB_ce)VYJ2+YR>Gw47gM__&;nB9GvI0DmxZm+TPlp`>c2u3;8I?z-52*&u(!x5Ng z1k-4K8|_OHrN>PL@`ALZ7$fTrU1zt{mCe4wDB~uugH47V5*>j_BAEOwo{qp|6O3|j ze3g{{3yrGnX)C+j*vS!?(FDUzzK+0bC73=%-LzlHE#0V!)L3;ZpgNtLUhIm{pv&H} z!YDm;i(tNp^VfdUr8Eq`o~bUF|C}^Z*4@}#*%k&;7zM_gU&vwyKz;!F3ovgJjIwLy zF#K#u8+eIpqc+MC6iwPZS=2$cX_N0~AeaMFi+NfiX&>o)FABc5>r$r!u2Y+#yZiIcICOop+EX_PhK0x+f${2Wv&}5m z7)M}c5KR7-I1PqIn?y%+i8o3;$*U1(r7gNEk}24S=>+iax0b6w5dTEYM@tzVJ;ZD4_T;CVU(w@6HM{I8LAuM z5DMkm)TB%`vd^{g*Y&Km*Pm(QuRL8vFnx;Nb_C`u!EA|p#}Sw?QurN;&VlGl3i=%* z`X13hqGgDhDaJUTXm6tBiKb9sHHm0HqLqne)V3+GlxPsqDnzp>+Ps43Afh#h=8?r$ z6Md6tZK4Gf6J1Ai1kw6L&r!#JMszgM#zZgB_}WBtEKzr&mnlB9m1rcDICq8*6drET6GqVYt%i9R4NzMtqcqFsnSrp4kQ(YJ|qC;FUL$RkAO z5Y^T&N*idiGM6a-?>EX)+CnaH0nt80D-cbkz4juauMu@7nn`E4WklZ~T9s%H1w2*~ z9Zb}f=t&A5tsy#$XdR-bDS)z`DF2@>8c@2BLK+*0h7b+xP7}nj=trk+Hkj6PqU6DZ zIE$*Np}jQcUq8XlJJmt4|2Aq2VO%D)rP^I=5~Q*uou^RqX) -> PyResult { + let mut handle_progress_event = |_progress_event| {}; + let input = match input { + FileInput::Train(file) => { + modelfox_core::train::TrainingDataSource::File(file.into()) + } + FileInput::TrainAndTest((file_train, file_test)) => { + modelfox_core::train::TrainingDataSource::TrainAndTest { + train: file_train.into(), + test: file_test.into(), + } + } + }; + // Load the dataset, compute stats, and prepare for training. + let mut trainer = modelfox_core::train::Trainer::prepare( + modelfox_id::Id::generate(), + input, + &target, + config.map(PathBuf::from).as_deref(), + &mut handle_progress_event, + ).map_err(ModelFoxError)?; + let kill_chip = modelfox_kill_chip::KillChip::new(); + let train_grid_item_outputs = trainer.train_grid(&kill_chip, &mut handle_progress_event).map_err(ModelFoxError)?; + let model = + trainer.test_and_assemble_model(train_grid_item_outputs, &mut handle_progress_event).map_err(ModelFoxError)?; + + // Write the model to the output path. + let output_path = PathBuf::from(output.clone()); + model.to_path(&output_path).map_err(ModelFoxError)?; + + // Announce that everything worked! + eprintln!("Your model was written to {}.", output_path.display()); + eprintln!( + "For help making predictions in your code, read the docs at https://www.modelfox.dev/docs." + ); + eprintln!( + "To learn more about how your model works and set up production monitoring, run `modelfox app`." + ); + + // TODO: load the model more efficiently + Model::from_path(cls, output, None) + } + /** Make a prediction! @@ -306,6 +367,12 @@ impl LoadModelOptions { } } +#[derive(FromPyObject)] +enum FileInput { + Train(String), + TrainAndTest((String, String)), +} + #[derive(FromPyObject)] enum PredictInputSingleOrMultiple { Single(PredictInput), diff --git a/languages/python/modelfox/tangram_python.pyi b/languages/python/modelfox/tangram_python.pyi index b945044f..731ff469 100644 --- a/languages/python/modelfox/tangram_python.pyi +++ b/languages/python/modelfox/tangram_python.pyi @@ -1,6 +1,5 @@ from typing import ( Any, - cast, Dict, List, Literal, @@ -26,6 +25,14 @@ class Model: ) -> "Model": ... @property def id(self) -> str: ... + @classmethod + def train( + cls, + input: Union[str, Tuple[str, str]], + target: str, + output: str, + config: Optional[str] = None, + ) -> "Model": ... @overload def predict( self, From 1f3af0c0485c228cf96d1150e70bfd6049be8a07 Mon Sep 17 00:00:00 2001 From: Chuxiao Feng Date: Sat, 8 Oct 2022 23:09:10 -0500 Subject: [PATCH 2/3] Arrow array support --- Cargo.lock | 71 ++++++++++++++ Cargo.toml | 1 + crates/cli/train.rs | 8 +- crates/core/Cargo.toml | 1 + crates/core/train.rs | 113 +++++++++++++--------- crates/table/Cargo.toml | 1 + crates/table/load.rs | 21 +++++ languages/python/Cargo.toml | 1 + languages/python/Makefile | 2 +- languages/python/examples/basic/train.py | 21 ++++- languages/python/lib.rs | 114 ++++++++++++++--------- 11 files changed, 259 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e516fa7e..306a939c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,24 @@ dependencies = [ "backtrace", ] +[[package]] +name = "arrow2" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467e88c67d518f9992bb1f6c8faa202eaf93b0ce244889d241c9c44d0df0ab46" +dependencies = [ + "ahash", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "ethnum", + "foreign_vec", + "hash_hasher", + "num-traits", + "simdutf8", +] + [[package]] name = "async-trait" version = "0.1.56" @@ -253,6 +271,26 @@ version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +[[package]] +name = "bytemuck" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5377c8865e74a160d21f29c2d40669f53286db6eab59b88540cbb12ffc8b835" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd2f4180c5721da6335cc9e9061cce522b87a35e51cc57636d28d22a9863c80" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -671,6 +709,12 @@ dependencies = [ "shared_child", ] +[[package]] +name = "dyn-clone" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f94fa09c2aeea5b8839e414b7b841bf429fd25b9c522116ac97ee87856d88b2" + [[package]] name = "either" version = "1.7.0" @@ -741,6 +785,12 @@ dependencies = [ "erl_nif_macro", ] +[[package]] +name = "ethnum" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac3c0b9fa6eb75255ebb42c0ba3e2210d102a66d2795afef6fed668f373311" + [[package]] name = "event-listener" version = "2.5.2" @@ -802,6 +852,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1035,6 +1091,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "hash_hasher" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" + [[package]] name = "hashbrown" version = "0.11.2" @@ -3060,6 +3122,7 @@ name = "modelfox_core" version = "0.8.0" dependencies = [ "anyhow", + "arrow2", "bitvec", "buffalo", "chrono", @@ -3270,6 +3333,7 @@ name = "modelfox_python" version = "0.8.0" dependencies = [ "anyhow", + "arrow2", "chrono", "memmap", "modelfox_core", @@ -3301,6 +3365,7 @@ name = "modelfox_table" version = "0.8.0" dependencies = [ "anyhow", + "arrow2", "csv", "fast-float", "fnv", @@ -4930,6 +4995,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "similar" version = "2.1.0" diff --git a/Cargo.toml b/Cargo.toml index 9412e23c..591f1aed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ version = "0.8.0" [workspace.dependencies] anyhow = { version = "1.0", features = ["backtrace"] } +arrow2 = { version = "0.14" } backtrace = "0.3" base64 = "0.13" bitvec = "1.0" diff --git a/crates/cli/train.rs b/crates/cli/train.rs index 9b94323f..23b45edc 100644 --- a/crates/cli/train.rs +++ b/crates/cli/train.rs @@ -53,12 +53,14 @@ pub fn train(args: TrainArgs) -> Result<()> { let input = match (&args.file, &args.file_train, &args.file_test, args.stdin) { (None, None, None, true) => modelfox_core::train::TrainingDataSource::Stdin, (Some(file_path), None, None, false) => { - modelfox_core::train::TrainingDataSource::File(file_path.to_owned()) + modelfox_core::train::TrainingDataSource::Train( + modelfox_core::train::FileOrArrow::File(file_path.to_owned()), + ) } (None, Some(file_path_train), Some(file_path_test), false) => { modelfox_core::train::TrainingDataSource::TrainAndTest { - train: file_path_train.to_owned(), - test: file_path_test.to_owned(), + train: modelfox_core::train::FileOrArrow::File(file_path_train.to_owned()), + test: modelfox_core::train::FileOrArrow::File(file_path_test.to_owned()), } } _ => bail!("Must use the stdin flag or provide training data files."), diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 48deecdc..bd15f6b6 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -18,6 +18,7 @@ path = "lib.rs" [dependencies] anyhow = { workspace = true } +arrow2 = { workspace = true } bitvec = { workspace = true } buffalo = { workspace = true } chrono = { workspace = true } diff --git a/crates/core/train.rs b/crates/core/train.rs index a503b54b..dcc29669 100644 --- a/crates/core/train.rs +++ b/crates/core/train.rs @@ -18,6 +18,7 @@ use crate::{ test, }; use anyhow::{anyhow, bail, Result}; +use arrow2::ffi::{ArrowArray, ArrowSchema}; use modelfox_id::Id; use modelfox_kill_chip::KillChip; use modelfox_progress_counter::ProgressCounter; @@ -35,12 +36,17 @@ use std::{ unreachable, }; +pub enum FileOrArrow { + File(std::path::PathBuf), + Arrow(*const ArrowArray, *const ArrowSchema), +} + pub enum TrainingDataSource { Stdin, - File(std::path::PathBuf), + Train(FileOrArrow), TrainAndTest { - train: std::path::PathBuf, - test: std::path::PathBuf, + train: FileOrArrow, + test: FileOrArrow, }, } @@ -82,7 +88,7 @@ impl Trainer { target_column_name, handle_progress_event, )?), - TrainingDataSource::File(file_path) => Dataset::Train(load_and_shuffle_dataset_train( + TrainingDataSource::Train(file_path) => Dataset::Train(load_and_shuffle_dataset_train( &file_path, &config, target_column_name, @@ -729,25 +735,31 @@ fn load_and_shuffle_dataset_stdin( } fn load_and_shuffle_dataset_train( - file_path: &Path, + file_path: &FileOrArrow, config: &Config, target_column_name: &str, handle_progress_event: &mut dyn FnMut(ProgressEvent), ) -> Result { + let mut handle_progress_event_inner = |progress_event| { + handle_progress_event(ProgressEvent::Load(LoadProgressEvent::Train( + progress_event, + ))) + }; // Get the column types from the config, if set. - let mut table = Table::from_path( - file_path, - modelfox_table::FromCsvOptions { - column_types: column_types_from_config(config), - infer_options: Default::default(), - ..Default::default() - }, - &mut |progress_event| { - handle_progress_event(ProgressEvent::Load(LoadProgressEvent::Train( - progress_event, - ))) - }, - )?; + let mut table = match file_path { + FileOrArrow::File(file_path) => Table::from_path( + file_path, + modelfox_table::FromCsvOptions { + column_types: column_types_from_config(config), + infer_options: Default::default(), + ..Default::default() + }, + &mut handle_progress_event_inner, + )?, + FileOrArrow::Arrow(array_ptr, schema_ptr) => { + Table::from_arrow(*array_ptr, *schema_ptr, &mut handle_progress_event_inner)? + } + }; // Drop any rows with invalid data in the target column drop_invalid_target_rows(&mut table, target_column_name, handle_progress_event); // Shuffle the table if enabled. @@ -761,27 +773,35 @@ fn load_and_shuffle_dataset_train( } fn load_and_shuffle_dataset_train_and_test( - file_path_train: &Path, - file_path_test: &Path, + file_path_train: &FileOrArrow, + file_path_test: &FileOrArrow, config: &Config, target_column_name: &str, handle_progress_event: &mut dyn FnMut(ProgressEvent), ) -> Result { + let mut handle_progress_event_inner = |progress_event| { + handle_progress_event(ProgressEvent::Load(LoadProgressEvent::Train( + progress_event, + ))) + }; // Get the column types from the config, if set. let column_types = column_types_from_config(config); - let mut table_train = Table::from_path( - file_path_train, - modelfox_table::FromCsvOptions { - column_types, - infer_options: Default::default(), - ..Default::default() - }, - &mut |progress_event| { - handle_progress_event(ProgressEvent::Load(LoadProgressEvent::Train( - progress_event, - ))) - }, - )?; + let mut table_train = match file_path_train { + FileOrArrow::File(file_path_train) => Table::from_path( + file_path_train, + modelfox_table::FromCsvOptions { + column_types, + infer_options: Default::default(), + ..Default::default() + }, + &mut handle_progress_event_inner, + )?, + FileOrArrow::Arrow(array_ptr_train, schema_ptr_train) => Table::from_arrow( + *array_ptr_train, + *schema_ptr_train, + &mut handle_progress_event_inner, + )?, + }; // Force the column types for table_test to be the same as table_train. let column_types = table_train .columns() @@ -802,17 +822,22 @@ fn load_and_shuffle_dataset_train_and_test( TableColumn::Text(column) => (column.name().to_owned().unwrap(), TableColumnType::Text), }) .collect(); - let mut table_test = Table::from_path( - file_path_test, - modelfox_table::FromCsvOptions { - column_types: Some(column_types), - infer_options: Default::default(), - ..Default::default() - }, - &mut |progress_event| { - handle_progress_event(ProgressEvent::Load(LoadProgressEvent::Test(progress_event))) - }, - )?; + let mut table_test = match file_path_test { + FileOrArrow::File(file_path_test) => Table::from_path( + file_path_test, + modelfox_table::FromCsvOptions { + column_types: Some(column_types), + infer_options: Default::default(), + ..Default::default() + }, + &mut handle_progress_event_inner, + )?, + FileOrArrow::Arrow(array_ptr_test, schema_ptr_test) => Table::from_arrow( + *array_ptr_test, + *schema_ptr_test, + &mut handle_progress_event_inner, + )?, + }; if table_train.columns().len() != table_test.columns().len() { bail!("Training data and test data must contain the same number of columns.") } diff --git a/crates/table/Cargo.toml b/crates/table/Cargo.toml index aeee218e..796bfa7b 100644 --- a/crates/table/Cargo.toml +++ b/crates/table/Cargo.toml @@ -21,6 +21,7 @@ insta = { workspace = true } [dependencies] anyhow = { workspace = true } +arrow2 = { workspace = true } csv = { workspace = true } fast-float = { workspace = true } fnv = { workspace = true } diff --git a/crates/table/load.rs b/crates/table/load.rs index efc5f896..a03160f5 100644 --- a/crates/table/load.rs +++ b/crates/table/load.rs @@ -1,5 +1,6 @@ use super::{Table, TableColumn, TableColumnType}; use anyhow::Result; +use arrow2::ffi; use modelfox_progress_counter::ProgressCounter; use modelfox_zip::zip; // NOTE - this import is actually used, false positive with the lint. @@ -244,6 +245,26 @@ impl Table { handle_progress_event(ProgressEvent::LoadDone); Ok(table) } + + pub fn from_arrow( + array_ptr: *const ffi::ArrowArray, + schema_ptr: *const ffi::ArrowSchema, + handle_progress_event: &mut impl FnMut(ProgressEvent), + ) -> Result { + let array = unsafe { std::ptr::read(array_ptr) }; + let schema = unsafe { std::ptr::read(schema_ptr) }; + + let field = unsafe { ffi::import_field_from_c(&schema)? }; + eprintln!("field = {:?}", field); + let array = unsafe { ffi::import_array_from_c(array, field.data_type)? }; + eprintln!("array = {:?}", array); + + handle_progress_event(ProgressEvent::LoadDone); + let column_names = vec![]; + let column_types = vec![]; + let table = Table::new(column_names, column_types); + Ok(table) + } } #[derive(Clone, Debug)] diff --git a/languages/python/Cargo.toml b/languages/python/Cargo.toml index 61914724..4db11b44 100644 --- a/languages/python/Cargo.toml +++ b/languages/python/Cargo.toml @@ -17,6 +17,7 @@ path = "lib.rs" [dependencies] anyhow = { workspace = true } +arrow2 = { workspace = true } chrono = { workspace = true } memmap = { workspace = true } pyo3 = { workspace = true } diff --git a/languages/python/Makefile b/languages/python/Makefile index 82efffdc..03873f27 100644 --- a/languages/python/Makefile +++ b/languages/python/Makefile @@ -1,7 +1,7 @@ .PHONY: test dev test: dev - .venv/bin/python examples/basic/main.py + .venv/bin/python examples/basic/train.py dev: .venv cargo build -p modelfox_python diff --git a/languages/python/examples/basic/train.py b/languages/python/examples/basic/train.py index dc9fa32a..8c8974af 100644 --- a/languages/python/examples/basic/train.py +++ b/languages/python/examples/basic/train.py @@ -1,12 +1,29 @@ import os +import pyarrow as pa +from pyarrow.cffi import ffi as arrow_c +import polars as pl import modelfox # Get the path to the CSV file. csv_path = os.path.join(os.path.dirname(__file__), "heart_disease.csv") # Get the path to the .modelfox file. model_path = os.path.join(os.path.dirname(__file__), "heart_disease.modelfox") -# Train a model. -model = modelfox.Model.train(csv_path, "diagnosis", model_path) + +# Read the CSV file into a PyArrow. +df = pl.read_csv(csv_path) +arrow = df.to_arrow() +with arrow_c.new("struct ArrowArray*") as c_array, \ + arrow_c.new("struct ArrowSchema*") as c_schema: + c_array_ptr = int(arrow_c.cast("uintptr_t", c_array)) + c_schema_ptr = int(arrow_c.cast("uintptr_t", c_schema)) + + # Export the Array and its schema to the C Data structures. + print(type(arrow[1]), arrow[1]) + arrow[1].combine_chunks()._export_to_c(c_array_ptr) + arrow[1].combine_chunks().type._export_to_c(c_schema_ptr) + + # Train a model. + model = modelfox.Model.train((c_array_ptr, c_schema_ptr), "diagnosis", model_path) # Create an example input matching the schema of the CSV file the model was trained on. Here the data is just hard-coded, but in your application you will probably get this from a database or user input. specimen = { diff --git a/languages/python/lib.rs b/languages/python/lib.rs index 67898df7..30edb1f0 100644 --- a/languages/python/lib.rs +++ b/languages/python/lib.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; +use arrow2::ffi::{ArrowArray, ArrowSchema}; use memmap::Mmap; use pyo3::{prelude::*, type_object::PyTypeObject, types::PyType}; use std::{collections::BTreeMap, path::PathBuf}; @@ -131,53 +132,56 @@ impl Model { #[classmethod] #[args(input, target, output, config = "None")] #[pyo3(text_signature = "(input, target, output, config=None)")] - pub fn train( - cls: &PyType, - input: FileInput, - target: String, - output: String, - config: Option) -> PyResult { - let mut handle_progress_event = |_progress_event| {}; - let input = match input { - FileInput::Train(file) => { - modelfox_core::train::TrainingDataSource::File(file.into()) - } - FileInput::TrainAndTest((file_train, file_test)) => { - modelfox_core::train::TrainingDataSource::TrainAndTest { - train: file_train.into(), - test: file_test.into(), - } - } - }; - // Load the dataset, compute stats, and prepare for training. - let mut trainer = modelfox_core::train::Trainer::prepare( - modelfox_id::Id::generate(), - input, - &target, - config.map(PathBuf::from).as_deref(), - &mut handle_progress_event, - ).map_err(ModelFoxError)?; - let kill_chip = modelfox_kill_chip::KillChip::new(); - let train_grid_item_outputs = trainer.train_grid(&kill_chip, &mut handle_progress_event).map_err(ModelFoxError)?; - let model = - trainer.test_and_assemble_model(train_grid_item_outputs, &mut handle_progress_event).map_err(ModelFoxError)?; - - // Write the model to the output path. - let output_path = PathBuf::from(output.clone()); - model.to_path(&output_path).map_err(ModelFoxError)?; - - // Announce that everything worked! - eprintln!("Your model was written to {}.", output_path.display()); - eprintln!( + pub fn train( + cls: &PyType, + input: Input, + target: String, + output: String, + config: Option, + ) -> PyResult { + let mut handle_progress_event = |_progress_event| {}; + let input = match input { + Input::Train(file) => modelfox_core::train::TrainingDataSource::Train(file.into()), + Input::TrainAndTest((file_train, file_test)) => { + modelfox_core::train::TrainingDataSource::TrainAndTest { + train: file_train.into(), + test: file_test.into(), + } + } + }; + // Load the dataset, compute stats, and prepare for training. + let mut trainer = modelfox_core::train::Trainer::prepare( + modelfox_id::Id::generate(), + input, + &target, + config.map(PathBuf::from).as_deref(), + &mut handle_progress_event, + ) + .map_err(ModelFoxError)?; + let kill_chip = modelfox_kill_chip::KillChip::new(); + let train_grid_item_outputs = trainer + .train_grid(&kill_chip, &mut handle_progress_event) + .map_err(ModelFoxError)?; + let model = trainer + .test_and_assemble_model(train_grid_item_outputs, &mut handle_progress_event) + .map_err(ModelFoxError)?; + + // Write the model to the output path. + let output_path = PathBuf::from(output.clone()); + model.to_path(&output_path).map_err(ModelFoxError)?; + + // Announce that everything worked! + eprintln!("Your model was written to {}.", output_path.display()); + eprintln!( "For help making predictions in your code, read the docs at https://www.modelfox.dev/docs." ); - eprintln!( + eprintln!( "To learn more about how your model works and set up production monitoring, run `modelfox app`." ); - // TODO: load the model more efficiently - Model::from_path(cls, output, None) - } + // TODO: load the model more efficiently + Model::from_path(cls, output, None) + } /** Make a prediction! @@ -368,9 +372,29 @@ impl LoadModelOptions { } #[derive(FromPyObject)] -enum FileInput { - Train(String), - TrainAndTest((String, String)), +enum FileOrArrow { + File(String), + Arrow((usize, usize)), +} + +#[derive(FromPyObject)] +enum Input { + Train(FileOrArrow), + TrainAndTest((FileOrArrow, FileOrArrow)), +} + +impl From for modelfox_core::train::FileOrArrow { + fn from(value: FileOrArrow) -> modelfox_core::train::FileOrArrow { + match value { + FileOrArrow::File(file) => modelfox_core::train::FileOrArrow::File(file.into()), + FileOrArrow::Arrow((array_ptr, schema_ptr)) => { + modelfox_core::train::FileOrArrow::Arrow( + array_ptr as *const ArrowArray, + schema_ptr as *const ArrowSchema, + ) + } + } + } } #[derive(FromPyObject)] From cfaf85122ec8d0ac0d8794d252b9772f6e373895 Mon Sep 17 00:00:00 2001 From: Chuxiao Feng Date: Wed, 9 Nov 2022 22:27:38 -0600 Subject: [PATCH 3/3] load table from arrow stream reader --- crates/core/train.rs | 24 ++-- crates/table/load.rs | 133 ++++++++++++++++-- .../examples/basic/heart_disease.modelfox | Bin 29071 -> 29098 bytes languages/python/examples/basic/train.py | 23 ++- languages/python/lib.rs | 11 +- 5 files changed, 145 insertions(+), 46 deletions(-) diff --git a/crates/core/train.rs b/crates/core/train.rs index dcc29669..4b938442 100644 --- a/crates/core/train.rs +++ b/crates/core/train.rs @@ -18,7 +18,7 @@ use crate::{ test, }; use anyhow::{anyhow, bail, Result}; -use arrow2::ffi::{ArrowArray, ArrowSchema}; +use arrow2::ffi::ArrowArrayStream; use modelfox_id::Id; use modelfox_kill_chip::KillChip; use modelfox_progress_counter::ProgressCounter; @@ -38,7 +38,7 @@ use std::{ pub enum FileOrArrow { File(std::path::PathBuf), - Arrow(*const ArrowArray, *const ArrowSchema), + Arrow(*const ArrowArrayStream), } pub enum TrainingDataSource { @@ -756,8 +756,8 @@ fn load_and_shuffle_dataset_train( }, &mut handle_progress_event_inner, )?, - FileOrArrow::Arrow(array_ptr, schema_ptr) => { - Table::from_arrow(*array_ptr, *schema_ptr, &mut handle_progress_event_inner)? + FileOrArrow::Arrow(stream_ptr) => { + Table::from_arrow(*stream_ptr, &mut handle_progress_event_inner)? } }; // Drop any rows with invalid data in the target column @@ -796,11 +796,9 @@ fn load_and_shuffle_dataset_train_and_test( }, &mut handle_progress_event_inner, )?, - FileOrArrow::Arrow(array_ptr_train, schema_ptr_train) => Table::from_arrow( - *array_ptr_train, - *schema_ptr_train, - &mut handle_progress_event_inner, - )?, + FileOrArrow::Arrow(stream_ptr_train) => { + Table::from_arrow(*stream_ptr_train, &mut handle_progress_event_inner)? + } }; // Force the column types for table_test to be the same as table_train. let column_types = table_train @@ -832,11 +830,9 @@ fn load_and_shuffle_dataset_train_and_test( }, &mut handle_progress_event_inner, )?, - FileOrArrow::Arrow(array_ptr_test, schema_ptr_test) => Table::from_arrow( - *array_ptr_test, - *schema_ptr_test, - &mut handle_progress_event_inner, - )?, + FileOrArrow::Arrow(stream_ptr_test) => { + Table::from_arrow(*stream_ptr_test, &mut handle_progress_event_inner)? + } }; if table_train.columns().len() != table_test.columns().len() { bail!("Training data and test data must contain the same number of columns.") diff --git a/crates/table/load.rs b/crates/table/load.rs index a03160f5..ebe60e8c 100644 --- a/crates/table/load.rs +++ b/crates/table/load.rs @@ -1,6 +1,10 @@ use super::{Table, TableColumn, TableColumnType}; use anyhow::Result; -use arrow2::ffi; +use arrow2::{ + array::{BooleanArray, PrimitiveArray, StructArray}, + datatypes::DataType, + ffi, +}; use modelfox_progress_counter::ProgressCounter; use modelfox_zip::zip; // NOTE - this import is actually used, false positive with the lint. @@ -9,6 +13,7 @@ use num::ToPrimitive; use std::{ collections::{BTreeMap, BTreeSet}, path::Path, + vec, }; #[derive(Clone)] @@ -247,22 +252,126 @@ impl Table { } pub fn from_arrow( - array_ptr: *const ffi::ArrowArray, - schema_ptr: *const ffi::ArrowSchema, + stream_ptr: *const ffi::ArrowArrayStream, handle_progress_event: &mut impl FnMut(ProgressEvent), ) -> Result
{ - let array = unsafe { std::ptr::read(array_ptr) }; - let schema = unsafe { std::ptr::read(schema_ptr) }; + let stream = unsafe { Box::from_raw(stream_ptr as *mut ffi::ArrowArrayStream) }; + + // copy fields out from stream reader + let mut iter = unsafe { ffi::ArrowArrayStreamReader::try_new(stream) }?; + let mut all_values = vec![]; + let mut column_names = vec![]; + let mut column_types = vec![]; + + while let Some(array) = unsafe { iter.next() } { + let array = array.unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); + let (fields, values, _) = array.clone().into_data(); + + for (field, value) in zip!(fields, values) { + let column_name = field.name.clone(); + let column_type = match field.data_type { + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Boolean => TableColumnType::Number, + DataType::Utf8 => { + let mut uniques = BTreeSet::new(); + if let Some(value) = value + .as_any() + .downcast_ref::>() + { + uniques + .extend(value.values_iter().map(std::string::ToString::to_string)); + } else if let Some(value) = value + .as_any() + .downcast_ref::>() + { + uniques + .extend(value.values_iter().map(std::string::ToString::to_string)); + } else { + unreachable!(); + } + let variants = uniques.into_iter().collect(); + TableColumnType::Enum { variants } + } + _ => TableColumnType::Unknown, + }; + + column_names.push(Some(column_name)); + column_types.push(column_type); + all_values.push(value); + } + } + std::mem::forget(iter); + handle_progress_event(ProgressEvent::InferDone); - let field = unsafe { ffi::import_field_from_c(&schema)? }; - eprintln!("field = {:?}", field); - let array = unsafe { ffi::import_array_from_c(array, field.data_type)? }; - eprintln!("array = {:?}", array); + // write table data + let mut table = Table::new(column_names, column_types); + + for (column, value) in zip!(&mut table.columns, all_values) { + match column { + TableColumn::Unknown(_) => { + unreachable!(); + } + TableColumn::Number(column) => { + if let Some(value) = value.as_any().downcast_ref::>() { + column.data.extend(value.values_iter()); + } else if let Some(value) = value.as_any().downcast_ref::>() + { + column.data.extend(value.values_iter().map(|&x| x as f32)); + } else if let Some(value) = value.as_any().downcast_ref::>() + { + column.data.extend(value.values_iter().map(|&x| x as f32)); + } else if let Some(value) = value.as_any().downcast_ref::>() + { + column.data.extend(value.values_iter().map(|&x| x as f32)); + } else if let Some(value) = value.as_any().downcast_ref::() { + column + .data + .extend(value.values_iter().map(|x| i32::from(x) as f32)); + } else { + unreachable!(); + } + } + TableColumn::Enum(column) => { + if let Some(value) = value + .as_any() + .downcast_ref::>() + { + let mut v: Vec> = Vec::new(); + for s in value.values_iter() { + v.push(column.value_for_variant(s)); + } + column.data.extend(v); + } else if let Some(value) = value + .as_any() + .downcast_ref::>() + { + let mut v: Vec> = Vec::new(); + for s in value.values_iter() { + v.push(column.value_for_variant(s)); + } + column.data.extend(v); + } else { + unreachable!(); + } + } + TableColumn::Text(_column) => { + unreachable!(); + } + } + } handle_progress_event(ProgressEvent::LoadDone); - let column_names = vec![]; - let column_types = vec![]; - let table = Table::new(column_names, column_types); Ok(table) } } diff --git a/languages/python/examples/basic/heart_disease.modelfox b/languages/python/examples/basic/heart_disease.modelfox index 0ad475fd5f9560a661b8174b9c7062aec82b2ec0..f00975e6ccd3cb017e0475c716e252d5c544ef2f 100644 GIT binary patch delta 7783 zcmb7J2~-qUvz{J6z=4R$>IMoyM1%nZ6o>CdBcqANT~WcfLc}Ezi5fL#j0=VkCGonE z#28$N5*PeYqecwVJv|#U?1CUF2u1}Jj2id&8m1@za^898{hZTXRbN$ox9+X(JItNT zMkcqB`Py~5JR&JHVs6yjp^>5CVNr=gljcVZT`+G*Vp8~!$gsJgL+vD%KY8$wV7aeo zgvrCg28D(W3YAZh4~`BU8XX!LJX9VUDVGn4ln<24qvdki#n0#DSATYzClzzr#qGcK z^<;pkcNRyO^0&{TSo4_S|K?av_8PqKXPy+h?wmn~i!wcS=C5|Jb#$;s(sf;jO|%uQ zF`~2*P8Mw!QH~eoE>TVtWwIy>-moYq=u&rYm9vO4Te?^@={?ae?IkMA0hz_cOt31N z&SG$}qv%Qyr!6fM>tuct3zYuFh%uW)MY`7lNE{{G{Rpk<=E?k;-`Y(o?VQrh;%Q{) z+z5Bt)onEsM>E`Nq@mqusO`x7Tz5YyGme&c#!GYEX%~BUI>Kw#zqKS^q?Sn5lYXL3 zJS@upyCpk6_C??DSS>pZX7GnAH5bfD=-w?*O`WVCug$Ekc}Id(-Si{jB&x{lA;mdjEb*(Hj*p?z0a5 z-Lym0evqfsHZZ^MpcPVS@F2P;^pNfSrcz^A5VMUw30v^r zHO)pW*4i|&W{I_F;yWcNlK-zw8({5Kme?z)-pLZk#93dpJh39iPgJDA;u0~|S54~o zUp9@L+9*YXduNzj#c%BwR23sN|)~$oK*l+6^CxX_;K}I(rlG#sx z#e61*dXor7L8p@S%q&_(qNPu5Xt%e*{_O!&{pY4{-TXVIC$J9{M~D&wGevudD6P8+ zlP3m9XQoK<2LuAgw9=i7eHhY+3y#8N1n!MC|e<{;H$> z$U|*kvh8;tQr6jr><{-Q*XHyg$=iFA5PdJwRoaU@B0Y)v^ByEY=S9xAc#$KsJW0TD z57JBOK^Dz$C-$e@NJUpSQn=ieoT=(ga$>p@)lc0>fNM9hdt+B}=CKP|v&e-Q8@iC} z_%5WW`VF#Wns8M+6P016Gl{KtA~!y8B1a#{$mi>1Bta%4^6xs6)}YShn?gt8G|iDr zx#vI<(i}*Sz78a#z@B^?Z%^KR(TSve--(PRoyg8cJ2ENVj;xj25sw;M;;_+{#0A@u zhqX4O`U@LkGsK20ye%a*-%3f*SSdL#k&@!mjEX#8%#h)}4B2)?LLTgpkeTBpq@c5e zgl~I|CWqHpzWFcI+582!O|Rh6=@qo;FA?75B_{290cY-GpZ^$>Tb zJp}vw4;x7(niZ8%TbP=B`-Prq%2?A=z_KfjOLx%Y7- z;XcN8xsNjT9{g6_!?d7#h`(_c0ax3Qw_7FL|P1^d;v5Qtl7vAG3> z@ppKA^EoP>ku%v4);)pwqA9JdT|A3YOY|-*(+GL`wA|syn?7{SMV_63U0eyL9ah+v7xFK z{eP*2?VeinTwROgnYFk(tQN<5*JA(c8t@IO8u)5za4x3?pQqPg?UEWC9A5(_yawCd zYY_MRGDcmyj70V_*qqB4k#QLhKe~+aDVI?@^fG#RU&fcVm(jDi8nq?W(4MYF&3D!C z-cXJ7h1KvKUyb?*0q<2UJjB)TXsN>L@+vUrt8g~E3K^>ODr`xq!pRv`crm;RO9QL$ zvYX(1aS2&Xm$27-3FppSg8h$|urmD;B&nBhA>k6zl$WqpehFK9T!Pm25`0=KL2D`z zsj0-RR(Kx&u!SZr^)2AGE&gIzsq72IBGB7n|sMD3faHb6ALuI(Lqf7;_O=Xy$ zT86@TWr&(oh6}^XFg>&kPJPQzrev67sXh6uNcp+7bCO0 z7=e5-F8@-DtlVOJu(uc^zb?k$jRL;17(&6%3)QDm{iYUg|0mV4ns~Ejq zim}(G7(uEhX7J5sq+Bz@zub(wycvb(%-DO(jBVLwr0p~#J>87#b!O;O%y^P$M#xMv z){Zlyc!Z!wnXzx689qK{9CkH>IGSH{q>uCLDU(1h=6k>I&;bn#px1k6hd{Ts?9~Pl) zK@r}LFT%lzMQ|Klq{12$;lbb{tQuGZhkixa>nQ|uDZ(wgBJ6%?#N>xYcs3hxv)+h; zY9sa)8?li$V)+Fl7Uc^v*N8RQMr_$*#NkXMOld|uS!YE56-Fd4G2+;KBVNuFcqSQf z{2hU7xDl!$0v00V^;a43x{nc=?nVT>VT9Vwh{RV0cs@1=`@aEOnhY3SXMk&^0k?__ z;5h?wel=iQz5$>AY=G*30qedKbZWr%48gb20CuebH&YF8TVlY71%f}pfOAs}uzk;f zNwEg(k1>FWG+8N)MIOa6Lvx>#-qR4^5CBj{bT~_SWODr!YV_JrZPkT(lL4{?cLh z6CE5L=&-R_2c}5})m6c+)*-7@hXA7vT26A+=gq>G)#-rky>mF z)naat7U-{qZ$B+0y|uXGp+%Ldn~>zJ#W@Epj@f9j|1Tan&xGp{kK9%sr<-{c-sDkQ z&!h1Qk7rdpJj;1Rnt4pu^Z1YLa=pmB{1R93EZ?JZKz`E|Ym|9LGcQ zt`Ho{!)7>-GzE{YLwI}_&SPK*4|SkG?9ZdMFOQ7gJp8?cf?atmcH!aU#6xQ@6lKH1 z_q7H!&o#*SQ-d*oXkdR|qXK_N0B&k9u0aF0s~R+3*5G)h25F@lOfhL7*K6RU(V*!! z4UE5Pkn@WMTk|zYJ)uGT&jQg;8jLugLC`)8{Jztm=WY$Ws0Q6LHE_)kt}PmPZq&eE zC2+0PAZnG6ld8e&k2F}mSO7CW5J=~1z|R&2iPykkx&{GLG?+9&gEeCX-)IfWM`~a@ zTmurLLCR1KjztKup&IlH)*vxZgTwwBwDr?qu(t;5dTLZn1o zod#M)DDp23>z;FH{FB27)k6-)TRFJiY#ZI2eX<$c_{S59csj&SBUf z4jzFVB>g!w`3im?PKA@bIegoL6aKN}u+mj1qN^~IH#m%!3Dr9YMc8o&mvRVr%_8un za6MxY@F$Dlhb)5HSVZ1qp=f4>PdgSuvm1B#hNQDzP!w0?bPMb@t@)Mr`TI?cisyM!Wkut@xd#ldYXZe|EE=`7}@u{g9*$XU-K>@yZC*Rm-1goXV|7SmD% z-!c}Dm$DeWn8mlMg)CYUSztbkopV^U&JrB)LeO*;Hq!+1$wFZhSon@(p&BC$GMW`O z7Z(3`n?>*8EH*>PA12ItC<|kxuo8n=91RtGAuM(W2^@he*7X;P^<(j2KOxqe#p2#9 zZubxh^%OW8-Gt@pF3jA8#RqS&Xp;#`=g8t!Ct>+)S-4198n2G3 zq0Uuf)nPT{IcnTHphjM{8jJS{MebFj^;ta5$@2;pC_Hm3|$}7eG z>RiPb!YTZ?jE>mIpeNCG0}m*!$~P!Gv{l#MgSJWtAvv6DDRTqK?nAISiTcuFR@?bnIj$lR~L*KTB&TE2Xdd(0)^5WlC`c zg=~6misGHEqMgY;{(IQ&;y_ku9k=yQFO*MCG#2mQMmK;)-54d|>{}7-vw1O9#VKSBqMgaEpxQVE6F}dcu5|v$*D~{8uzC|G|?%kIhiZ%0;$GXMdo}W++B` zy=h_ZTz%kgcI&fYeO9bRrhhS-FmuoK7mK`j5gkSDzmSGGbi+)g>?e_It?P06)l7x# zzG#Ps=hKRr3gL%^xOk=PQ-8~l2bUGlx$z2yqc`J~OlKODpmd(t-{LcwOz27<#Cy>3 z3BJr8dS38d7kx3EtN+F4Pe;#EIxh{d4QG4bM9O#)A%>tz_P(>xKIvs++BJz8h#6+x56TaiL`k zJm`CKeVxMxTF_-No&N=0Bcf)L)4!l<5gm}w<)0|yKm+G0nejAbuF`o|kcBO#v&TOp zq*sHs&x>XJsB)f?SxmRiQ_B7^$hr=F>4bR->9aw!*Zf%7Tfr7MDapSbX135H^Oa07 zZ4>TpG-`p8NuZxCP|6O3SmMIN!|Ct^3Tbl)bxw?xg^DNqEfa^d!?I6AdsIVoyUqMY z8xoaFCmN8Xbbc??LXnU;f;e@o#+W2uW(Pf!q?BF>rGF*G%KCMjO-m%pV&0{NNeU*J zW_+M@&J)oA3FC>34o~)=iiN(?CtAXGC zI_p$oN9RO%yxG+PNLHH5V~dAh_2I@Shej D?F+Re delta 7782 zcmbtZ2Urxzww@kTB&nD{Q9)o46M{rx_{#vJ>l(oo7ZC%ZtC(HW)pZQ3t~r!xjf-j3 zYex5?PtfV^iOdW+2Sq?dRCJX!>uZ>v_q}kx_w9SPzpv|@I(5!}s!pA%9uDWLn4MM3 zuJ#lBrUv^@3!0wb>p#H1zn|ZXpaF@#1N{5~227g~(myb8YIBMC@%0Ju@%0cp{eAoQ z@9R6Dubb5^y47UKxm^x>1Vlxro zheb;gr6S*5q>n}FDbk)IT`n3ep}Ra?saxkADJxnwv!N3QhJ9@%stgfn8<7%`?icAG zk~L`HhlERfhpsIMoX&)YgnPqwC;J^fM!w98_c@2G8ugHqC#-t_vF zMsy63c4|lcnnk3r9X+K?5Pjr4QM#uiZEf9=#&(+Y*JGU`dbPybUF`4cXl5+2{$0A* ziH_|XnbOteiWSq9n!25mw(dgb`bMOry6$C|BXneU9g{+x+}BBKUFl)>gnzm(mdIN6 zWv=MXvM;e>zm=PrvFr=;U-#t~%XZjUm=F<}DHL~s`N^^!h2nNdrIvkZZ`l`1M*Zb| ziDB$he)inYFypDK*Fi=`Q@x6%9X*?NqOeB*!wgBW?|E4&edSHldN1zvms38F?oM#B z`P19HYD>_M(cr$$(!8E2WBV?VNHjq9meXO0dQUORmZ*zaE?MvUzee5B6702Nu%%b*%?1pyoQ8uf%@S#ywa9c7 z8OCx5r1t;!)9^9CS-RFwI1OGYvcMXq`Sm+%u#c_{cB0|I#Hv-u4XOy9&_Zgr1m7^U zBBUdI6--*3opFJjF?UCWY_XELoTa59>!rK0>68IKwRn@RNfOh+iDB+bZ+R^ z3ODntCB`)j8rkx!@^t->tR^1He&qU(%hLS6+)Mv3FfRR~Lu)+t>eVzBczs*YrkN`lIRTTI1J<-1=ve{_)@q;j26o;63-(^oW6j$m%Y( z;aB3P*@nx%^uP0EK?mYS!@~TV!fUOHCN*`wpo_vnd}i|%zHY!opQ#lAP~)Onv{jMy71 zQW>QPVsfb-2}~jNA%VNJ+?i}Jbt0Kg zoyfK6&cywO6Jcze$bcyw$&Fhb$oI}2h~J9##G$+$DIC&{?7rBRbnM)gOxf&6;y$z? zFXy)*E{|Fh?UdH!VOc9Obz&Laa!^V<{PaNJ?IgkP==ZCAV%cM3=;nSMH30^eUB* z#d{?rZIp!A+DgcYZC~JN{RJ!6f5tWDGx)Wiu+Qofyf%GAlKn@-(hqp$@&WzNyvJ+5 z_ZW5O9ehT;!QC&6{}adFHo$6c1CIGNV5R9b4$OUx zGKbd~cUkcYFUGz?YQrC(hyK8pfq&qaM=x=8=SviYyo7bt3;67Kf%woDSox?P2lmxt ze?&cYync?kr=P<&<~d9@&oL(L8LlpThH|fGs4jbotMn;GeDf5yte&Fkb{+DU)?uSx z9U7iKL6>t+@HX)YHhVmQPElElTff#~^psj0a;wGW%Exd#`4~eJAET4^V{E8fBcm^t7Pg1$V&e$_*4*!B<&(GS6Nd59g=H8^*%27{N?VAh}-w6w25ufl4WPE@0A zUNsb9)!5Op8eI#j;CiYGI~G-8#h@yf+EhV7?>~U{$^#^=eSnow5AfRK0kRq@;iRpE z>QE)B=2l|Nph~!PsKmtD`*@RfAGLe#BkcS8cp5Ie?e1gTlY6jH-h<1*dr-{32Qu^? zM!4R?^*0qr%&I`rxeBoBDlj9i0+WL(aJWqcyz9!*QeCbywN z%HjB|4Ar_ad^l5vQ9qU8_0%%xi6HA*hT$Je@u;K}s+*;_f3Oq-mzSbArW6;0OOftW ziZ*Xcu)C-PQ8!C4{7?zjtt>%ZTnRR!1k>C~ut-{h)aqjN(-fopoT3l z7voS+F@|<7Mvu=$2!2q6rRpM7pD)7tokjTOhawF8t_aIe1fxe0qHT)s<#{1W@(NLT zs}Rn~g_yso5MSmN;_A3UYz-^KM)yM8v?+wc%K~gKDuDmp0yH~YfR{T9V86To!xd8s zaD7+-0{jY4*Qo$ErNUd6kG#Bmv`Np$;@|Qi*`1Fwf6K?L>G_yADj%~$@^RcfA5ZOt z{+m2Jt<1v-LmuY;BM%8D@~~`s9!@XM!<&RWOdXwv`p`V=_sYZMws{!*IT!LLxtLmz zi)1zzig%ZCk#sm0A2#LU(hmYZH5bvNaxp3_7c;$cai~KsYNWXs{UQg(@*E^(`jAj>%iORRI?^Jg~fYO?VuHyiKRY&c!Z z21(Aw_FdViQ)DYJd44vk;{Ddo-NsDRF3&{OY+*1i6YWN4V%VTetoP4E&R3ZTa?ZqA z+f4NMWI}4a2{8{$u*o-p*O;*TmI;e4nlSNK;iV?b+-Smz6(*dUV?z046M99Puv%_H z)c_Ns`kG*LGajWeQZm=ST|MpXM5q3|-o!^Mc~)<$e-X2ghh2Dm>r;KKt09u*o; zY!F_h0ndInp!o#@Jd+KG*>Awktp+Fx))?Tm)PVI#2GmY5VB&ZKO1?2*+8_ho1{py6 z7!c}iK)sUz=Nt@JY-Ip>mjSo>4A|FXprIrKZ?ZD*R-J*?X&LZJ$${_8TZad`$F&&xnmVg}MCW*{Ug19yjIC@_j-pgJ%EKlTxfJu;BrDFZv&WMI5? z20TCO(ePT2vN}C(J<#Jqu^#8M^f;^0<662N{0%)yF6d!>N)PYDdQ9G<$Fa?NRIbsZ z_i{bf&)1`3h91M>^+^3z&>5-6i3nj#CgAz&F}$}zj~aJ9XlFfw+v@SsMvrrh9t+>; z5b{z7hguz~@9V%9>2M)ShXYz2_TSZEcd8EiuIO;$tWLPTb;v%bL&Gi|TsP|wwN{6% zD+HT`I@rzDVcaww&c*6r6|F`9R=mqI=r?P z97%LSn6-HEQi~H$w1}(H!lg`$+B_|88nxK0)*|ta7DH3D@VTmm^LZ`WoYbP(5iP## z)xv&<79BQe;k`x+vO+S{Np2VHKlA=ol>&KYp#n-N9No$h3$F)Z&_t z7S`Tc%gFrufeNg4G!dL5N6b% zR;|IlyBZ9?r9t!S8t@l2*!-IY(a9RP9oC?JuLk-Z8eIKZgDvYc3d~!jLHtq;MlH}_ z@N5la(*>Y-flkn1M3e?GBQ%&L*FZ5)gF~Sj+z!y7+D8NHo*IOBXfVI42FXqulsIbO zYA;~5(BPs(gSYS1kTQtQynO)Ohp{ zHRfJZ<`Iu2RE)nHrB5s*yZLjrf^r zxKC5#S)3YIW7Jq3rN)?%YIqD&gBhYmQMlkdM2+MAYHaXPV@XfJr>8I}g{vChbQazY zY7B29P&+k7w@_muqsEMnDlB=U!qz`jIQ2{g_OS}(RVsX{P{FH2g;Dt`tjkj2LWT-e zY86~q6~?EjKvPx7OHsk;vI^7At8nSG3h#bZVayQ~&K?l#_oxuJLxt4MDkxkwsIYO3 z3NKcw5Wh?X!y*+1&Qsy~EEW6{Rk%7ug+6ftjZq;uS_OWT3d4u1P!_2|(jXNi;VK*p z5yk>kDCnociry-?^-w|WuEM--Ds=6vLQV%2HaV&g=Ac4DO988y3QHv_1bpNbc>9J2 z`-;cTdLFTLJbFFi@#O)Jf(jm&N_cE4;4wFc#~2fjU_B3KH4h2Pqb`j{mI_W5 z@i;e6P@c`>#S9)U2|OYu^H?^K$C(%&Wzjs^jNvh21dk2Fcqk)ySPkMK59hIdfG~+b z0kl6a{OIIyu@?{P9z4c*@Hp?vgHE&tb(H4naS1c(X!qyp+Sz#T>#GaQHNbgMJo=Ju?L72^@M);qW1jgW)?4 zC&qJF9L-_G7!F<|IY@?cD3^0cAHw1IKn@$jIm`$Z#)CNo1aNTk;}mG^!{Kc&4v%_p zDE1V<8a%B!wqK+mpX8`*j89YYvHx$aMnhkErc;EVWJX2_Y;e(_biIvu&8~_ zqTwYA>v|Slp0Wt2Wijj_i^M7xtM9WoRL5XV0^!{*8s}X%_J(SR@~3QE^1*A7YVk zfW?KqEZ*&A5w?@XmTfEwHnVWwB-sAMf?dm^li+vuDi&#q6+&kji$y=MP%mWRHD8$N z92S+cSYRfLW77r4(*)4TEc9{0iegzDk6~f+EsKRwEbfnHF?u8m(7$G=qQ@Cag3>zz<^a)SpF?pAaP9rDV z{!rq2y%N#Slz8(*iQ|uz!d58ptV#)5sRX&F#G^8$0*6b4fg&Z|gc2pkl{kA;iG_!ikpCjs9ZJlF-)BhG@C8k1Guy7a6tK)eLoMXyt#n5vMG1{BK@n%?r z6%OQ<2(y!IpxejE>=-w*ZECmX^u@O_W*D9Dt(@6Oj|*)9efX_h+O<3N86U|^r$3CB zGk>T5HePP`M|ZQ4uw=S=yv%NZSiTOojiHq>GG-0k6(hIPh|+EwlBo9tnO!Rnv(-*^ z37tPd#(YZ?C&=xNh|<(<8=9n5^wk8pLoZLWZqK-#Xiv9)=j16CkR%Zi5sk}oBTTK^OQq*Cp7a^?$d9V;Iu|H=$yTX+)f%&buCH$URmNyI{v z#Ap~nn9|?-5=QPci7;_H2a`r7ZQ~+rr$=Muc6lP$%J!FN;6#~&iUym`~JUvfMtR zmly$wgwX8xFlH+~EQoVNv5@plV*5_L&3f%GS(^n@M8Se^TeILdQD8LKn+4BBLC?6> zO#-{XK4y#!b&gayMaHb6+osCxI8hpI>qG}8$fUM?smruTW-OgLP0k#oXNC42t)C{B zdil`cgh-))`(uLK?z)c|i*4|3v|>I`yXkVf!TrqAmF<0~U!siJOn;d!XR_%1>2j%) zFZE806#f}8FH!DrTCCp=tqdg&bnH|oPfNl#WkN-{klWdjLX9)3+MhHsL3?AupVE(l zslTNpEE!1b8%?R}3=g}%iBjP}&?_@#jFM*0kV{+kr=4a-GNb9_nQ~@7JtMT`^w~_g zgQveG6k~}U^_%TP+kEfAB+-@M%N?$XW5QiQY@5bvgt2$jW|o{8NXH26CVD_!RxY`McpQOY)aPpq3Tx+ZxrPw4yEa>kDiN|H0n>CZ`WhdV)L zYk&X6q#3<3$LXuKW`V@t-(S2Og@XX5X!B7tGL54xf~nJ-NMSPtNcj for modelfox_core::train::FileOrArrow { fn from(value: FileOrArrow) -> modelfox_core::train::FileOrArrow { match value { FileOrArrow::File(file) => modelfox_core::train::FileOrArrow::File(file.into()), - FileOrArrow::Arrow((array_ptr, schema_ptr)) => { - modelfox_core::train::FileOrArrow::Arrow( - array_ptr as *const ArrowArray, - schema_ptr as *const ArrowSchema, - ) + FileOrArrow::Arrow(stream_ptr) => { + modelfox_core::train::FileOrArrow::Arrow(stream_ptr as *const ArrowArrayStream) } } }