{
 "test": {
  "n": 184,
  "positives": 102,
  "y_true": [
   1,
   1,
   1,
   1,
   0,
   0,
   0,
   1,
   0,
   1,
   1,
   0,
   0,
   0,
   0,
   1,
   0,
   1,
   0,
   1,
   0,
   1,
   0,
   0,
   1,
   1,
   0,
   1,
   1,
   0,
   0,
   1,
   0,
   0,
   1,
   1,
   1,
   1,
   0,
   1,
   1,
   1,
   1,
   0,
   1,
   1,
   1,
   0,
   1,
   0,
   0,
   0,
   0,
   0,
   1,
   0,
   1,
   0,
   0,
   0,
   0,
   1,
   0,
   1,
   0,
   1,
   1,
   1,
   0,
   0,
   0,
   1,
   0,
   1,
   0,
   1,
   1,
   1,
   1,
   1,
   1,
   1,
   1,
   0,
   1,
   0,
   1,
   1,
   0,
   1,
   0,
   1,
   0,
   0,
   0,
   1,
   1,
   1,
   0,
   1,
   1,
   1,
   1,
   0,
   1,
   1,
   1,
   1,
   0,
   1,
   1,
   0,
   0,
   1,
   1,
   1,
   1,
   0,
   0,
   1,
   0,
   1,
   0,
   0,
   0,
   1,
   0,
   1,
   1,
   0,
   0,
   0,
   0,
   0,
   1,
   1,
   1,
   0,
   1,
   0,
   1,
   1,
   0,
   0,
   1,
   1,
   0,
   0,
   1,
   1,
   1,
   0,
   1,
   0,
   1,
   1,
   0,
   0,
   1,
   1,
   0,
   1,
   0,
   0,
   1,
   0,
   1,
   1,
   0,
   1,
   1,
   0,
   1,
   1,
   0,
   1,
   0,
   0,
   1,
   0,
   1,
   1,
   1,
   1
  ],
  "proba": [
   0.9207754622292952,
   0.9096187077609306,
   0.87856722780582,
   0.942396916379984,
   0.3639704427508606,
   0.9680899614411214,
   0.6527353248132357,
   0.9044829773573548,
   0.2236359342448353,
   0.949546485973398,
   0.2208781854827972,
   0.2571888921852379,
   0.1060496876105439,
   0.1933030942563456,
   0.1714289168727596,
   0.970006017925428,
   0.4451050829220098,
   0.9330380664545996,
   0.1820145223119752,
   0.823155706709858,
   0.2392413123566925,
   0.3995183536207224,
   0.1393346397656679,
   0.2157368356093327,
   0.6790092722105863,
   0.776695282683368,
   0.570444753241362,
   0.7567774510960406,
   0.7022560453838694,
   0.5274632266225067,
   0.514569374357983,
   0.2691472772322473,
   0.0282592180873991,
   0.2783429400396947,
   0.963274677849038,
   0.8875082649808795,
   0.7569286984034713,
   0.6497019544035599,
   0.316532658919668,
   0.9160454361206296,
   0.5371657984964961,
   0.8250228940881204,
   0.6926290905039286,
   0.0389258973193794,
   0.2531657452357646,
   0.5971496071133294,
   0.683302948970506,
   0.1893241722392568,
   0.91967357873968,
   0.3693671459990884,
   0.1234676663515201,
   0.1253486088952581,
   0.7357568967270639,
   0.0268308133674615,
   0.9765945238569204,
   0.0603422604351022,
   0.867845058356782,
   0.1344655421116772,
   0.0231114959569036,
   0.1491738543458537,
   0.7338854215502164,
   0.5082956199154034,
   0.0309809579919591,
   0.8028492455887147,
   0.0246821703534106,
   0.8965767842213404,
   0.9216336288573752,
   0.8589364812956417,
   0.8844642347889904,
   0.0203225396989625,
   0.2261045464642784,
   0.4474447329273516,
   0.0684432404536631,
   0.4276914136218382,
   0.148136154277927,
   0.9609289161871778,
   0.8681643755821085,
   0.9539081760527572,
   0.9304800866165888,
   0.4066702839027421,
   0.9514065325082616,
   0.7619767306622894,
   0.7810726214891777,
   0.5758418095354648,
   0.1493720208725696,
   0.3170204059613387,
   0.4653391320147172,
   0.661048043026017,
   0.4533159061599373,
   0.3696953041278049,
   0.3835168432754239,
   0.5740752170965719,
   0.1484857775321696,
   0.2484373707405813,
   0.1313860717689829,
   0.9210219196081808,
   0.958135539836289,
   0.0559922699637847,
   0.7221434862898694,
   0.4897229562545964,
   0.925155836216474,
   0.907617680202555,
   0.960218969210419,
   0.0723698986993293,
   0.9810529125155748,
   0.6665845441898455,
   0.8068747736428658,
   0.3132655854563579,
   0.586420835378583,
   0.7684638847883354,
   0.9486985789051126,
   0.1314988363567089,
   0.7820063078132184,
   0.9379638588745368,
   0.9802796507201634,
   0.1264731216030581,
   0.7027333329648263,
   0.108926383893646,
   0.8528081404902531,
   0.930608515392732,
   0.4974399491185262,
   0.6977189330468723,
   0.2945313655958383,
   0.1188920294715077,
   0.2535817457357913,
   0.7518985673988826,
   0.3282594715724276,
   0.3831284750716164,
   0.4878463431169609,
   0.6020840644877467,
   0.3980862808896738,
   0.3704969065132172,
   0.0932421569078439,
   0.0290656120042622,
   0.915367110029436,
   0.8004021863727642,
   0.9801318991117148,
   0.2007023179709651,
   0.8655032937825842,
   0.1109293083911623,
   0.6599770610871216,
   0.5484453859136085,
   0.0618622770319101,
   0.5968267104466303,
   0.0306735271706031,
   0.9654129426877234,
   0.1994079255207159,
   0.5480869849310183,
   0.527664928772553,
   0.9230346096229584,
   0.9674217072716716,
   0.1638059433732745,
   0.9175259710996682,
   0.7100796627332422,
   0.7887010501777898,
   0.909151373794378,
   0.4588154194288001,
   0.5837661956123914,
   0.9730814272106908,
   0.6010715135524686,
   0.0869209856540805,
   0.4831872723556762,
   0.0510368486307685,
   0.0381585899323688,
   0.9817440567833994,
   0.5340484083802371,
   0.1932788337482241,
   0.9053224070050369,
   0.2219651067615528,
   0.6695226230535037,
   0.8071037120031999,
   0.1294532976948359,
   0.9593899391697712,
   0.8598862910553381,
   0.662551724812046,
   0.7523445756094197,
   0.2264159821888116,
   0.8855525731088054,
   0.8440316001183863,
   0.1770020670616716,
   0.9289716333262308,
   0.9264533587445212,
   0.9345783755023872,
   0.2850802162432763
  ],
  "auc": 0.8715925394548063,
  "average_precision": 0.8911054764237094,
  "at_threshold": {
   "threshold": 0.3883,
   "recall": 0.8823529411764706,
   "precision": 0.7758620689655172,
   "specificity": 0.6829268292682927,
   "tp": 90,
   "fp": 26,
   "fn": 12,
   "tn": 56
  }
 },
 "roc": {
  "fpr": [
   0.0,
   0.0,
   0.0,
   0.012195121951219513,
   0.012195121951219513,
   0.036585365853658534,
   0.036585365853658534,
   0.04878048780487805,
   0.04878048780487805,
   0.06097560975609756,
   0.06097560975609756,
   0.10975609756097561,
   0.10975609756097561,
   0.12195121951219512,
   0.12195121951219512,
   0.13414634146341464,
   0.13414634146341464,
   0.14634146341463414,
   0.14634146341463414,
   0.1951219512195122,
   0.1951219512195122,
   0.2073170731707317,
   0.2073170731707317,
   0.21951219512195122,
   0.21951219512195122,
   0.23170731707317074,
   0.23170731707317074,
   0.25609756097560976,
   0.25609756097560976,
   0.2682926829268293,
   0.2682926829268293,
   0.2926829268292683,
   0.2926829268292683,
   0.3048780487804878,
   0.3048780487804878,
   0.32926829268292684,
   0.32926829268292684,
   0.34146341463414637,
   0.34146341463414637,
   0.4024390243902439,
   0.4024390243902439,
   0.4146341463414634,
   0.4146341463414634,
   0.4268292682926829,
   0.4268292682926829,
   0.45121951219512196,
   0.45121951219512196,
   0.524390243902439,
   0.524390243902439,
   0.573170731707317,
   0.573170731707317,
   0.6341463414634146,
   0.6341463414634146,
   0.7317073170731707,
   0.7317073170731707,
   0.8780487804878049,
   0.8780487804878049,
   0.926829268292683,
   0.926829268292683,
   1.0
  ],
  "tpr": [
   0.0,
   0.00980392156862745,
   0.06862745098039216,
   0.06862745098039216,
   0.4117647058823529,
   0.4117647058823529,
   0.47058823529411764,
   0.47058823529411764,
   0.5490196078431373,
   0.5490196078431373,
   0.6274509803921569,
   0.6274509803921569,
   0.7058823529411765,
   0.7058823529411765,
   0.7254901960784313,
   0.7254901960784313,
   0.7352941176470589,
   0.7352941176470589,
   0.7549019607843137,
   0.7549019607843137,
   0.7647058823529411,
   0.7647058823529411,
   0.7745098039215687,
   0.7745098039215687,
   0.7843137254901961,
   0.7843137254901961,
   0.7941176470588235,
   0.7941176470588235,
   0.803921568627451,
   0.803921568627451,
   0.8431372549019608,
   0.8431372549019608,
   0.8529411764705882,
   0.8529411764705882,
   0.8823529411764706,
   0.8823529411764706,
   0.8921568627450981,
   0.8921568627450981,
   0.9019607843137255,
   0.9019607843137255,
   0.9117647058823529,
   0.9117647058823529,
   0.9215686274509803,
   0.9215686274509803,
   0.9313725490196079,
   0.9313725490196079,
   0.9411764705882353,
   0.9411764705882353,
   0.9509803921568627,
   0.9509803921568627,
   0.9607843137254902,
   0.9607843137254902,
   0.9705882352941176,
   0.9705882352941176,
   0.9803921568627451,
   0.9803921568627451,
   0.9901960784313726,
   0.9901960784313726,
   1.0,
   1.0
  ]
 },
 "pr": {
  "recall": [
   1.0,
   1.0,
   1.0,
   1.0,
   1.0,
   1.0,
   1.0,
   0.9901960784313726,
   0.9901960784313726,
   0.9901960784313726,
   0.9901960784313726,
   0.9901960784313726,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9803921568627451,
   0.9705882352941176,
   0.9705882352941176,
   0.9705882352941176,
   0.9705882352941176,
   0.9705882352941176,
   0.9705882352941176,
   0.9705882352941176,
   0.9705882352941176,
   0.9705882352941176,
   0.9607843137254902,
   0.9607843137254902,
   0.9607843137254902,
   0.9607843137254902,
   0.9607843137254902,
   0.9607843137254902,
   0.9509803921568627,
   0.9509803921568627,
   0.9509803921568627,
   0.9509803921568627,
   0.9509803921568627,
   0.9411764705882353,
   0.9411764705882353,
   0.9411764705882353,
   0.9411764705882353,
   0.9411764705882353,
   0.9411764705882353,
   0.9411764705882353,
   0.9313725490196079,
   0.9313725490196079,
   0.9313725490196079,
   0.9215686274509803,
   0.9215686274509803,
   0.9117647058823529,
   0.9117647058823529,
   0.9019607843137255,
   0.9019607843137255,
   0.9019607843137255,
   0.9019607843137255,
   0.9019607843137255,
   0.9019607843137255,
   0.8921568627450981,
   0.8921568627450981,
   0.8823529411764706,
   0.8823529411764706,
   0.8823529411764706,
   0.8725490196078431,
   0.8627450980392157,
   0.8529411764705882,
   0.8529411764705882,
   0.8431372549019608,
   0.8431372549019608,
   0.8431372549019608,
   0.8333333333333334,
   0.8235294117647058,
   0.8137254901960784,
   0.803921568627451,
   0.803921568627451,
   0.7941176470588235,
   0.7941176470588235,
   0.7941176470588235,
   0.7843137254901961,
   0.7843137254901961,
   0.7745098039215687,
   0.7745098039215687,
   0.7647058823529411,
   0.7647058823529411,
   0.7549019607843137,
   0.7549019607843137,
   0.7549019607843137,
   0.7549019607843137,
   0.7549019607843137,
   0.7450980392156863,
   0.7352941176470589,
   0.7352941176470589,
   0.7254901960784313,
   0.7254901960784313,
   0.7156862745098039,
   0.7058823529411765,
   0.7058823529411765,
   0.696078431372549,
   0.6862745098039216,
   0.6764705882352942,
   0.6666666666666666,
   0.6568627450980392,
   0.6470588235294118,
   0.6372549019607843,
   0.6274509803921569,
   0.6274509803921569,
   0.6274509803921569,
   0.6274509803921569,
   0.6274509803921569,
   0.6176470588235294,
   0.6078431372549019,
   0.5980392156862745,
   0.5882352941176471,
   0.5784313725490197,
   0.5686274509803921,
   0.5588235294117647,
   0.5490196078431373,
   0.5490196078431373,
   0.5392156862745098,
   0.5294117647058824,
   0.5196078431372549,
   0.5098039215686274,
   0.5,
   0.49019607843137253,
   0.4803921568627451,
   0.47058823529411764,
   0.47058823529411764,
   0.46078431372549017,
   0.45098039215686275,
   0.4411764705882353,
   0.43137254901960786,
   0.4215686274509804,
   0.4117647058823529,
   0.4117647058823529,
   0.4117647058823529,
   0.4019607843137255,
   0.39215686274509803,
   0.38235294117647056,
   0.37254901960784315,
   0.3627450980392157,
   0.35294117647058826,
   0.3431372549019608,
   0.3333333333333333,
   0.3235294117647059,
   0.3137254901960784,
   0.30392156862745096,
   0.29411764705882354,
   0.28431372549019607,
   0.27450980392156865,
   0.2647058823529412,
   0.2549019607843137,
   0.24509803921568626,
   0.23529411764705882,
   0.22549019607843138,
   0.21568627450980393,
   0.20588235294117646,
   0.19607843137254902,
   0.18627450980392157,
   0.17647058823529413,
   0.16666666666666666,
   0.1568627450980392,
   0.14705882352941177,
   0.13725490196078433,
   0.12745098039215685,
   0.11764705882352941,
   0.10784313725490197,
   0.09803921568627451,
   0.08823529411764706,
   0.0784313725490196,
   0.06862745098039216,
   0.06862745098039216,
   0.058823529411764705,
   0.049019607843137254,
   0.0392156862745098,
   0.029411764705882353,
   0.0196078431372549,
   0.00980392156862745,
   0.0
  ],
  "precision": [
   0.5543478260869565,
   0.5573770491803278,
   0.5604395604395604,
   0.56353591160221,
   0.5666666666666667,
   0.5698324022346368,
   0.5730337078651685,
   0.5706214689265536,
   0.5738636363636364,
   0.5771428571428572,
   0.5804597701149425,
   0.5838150289017341,
   0.5813953488372093,
   0.5847953216374269,
   0.5882352941176471,
   0.591715976331361,
   0.5952380952380952,
   0.5988023952095808,
   0.6024096385542169,
   0.6060606060606061,
   0.6097560975609756,
   0.6134969325153374,
   0.6172839506172839,
   0.6211180124223602,
   0.625,
   0.6226415094339622,
   0.6265822784810127,
   0.6305732484076433,
   0.6346153846153846,
   0.6387096774193548,
   0.6428571428571429,
   0.6470588235294118,
   0.6513157894736842,
   0.6556291390728477,
   0.6533333333333333,
   0.6577181208053692,
   0.6621621621621622,
   0.6666666666666666,
   0.6712328767123288,
   0.6758620689655173,
   0.6736111111111112,
   0.6783216783216783,
   0.6830985915492958,
   0.6879432624113475,
   0.6928571428571428,
   0.6906474820143885,
   0.6956521739130435,
   0.7007299270072993,
   0.7058823529411765,
   0.7111111111111111,
   0.7164179104477612,
   0.7218045112781954,
   0.7196969696969697,
   0.7251908396946565,
   0.7307692307692307,
   0.7286821705426356,
   0.734375,
   0.7322834645669292,
   0.7380952380952381,
   0.736,
   0.7419354838709677,
   0.7479674796747967,
   0.7540983606557377,
   0.7603305785123967,
   0.7666666666666667,
   0.7647058823529411,
   0.7711864406779662,
   0.7692307692307693,
   0.7758620689655172,
   0.782608695652174,
   0.7807017543859649,
   0.7787610619469026,
   0.7767857142857143,
   0.7837837837837838,
   0.7818181818181819,
   0.7889908256880734,
   0.7962962962962963,
   0.794392523364486,
   0.7924528301886793,
   0.7904761904761904,
   0.7884615384615384,
   0.7961165048543689,
   0.7941176470588235,
   0.801980198019802,
   0.81,
   0.8080808080808081,
   0.8163265306122449,
   0.8144329896907216,
   0.8229166666666666,
   0.8210526315789474,
   0.8297872340425532,
   0.8279569892473119,
   0.8369565217391305,
   0.8461538461538461,
   0.8555555555555555,
   0.8651685393258427,
   0.8636363636363636,
   0.8620689655172413,
   0.872093023255814,
   0.8705882352941177,
   0.8809523809523809,
   0.8795180722891566,
   0.8780487804878049,
   0.8888888888888888,
   0.8875,
   0.8860759493670886,
   0.8846153846153846,
   0.8831168831168831,
   0.881578947368421,
   0.88,
   0.8783783783783784,
   0.8767123287671232,
   0.8888888888888888,
   0.9014084507042254,
   0.9142857142857143,
   0.927536231884058,
   0.9264705882352942,
   0.9253731343283582,
   0.9242424242424242,
   0.9230769230769231,
   0.921875,
   0.9206349206349206,
   0.9193548387096774,
   0.9180327868852459,
   0.9333333333333333,
   0.9322033898305084,
   0.9310344827586207,
   0.9298245614035088,
   0.9285714285714286,
   0.9272727272727272,
   0.9259259259259259,
   0.9245283018867925,
   0.9230769230769231,
   0.9411764705882353,
   0.94,
   0.9387755102040817,
   0.9375,
   0.9361702127659575,
   0.9347826086956522,
   0.9333333333333333,
   0.9545454545454546,
   0.9767441860465116,
   0.9761904761904762,
   0.975609756097561,
   0.975,
   0.9743589743589743,
   0.9736842105263158,
   0.972972972972973,
   0.9722222222222222,
   0.9714285714285714,
   0.9705882352941176,
   0.9696969696969697,
   0.96875,
   0.967741935483871,
   0.9666666666666667,
   0.9655172413793104,
   0.9642857142857143,
   0.9629629629629629,
   0.9615384615384616,
   0.96,
   0.9583333333333334,
   0.9565217391304348,
   0.9545454545454546,
   0.9523809523809523,
   0.95,
   0.9473684210526315,
   0.9444444444444444,
   0.9411764705882353,
   0.9375,
   0.9333333333333333,
   0.9285714285714286,
   0.9230769230769231,
   0.9166666666666666,
   0.9090909090909091,
   0.9,
   0.8888888888888888,
   0.875,
   1.0,
   1.0,
   1.0,
   1.0,
   1.0,
   1.0,
   1.0,
   1.0
  ]
 },
 "calibration": [
  {
   "predicted": 0.07985748735005446,
   "observed": 0.0967741935483871,
   "n": 31
  },
  {
   "predicted": 0.22922035561717113,
   "observed": 0.22580645161290322,
   "n": 31
  },
  {
   "predicted": 0.46888199171115896,
   "observed": 0.5,
   "n": 30
  },
  {
   "predicted": 0.6888746162738209,
   "observed": 0.6774193548387096,
   "n": 31
  },
  {
   "predicted": 0.8670513635440807,
   "observed": 0.8666666666666667,
   "n": 30
  },
  {
   "predicted": 0.9517632168004259,
   "observed": 0.967741935483871,
   "n": 31
  }
 ],
 "train_scores": {
  "edges": [
   0.0,
   0.05,
   0.1,
   0.15,
   0.2,
   0.25,
   0.3,
   0.35,
   0.4,
   0.45,
   0.5,
   0.55,
   0.6,
   0.65,
   0.7,
   0.75,
   0.8,
   0.85,
   0.9,
   0.95,
   1.0
  ],
  "counts": [
   34,
   53,
   37,
   35,
   31,
   27,
   26,
   24,
   22,
   17,
   32,
   35,
   23,
   34,
   28,
   49,
   45,
   49,
   82,
   53
  ],
  "counts_disease": [
   0,
   4,
   3,
   3,
   3,
   6,
   13,
   10,
   11,
   11,
   17,
   20,
   15,
   22,
   21,
   39,
   41,
   44,
   73,
   51
  ],
  "n": 736,
  "note": "Out-of-fold training scores, binned; no patient rows."
 },
 "cv": [
  {
   "key": "dummy_majority",
   "label": "Majority-class dummy",
   "chosen": false,
   "metrics": {
    "accuracy": {
     "mean": 0.55298768155911,
     "std": 0.002688092205552067,
     "folds": [
      0.5540540540540541,
      0.5510204081632653,
      0.5510204081632653,
      0.5510204081632653,
      0.5578231292517006
     ]
    },
    "precision": {
     "mean": 0.55298768155911,
     "std": 0.002688092205552067,
     "folds": [
      0.5540540540540541,
      0.5510204081632653,
      0.5510204081632653,
      0.5510204081632653,
      0.5578231292517006
     ]
    },
    "recall": {
     "mean": 1.0,
     "std": 0.0,
     "folds": [
      1.0,
      1.0,
      1.0,
      1.0,
      1.0
     ]
    },
    "f1": {
     "mean": 0.712155926173893,
     "std": 0.002225527932351438,
     "folds": [
      0.7130434782608696,
      0.7105263157894737,
      0.7105263157894737,
      0.7105263157894737,
      0.7161572052401747
     ]
    },
    "roc_auc": {
     "mean": 0.5,
     "std": 0.0,
     "folds": [
      0.5,
      0.5,
      0.5,
      0.5,
      0.5
     ]
    }
   }
  },
  {
   "key": "logistic_regression",
   "label": "Logistic regression",
   "chosen": true,
   "metrics": {
    "accuracy": {
     "mean": 0.793482257767972,
     "std": 0.022558739562055938,
     "folds": [
      0.7905405405405406,
      0.7891156462585034,
      0.8163265306122449,
      0.8163265306122449,
      0.7551020408163265
     ]
    },
    "precision": {
     "mean": 0.7989457831325301,
     "std": 0.018913985076087575,
     "folds": [
      0.8072289156626506,
      0.8125,
      0.8068181818181818,
      0.8068181818181818,
      0.7613636363636364
     ]
    },
    "recall": {
     "mean": 0.837940379403794,
     "std": 0.031967004736662405,
     "folds": [
      0.8170731707317073,
      0.8024691358024691,
      0.8765432098765432,
      0.8765432098765432,
      0.8170731707317073
     ]
    },
    "f1": {
     "mean": 0.8176566590337984,
     "std": 0.020100145881450315,
     "folds": [
      0.8121212121212121,
      0.8074534161490683,
      0.8402366863905325,
      0.8402366863905325,
      0.788235294117647
     ]
    },
    "roc_auc": {
     "mean": 0.8759976879489075,
     "std": 0.007264333086419248,
     "folds": [
      0.8878418329637842,
      0.877104377104377,
      0.873924429479985,
      0.865132809577254,
      0.8759849906191369
     ]
    }
   }
  },
  {
   "key": "random_forest",
   "label": "Random forest",
   "chosen": false,
   "metrics": {
    "accuracy": {
     "mean": 0.7961481890053319,
     "std": 0.019648448082642843,
     "folds": [
      0.831081081081081,
      0.7755102040816326,
      0.7891156462585034,
      0.8027210884353742,
      0.782312925170068
     ]
    },
    "precision": {
     "mean": 0.7960531921823654,
     "std": 0.02356198466402039,
     "folds": [
      0.8352941176470589,
      0.7790697674418605,
      0.7976190476190477,
      0.8023255813953488,
      0.7659574468085106
     ]
    },
    "recall": {
     "mean": 0.8500150557061126,
     "std": 0.02041953418761472,
     "folds": [
      0.8658536585365854,
      0.8271604938271605,
      0.8271604938271605,
      0.8518518518518519,
      0.8780487804878049
     ]
    },
    "f1": {
     "mean": 0.821868989294139,
     "std": 0.01622458703939755,
     "folds": [
      0.8502994011976048,
      0.8023952095808383,
      0.8121212121212121,
      0.8263473053892215,
      0.8181818181818182
     ]
    },
    "roc_auc": {
     "mean": 0.8703266210854286,
     "std": 0.012328953266609564,
     "folds": [
      0.8898743532889875,
      0.8558735503179947,
      0.8624205013093902,
      0.8645716423494201,
      0.8788930581613508
     ]
    }
   }
  },
  {
   "key": "xgboost",
   "label": "XGBoost",
   "chosen": false,
   "metrics": {
    "accuracy": {
     "mean": 0.8069957712814855,
     "std": 0.0281004436954349,
     "folds": [
      0.8581081081081081,
      0.7755102040816326,
      0.8027210884353742,
      0.8095238095238095,
      0.7891156462585034
     ]
    },
    "precision": {
     "mean": 0.8137956350288029,
     "std": 0.029054966046586195,
     "folds": [
      0.8674698795180723,
      0.8,
      0.8095238095238095,
      0.8117647058823529,
      0.7802197802197802
     ]
    },
    "recall": {
     "mean": 0.8450767841011745,
     "std": 0.030382996864809778,
     "folds": [
      0.8780487804878049,
      0.7901234567901234,
      0.8395061728395061,
      0.8518518518518519,
      0.8658536585365854
     ]
    },
    "f1": {
     "mean": 0.8288270605260102,
     "std": 0.02514336479113761,
     "folds": [
      0.8727272727272727,
      0.7950310559006211,
      0.8242424242424242,
      0.8313253012048193,
      0.8208092485549133
     ]
    },
    "roc_auc": {
     "mean": 0.8653932978594225,
     "std": 0.015616685959143624,
     "folds": [
      0.8898743532889875,
      0.8430602319491208,
      0.867564534231201,
      0.8705574261129817,
      0.8559099437148218
     ]
    }
   }
  }
 ],
 "metric_labels": {
  "roc_auc": "ROC-AUC",
  "accuracy": "Accuracy",
  "precision": "Precision",
  "recall": "Recall",
  "f1": "F1"
 },
 "loho": [
  {
   "hospital": "Cleveland",
   "n": 304,
   "positives": 139,
   "negatives": 165,
   "auc": 0.8719424460431655,
   "recall": 0.935251798561151,
   "precision": 0.625,
   "specificity": 0.5272727272727272
  },
  {
   "hospital": "Hungary",
   "n": 293,
   "positives": 106,
   "negatives": 187,
   "auc": 0.88356371708203,
   "recall": 0.9056603773584906,
   "precision": 0.5925925925925926,
   "specificity": 0.6470588235294118
  },
  {
   "hospital": "Switzerland",
   "n": 123,
   "positives": 115,
   "negatives": 8,
   "auc": 0.7250000000000001,
   "recall": 0.8434782608695652,
   "precision": 0.97,
   "specificity": 0.625
  },
  {
   "hospital": "VA Long Beach",
   "n": 200,
   "positives": 149,
   "negatives": 51,
   "auc": 0.74667719436768,
   "recall": 0.8926174496644296,
   "precision": 0.8109756097560976,
   "specificity": 0.392156862745098
  }
 ],
 "importance": [
  {
   "feature": "cp",
   "label": "Chest pain type",
   "value": 0.8416924925895514
  },
  {
   "feature": "oldpeak",
   "label": "ST depression after exercise (oldpeak)",
   "value": 0.5117848074596055
  },
  {
   "feature": "sex",
   "label": "Sex",
   "value": 0.4163478083569889
  },
  {
   "feature": "thalch",
   "label": "Maximum heart rate",
   "value": 0.3369133995885446
  },
  {
   "feature": "exang",
   "label": "Exercise-induced angina",
   "value": 0.322493962229176
  },
  {
   "feature": "age",
   "label": "Age",
   "value": 0.2444458695727379
  },
  {
   "feature": "chol",
   "label": "Cholesterol",
   "value": 0.1083854526456157
  },
  {
   "feature": "trestbps",
   "label": "Resting blood pressure",
   "value": 0.0540768505526408
  },
  {
   "feature": "restecg",
   "label": "Resting ECG result",
   "value": 0.0122722448011254
  },
  {
   "feature": "fbs",
   "label": "Fasting blood sugar above 120 mg/dl",
   "value": 0.0
  }
 ],
 "oof_auc": 0.8777473245558353,
 "leakage_checks": {
  "label_shuffle_test_auc": 0.4833492746692173,
  "train_test_auc_gap": 0.014041090859675087,
  "oof_vs_test_auc_gap": 0.006154785101028959
 },
 "summary": {
  "rows": 920,
  "columns_raw": 16,
  "prevalence": 0.5532608695652174,
  "pct_male": 0.7891304347826087,
  "n_train": 736,
  "n_test": 184,
  "test_positives": 102,
  "test_negatives": 82,
  "by_hospital": {
   "Cleveland": {
    "n": 304,
    "prevalence": 0.45723684210526316,
    "chol_zero": 0
   },
   "Hungary": {
    "n": 293,
    "prevalence": 0.36177474402730375,
    "chol_zero": 0
   },
   "Switzerland": {
    "n": 123,
    "prevalence": 0.9349593495934959,
    "chol_zero": 123
   },
   "VA Long Beach": {
    "n": 200,
    "prevalence": 0.745,
    "chol_zero": 49
   }
  },
  "chol_zero": 172,
  "trestbps_zero": 1,
  "missing_pct_dropped": {
   "ca": 0.6641304347826087,
   "thal": 0.5282608695652173,
   "slope": 0.3358695652173913
  },
  "missing_pct_kept": {
   "age": 0.0,
   "trestbps": 0.06521739130434782,
   "chol": 0.21956521739130436,
   "thalch": 0.059782608695652176,
   "oldpeak": 0.06739130434782609,
   "fbs": 0.09782608695652174,
   "exang": 0.059782608695652176,
   "restecg": 0.002173913043478261
  },
  "train_range": {
   "age": [
    28.0,
    77.0
   ],
   "trestbps": [
    80.0,
    200.0
   ],
   "chol": [
    85.0,
    603.0
   ],
   "thalch": [
    60.0,
    202.0
   ],
   "oldpeak": [
    -2.6,
    6.2
   ]
  },
  "era": "1980s",
  "source": "UCI Heart Disease dataset (Cleveland, Hungary, VA Long Beach, Switzerland)"
 },
 "glance": {
  "age": {
   "edges": [
    25,
    30,
    35,
    40,
    45,
    50,
    55,
    60,
    65,
    70,
    75,
    80
   ],
   "no_disease": [
    4,
    11,
    39,
    66,
    61,
    94,
    68,
    37,
    22,
    7,
    2
   ],
   "disease": [
    0,
    6,
    20,
    32,
    53,
    86,
    127,
    113,
    50,
    17,
    5
   ]
  },
  "missing": {
   "hospitals": [
    "Cleveland",
    "Hungary",
    "Switzerland",
    "VA Long Beach"
   ],
   "kept": {
    "age": {
     "Cleveland": 0.0,
     "Hungary": 0.0,
     "Switzerland": 0.0,
     "VA Long Beach": 0.0
    },
    "sex": {
     "Cleveland": 0.0,
     "Hungary": 0.0,
     "Switzerland": 0.0,
     "VA Long Beach": 0.0
    },
    "cp": {
     "Cleveland": 0.0,
     "Hungary": 0.0,
     "Switzerland": 0.0,
     "VA Long Beach": 0.0
    },
    "trestbps": {
     "Cleveland": 0.0,
     "Hungary": 0.0034129692832764505,
     "Switzerland": 0.016260162601626018,
     "VA Long Beach": 0.285
    },
    "chol": {
     "Cleveland": 0.0,
     "Hungary": 0.07849829351535836,
     "Switzerland": 1.0,
     "VA Long Beach": 0.28
    },
    "fbs": {
     "Cleveland": 0.0,
     "Hungary": 0.027303754266211604,
     "Switzerland": 0.6097560975609756,
     "VA Long Beach": 0.035
    },
    "restecg": {
     "Cleveland": 0.0,
     "Hungary": 0.0034129692832764505,
     "Switzerland": 0.008130081300813009,
     "VA Long Beach": 0.0
    },
    "thalch": {
     "Cleveland": 0.0,
     "Hungary": 0.0034129692832764505,
     "Switzerland": 0.008130081300813009,
     "VA Long Beach": 0.265
    },
    "exang": {
     "Cleveland": 0.0,
     "Hungary": 0.0034129692832764505,
     "Switzerland": 0.008130081300813009,
     "VA Long Beach": 0.265
    },
    "oldpeak": {
     "Cleveland": 0.0,
     "Hungary": 0.0,
     "Switzerland": 0.04878048780487805,
     "VA Long Beach": 0.28
    }
   },
   "dropped": {
    "ca": {
     "Cleveland": 0.01644736842105263,
     "Hungary": 0.9897610921501706,
     "Switzerland": 0.959349593495935,
     "VA Long Beach": 0.99
    },
    "thal": {
     "Cleveland": 0.009868421052631578,
     "Hungary": 0.9044368600682594,
     "Switzerland": 0.42276422764227645,
     "VA Long Beach": 0.83
    },
    "slope": {
     "Cleveland": 0.003289473684210526,
     "Hungary": 0.6450511945392492,
     "Switzerland": 0.13821138211382114,
     "VA Long Beach": 0.51
    }
   }
  },
  "balance": {
   "Cleveland": {
    "no_disease": 165,
    "disease": 139
   },
   "Hungary": {
    "no_disease": 187,
    "disease": 106
   },
   "Switzerland": {
    "no_disease": 8,
    "disease": 115
   },
   "VA Long Beach": {
    "no_disease": 51,
    "disease": 149
   }
  },
  "chest_pain": {
   "asymptomatic": {
    "n": 496,
    "disease": 392
   },
   "atypical angina": {
    "n": 174,
    "disease": 24
   },
   "non-anginal": {
    "n": 204,
    "disease": 73
   },
   "typical angina": {
    "n": 46,
    "disease": 20
   }
  }
 },
 "method": {
  "cv_folds": 5,
  "bootstrap_models": 200,
  "bootstrap_resamples": 1000,
  "split": {
   "train": 736,
   "test": 184,
   "stratified_on": "outcome and hospital"
  },
  "tuned": {
   "logistic_regression": {
    "nested_auc_mean": 0.875,
    "nested_auc_std": 0.0191
   },
   "xgboost": {
    "nested_auc_mean": 0.8794,
    "nested_auc_std": 0.0142
   }
  },
  "label_shuffle_auc": 0.4833492746692173,
  "shuffles": 30,
  "train_test_auc_gap": 0.014041090859675087,
  "steps": [
   "Clean the data: zero cholesterol and zero blood pressure are treated as missing; three mostly empty columns and the row id are dropped.",
   "Split first: the data is divided into training and test sets before any filling in, scaling or encoding, so nothing is learned from the test set.",
   "Fit one pipeline on the training set only: median or most-frequent fill-in, scaling, one-hot encoding, then logistic regression.",
   "Compare four models with 5-fold cross-validation on the training set, then tune the two best with cross-validation only.",
   "Choose the decision threshold from cross-validated training predictions so that at least 90% of true cases are caught.",
   "Evaluate once on the held-out test set, then check leave-one-hospital-out performance and a shuffled-label leakage test."
  ]
 },
 "citation": {
  "authors": "Janosi, A., Steinbrunn, W., Pfisterer, M., Detrano, R.",
  "title": "Heart Disease",
  "publisher": "UCI Machine Learning Repository",
  "year": "1988",
  "url": "https://archive.ics.uci.edu/dataset/45/heart+disease"
 },
 "intervals": {
  "auc": {
   "point": 0.8715925394548063,
   "lo": 0.8169184287609008,
   "hi": 0.9207031250000001
  },
  "average_precision": {
   "point": 0.8911054764237094,
   "lo": 0.8227039552923122,
   "hi": 0.9427172499127178
  },
  "recall": {
   "point": 0.8823529411764706,
   "lo": 0.8124845297029702,
   "hi": 0.9406086196854979
  },
  "specificity": {
   "point": 0.6829268292682927,
   "lo": 0.5783132530120482,
   "hi": 0.7777777777777778
  },
  "precision": {
   "point": 0.7758620689655172,
   "lo": 0.7024612150210237,
   "hi": 0.8508771929824561
  },
  "_meta": {
   "resamples": 1000,
   "valid_resamples": 1000,
   "seed": 42,
   "n_test": 184,
   "method": "percentile interval of 1,000 bootstrap resamples of the held-out test set"
  }
 },
 "subgroups": {
  "rows": [
   {
    "kind": "Sex",
    "group": "Female",
    "n": 35,
    "n_pos": 9,
    "n_neg": 26,
    "tp": 4,
    "fn": 5,
    "fp": 2,
    "tn": 24,
    "recall": 0.4444444444444444,
    "auc": {
     "point": 0.7564102564102564,
     "lo": 0.5439,
     "hi": 0.9212962962962963
    },
    "too_few": true,
    "recall_interval": {
     "point": 0.4444444444444444,
     "lo": 0.125,
     "hi": 0.7694444444444443
    }
   },
   {
    "kind": "Sex",
    "group": "Male",
    "n": 149,
    "n_pos": 93,
    "n_neg": 56,
    "tp": 86,
    "fn": 7,
    "fp": 24,
    "tn": 32,
    "recall": 0.9247311827956989,
    "auc": {
     "point": 0.8703917050691244,
     "lo": 0.8114529732904089,
     "hi": 0.923805490376383
    },
    "too_few": false,
    "recall_interval": {
     "point": 0.9247311827956989,
     "lo": 0.8636079545454546,
     "hi": 0.9756171025565678
    }
   },
   {
    "kind": "Age band",
    "group": "Under 45",
    "n": 38,
    "n_pos": 17,
    "n_neg": 21,
    "tp": 15,
    "fn": 2,
    "fp": 3,
    "tn": 18,
    "recall": 0.8823529411764706,
    "auc": {
     "point": 0.9663865546218486,
     "lo": 0.9090909090909091,
     "hi": 1.0
    },
    "too_few": false,
    "recall_interval": {
     "point": 0.8823529411764706,
     "lo": 0.7141369047619048,
     "hi": 1.0
    }
   },
   {
    "kind": "Age band",
    "group": "45 to 59",
    "n": 91,
    "n_pos": 46,
    "n_neg": 45,
    "tp": 39,
    "fn": 7,
    "fp": 12,
    "tn": 33,
    "recall": 0.8478260869565217,
    "auc": {
     "point": 0.8478260869565217,
     "lo": 0.7540585012371862,
     "hi": 0.9292705578909304
    },
    "too_few": false,
    "recall_interval": {
     "point": 0.8478260869565217,
     "lo": 0.7441711389385808,
     "hi": 0.9411764705882353
    }
   },
   {
    "kind": "Age band",
    "group": "60 and over",
    "n": 55,
    "n_pos": 39,
    "n_neg": 16,
    "tp": 36,
    "fn": 3,
    "fp": 11,
    "tn": 5,
    "recall": 0.9230769230769231,
    "auc": {
     "point": 0.7980769230769231,
     "lo": 0.6748352498894293,
     "hi": 0.905936080037492
    },
    "too_few": false,
    "recall_interval": {
     "point": 0.9230769230769231,
     "lo": 0.8332142857142858,
     "hi": 1.0
    }
   },
   {
    "kind": "Hospital",
    "group": "Cleveland",
    "n": 61,
    "n_pos": 28,
    "n_neg": 33,
    "tp": 24,
    "fn": 4,
    "fp": 9,
    "tn": 24,
    "recall": 0.8571428571428571,
    "auc": {
     "point": 0.8982683982683983,
     "lo": 0.7833081896551723,
     "hi": 0.9730617931961438
    },
    "too_few": false,
    "recall_interval": {
     "point": 0.8571428571428571,
     "lo": 0.7,
     "hi": 0.9666666666666667
    }
   },
   {
    "kind": "Hospital",
    "group": "Hungary",
    "n": 58,
    "n_pos": 21,
    "n_neg": 37,
    "tp": 19,
    "fn": 2,
    "fp": 9,
    "tn": 28,
    "recall": 0.9047619047619048,
    "auc": {
     "point": 0.9034749034749034,
     "lo": 0.800216649159664,
     "hi": 0.9837775298589503
    },
    "too_few": false,
    "recall_interval": {
     "point": 0.9047619047619048,
     "lo": 0.7616071428571428,
     "hi": 1.0
    }
   },
   {
    "kind": "Hospital",
    "group": "Switzerland",
    "n": 25,
    "n_pos": 23,
    "n_neg": 2,
    "tp": 21,
    "fn": 2,
    "fp": 1,
    "tn": 1,
    "recall": 0.9130434782608695,
    "auc": {
     "point": 0.7391304347826086,
     "lo": 0.375,
     "hi": 1.0
    },
    "too_few": true,
    "recall_interval": {
     "point": 0.9130434782608695,
     "lo": 0.782608695652174,
     "hi": 1.0
    }
   },
   {
    "kind": "Hospital",
    "group": "VA Long Beach",
    "n": 40,
    "n_pos": 30,
    "n_neg": 10,
    "tp": 26,
    "fn": 4,
    "fp": 7,
    "tn": 3,
    "recall": 0.8666666666666667,
    "auc": {
     "point": 0.6866666666666668,
     "lo": 0.4882215541731975,
     "hi": 0.8572142857142857
    },
    "too_few": false,
    "recall_interval": {
     "point": 0.8666666666666667,
     "lo": 0.7272727272727273,
     "hi": 0.96875
    }
   }
  ],
  "min_patients": 30,
  "min_per_outcome": 10,
  "resamples": 1000,
  "seed": 42,
  "note": "Aggregates only: counts and metrics per group, no patient-level features."
 },
 "baseline": {
  "features": [
   "age",
   "sex"
  ],
  "model": "logistic regression on age and sex only",
  "threshold": 0.3534675885239235,
  "threshold_rule": "same rule as the main model: highest threshold keeping at least 90% recall in training cross-validation",
  "auc": {
   "point": 0.6778455284552846,
   "lo": 0.6012623727039117,
   "hi": 0.7563709677419355
  },
  "recall": {
   "point": 0.9117647058823529,
   "lo": 0.8529411764705882,
   "hi": 0.9626168224299065
  },
  "specificity": {
   "point": 0.2926829268292683,
   "lo": 0.2,
   "hi": 0.3924163652802893
  },
  "tp": 93,
  "fn": 9,
  "fp": 58,
  "tn": 24,
  "auc_difference": {
   "point": 0.19374701099952174,
   "lo": 0.11992106763935872,
   "hi": 0.2655013355531666
  },
  "note": "Same training split, same test set, same 1,000 bootstrap resamples as the main model."
 },
 "reliability": {
  "bands": [
   {
    "label": "Low",
    "lo_edge": 0.0,
    "hi_edge": 0.24467766128077445,
    "n": 50,
    "positives": 6,
    "rate": 0.12,
    "interval": {
     "point": 0.12,
     "lo": 0.036347402597402594,
     "hi": 0.22239583333333315
    }
   },
   {
    "label": "Moderate",
    "lo_edge": 0.24467766128077445,
    "hi_edge": 0.38828193438013014,
    "n": 18,
    "positives": 6,
    "rate": 0.3333333333333333,
    "interval": {
     "point": 0.3333333333333333,
     "lo": 0.11994117647058823,
     "hi": 0.5833333333333334
    }
   },
   {
    "label": "High",
    "lo_edge": 0.38828193438013014,
    "hi_edge": 1.0,
    "n": 116,
    "positives": 90,
    "rate": 0.7758620689655172,
    "interval": {
     "point": 0.7758620689655172,
     "lo": 0.7024612150210237,
     "hi": 0.8508771929824561
    }
   }
  ],
  "bins": [
   {
    "label": "0% to 20%",
    "lo_edge": 0.0,
    "hi_edge": 0.2,
    "n": 42,
    "positives": 5,
    "rate": 0.11904761904761904,
    "interval": {
     "point": 0.11904761904761904,
     "lo": 0.024984756097560977,
     "hi": 0.23533281733746125
    }
   },
   {
    "label": "20% to 40%",
    "lo_edge": 0.2,
    "hi_edge": 0.4,
    "n": 28,
    "positives": 8,
    "rate": 0.2857142857142857,
    "interval": {
     "point": 0.2857142857142857,
     "lo": 0.1332608695652174,
     "hi": 0.47619047619047616
    }
   },
   {
    "label": "40% to 60%",
    "lo_edge": 0.4,
    "hi_edge": 0.6,
    "n": 26,
    "positives": 13,
    "rate": 0.5,
    "interval": {
     "point": 0.5,
     "lo": 0.3076086956521739,
     "hi": 0.7045788770053476
    }
   },
   {
    "label": "60% to 80%",
    "lo_edge": 0.6,
    "hi_edge": 0.8,
    "n": 29,
    "positives": 21,
    "rate": 0.7241379310344828,
    "interval": {
     "point": 0.7241379310344828,
     "lo": 0.5624375,
     "hi": 0.875
    }
   },
   {
    "label": "80% to 100%",
    "lo_edge": 0.8,
    "hi_edge": 1.0,
    "n": 59,
    "positives": 55,
    "rate": 0.9322033898305084,
    "interval": {
     "point": 0.9322033898305084,
     "lo": 0.8595864661654136,
     "hi": 0.9843810096153847
    }
   }
  ],
  "resamples": 1000,
  "seed": 42,
  "method": "95% percentile interval of 1,000 bootstrap resamples of the held-out test set"
 },
 "data_version": "2026-09-30-899a98fc"
}